Bash Script for Web Downloading

etsi_clone_latest.py

import os
import re
import requests
from bs4 import BeautifulSoup
from urllib.parse import urljoin

# Config
START_URL = "https://www.etsi.org/deliver/etsi_ts/"
DOWNLOAD_DIR = "./pdfs"
REQ_TIMEOUT = (10, 30)
CHUNK_SIZE = 8192

# Standard headers to bypass firewalls
HEADERS = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
    'Accept-Language': 'en-US,en;q=0.5',
    'Connection': 'keep-alive',
}

# Regex for version parsing
VERSION_PARSE_RE = re.compile(r'[._\-]')

# Persistent session
session = requests.Session()
session.headers.update(HEADERS)

def _parse_version(version_str: str):
    """Parses '19.00.00_60' into (19, 0, 0, 60) for accurate sorting."""
    parts = VERSION_PARSE_RE.split(version_str)
    numeric_parts = []
    for part in parts:
        try:
            numeric_parts.append(int(part))
        except ValueError:
            numeric_parts.append(part)
    return tuple(numeric_parts)

def get_soup(url: str):
    """Fetches and parses HTML with timeout handling."""
    print(f"  -> [Network] Scanning: {url}")
    try:
        response = session.get(url, timeout=REQ_TIMEOUT)
        response.raise_for_status()
        return BeautifulSoup(response.text, 'html.parser')
    except requests.exceptions.Timeout:
        print(f"[!] Server timed out on: {url}")
    except requests.exceptions.HTTPError as e:
        print(f"[!] HTTP Error {response.status_code} on: {url}")
    except Exception as e:
        print(f"[!] Unexpected error [{url}]: {e}")
    return None

def is_valid_dir_link(link_element) -> bool:
    """Helper to ignore parent links and non-directories."""
    href = link_element.get('href', '')
    text = link_element.text
    return href.endswith('/') and '[To Parent Directory]' not in text and '..' not in href

def crawl_and_download():
    os.makedirs(DOWNLOAD_DIR, exist_ok=True)
    print(f"Starting targeted ETSI clone process from: {START_URL}")
    print("-" * 50)

    # L1: Scope ranges (e.g. 122200_122299/)
    soup_l1 = get_soup(START_URL)
    if not soup_l1: return

    for link_l1 in soup_l1.find_all('a', href=True):
        if not is_valid_dir_link(link_l1): continue
        url_l2 = urljoin(START_URL, link_l1['href'])
        
        soup_l2 = get_soup(url_l2)
        if not soup_l2: continue

        # L2: Document IDs (e.g. 122279/)
        for link_l2 in soup_l2.find_all('a', href=True):
            if not is_valid_dir_link(link_l2): continue
            url_l3 = urljoin(url_l2, link_l2['href'])
            
            soup_l3 = get_soup(url_l3)
            if not soup_l3: continue

            # L3: Version directories (e.g. 19.00.00_60/)
            version_dirs = []
            for link_l3 in soup_l3.find_all('a', href=True):
                if not is_valid_dir_link(link_l3): continue
                
                # Extract just the folder name from the absolute path
                href_l3 = link_l3['href']
                folder_name = os.path.basename(href_l3.rstrip('/'))
                
                # Ensure it starts with a digit (validates it is a version string)
                if folder_name and folder_name[0].isdigit():
                    version_dirs.append(folder_name)

            if not version_dirs: continue

            # Sort to find the latest version
            try:
                version_dirs.sort(key=_parse_version)
            except TypeError:
                version_dirs.sort()

            latest_version = version_dirs[-1]
            target_version_url = urljoin(url_l3, f"{latest_version}/")
            
            # Fetch target PDFs in the latest directory
            soup_pdf = get_soup(target_version_url)
            if not soup_pdf: continue

            for pdf_link in soup_pdf.find_all('a', href=True):
                pdf_href = pdf_link['href']
                
                if pdf_href.lower().endswith('.pdf'):
                    pdf_url = urljoin(target_version_url, pdf_href)
                    filename = os.path.basename(pdf_href)
                    dest_path = os.path.join(DOWNLOAD_DIR, filename)

                    if os.path.exists(dest_path):
                        print(f"  -> [Skip] Already downloaded: {filename}")
                        continue

                    print(f"*** DOWNLOADING [{latest_version}] -> {filename} ***")
                    try:
                        with session.get(pdf_url, stream=True, timeout=REQ_TIMEOUT) as r:
                            r.raise_for_status()
                            with open(dest_path, 'wb') as f:
                                for chunk in r.iter_content(chunk_size=CHUNK_SIZE):
                                    f.write(chunk)
                    except Exception as e:
                        print(f"[!] Error downloading {filename}: {e}")

if __name__ == "__main__":
    crawl_and_download()

mirror.sh

#!/bin/bash

# Mirror specific paths (Docs and Assets) into a local 'offline_docs' directory
httrack \
  "https://www.openproject.org/docs/" \
  -O "./offline_docs" \
  "-*" \
  "+*.openproject.org/docs/*" \
  "+*.openproject.org/assets/*"

# Download only PDF files found within the docs into a 'pdf_files' directory
httrack \
  "https://www.etsi.org/deliver/etsi_ts/" \
  -O "./pdfs" \
  "-*" \
  "+www.etsi.org/deliver/etsi_ts/*" \
  "+*.pdf" \
  --update

# Mirror the entire docs site completely for offline browsing
httrack \
  "https://www.openproject.org/docs/" \
  -O "./full_mirror"


wget \
  --recursive \
  --level=inf \
  --no-directories \
  --no-clobber \
  --accept="*.pdf" \
  --directory-prefix="./pdfs" \
  --execute robots=off \
  "https://www.etsi.org/deliver/etsi_ts/"