Fontawesome universal scraper

fa-scrape.py

import os
import re
import requests
from urllib.parse import urljoin, urlparse

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 14.5; rv:128.0) Gecko/20100101 Firefox/128.0",
    "Accept": "*/*",
    "Accept-Language": "en-US,en;q=0.5",
    "Referer": "https://site-assets.fontawesome.com/",
}

VERSION = "v7.2.0"
BASE_URL = f"https://site-assets.fontawesome.com/releases/{VERSION}/"
DOWNLOAD_DIR = f"./releases/{VERSION}/"

ENTRY_CSS = [
    "css/all.css",
    "css/utility-semibold.css",
    "css/sharp-solid.css",
    "css/sharp-regular.css",
    "css/sharp-light.css",
    "css/sharp-thin.css",
    "css/sharp-duotone-solid.css",
    "css/duotone.css",
    "css/brands.css",
    "css/solid.css",
    "css/regular.css",
    "css/light.css",
    "css/thin.css",
]

downloaded_files = set()
visited_css = set()

def save(url, filepath):
    r = requests.get(url, headers=HEADERS)
    if r.status_code != 200:
        print(f"  ! {r.status_code} {url}")
        return False
    os.makedirs(os.path.dirname(filepath), exist_ok=True)
    with open(filepath, "wb") as f:
        f.write(r.content)
    return True

def process_css(css_url):
    if css_url in visited_css:
        return
    visited_css.add(css_url)

    r = requests.get(css_url, headers=HEADERS)
    if r.status_code != 200:
        print(f"CSS {r.status_code}: {css_url}")
        return

    rel_path = urlparse(css_url).path.split(f"/releases/{VERSION}/", 1)[-1]
    css_filepath = os.path.join(DOWNLOAD_DIR, rel_path)
    os.makedirs(os.path.dirname(css_filepath), exist_ok=True)
    with open(css_filepath, "wb") as f:
        f.write(r.content)
    print(f"CSS  {rel_path}")

    css_text = r.text

    for imp in re.findall(r"@import\s+(?:url\()?['\"]?([^'\")]+)['\"]?\)?", css_text):
        process_css(urljoin(css_url, imp))

    for raw in re.findall(r"url\(([^)]+)\)", css_text):
        u = raw.strip("'\" ")
        if not (u.endswith(".woff2") or u.endswith(".ttf") or u.endswith(".woff")):
            continue
        font_url = urljoin(css_url, u)
        if font_url in downloaded_files:
            continue
        downloaded_files.add(font_url)
        font_rel = urlparse(font_url).path.split(f"/releases/{VERSION}/", 1)[-1]
        font_path = os.path.join(DOWNLOAD_DIR, font_rel)
        if save(font_url, font_path):
            print(f"FONT {font_rel}")

def main():
    for css in ENTRY_CSS:
        process_css(urljoin(BASE_URL, css))
    print(f"\nDone. {len(downloaded_files)} font files, {len(visited_css)} CSS files.")

if __name__ == "__main__":
    main()