Create new requirements.txt file with pinned versions based on the latest version at a specific date

update_requirements.py

import requests
from datetime import datetime, timezone # <-- Add timezone here
from packaging.version import parse as parse_version
from packaging.specifiers import SpecifierSet
import sys
import time

def get_latest_version_before_date(package_name, target_date_str, target_python_version_str=None):
    """
    Fetches the latest version of a package available on PyPI before a target date,
    optionally checking for Python compatibility.
    """
    # 1. Make target_date timezone-aware (UTC) at the beginning of the day
    # This assumes your TARGET_DATE (e.g., '2018-06-30') means 2018-06-30 00:00:00 UTC
    target_date = datetime.strptime(target_date_str, '%Y-%m-%d').replace(tzinfo=timezone.utc)

    url = f"https://pypi.org/pypi/{package_name}/json"

    try:
        response = requests.get(url, timeout=10)
        response.raise_for_status()
        data = response.json()
    except requests.exceptions.RequestException as e:
        print(f"Error fetching {package_name}: {e}", file=sys.stderr)
        return None

    latest_suitable_version = None
    latest_suitable_upload_time = datetime.min.replace(tzinfo=timezone.utc) # Ensure this is also aware

    releases = data.get('releases', {})
    sorted_versions = sorted(releases.keys(), key=parse_version, reverse=True)

    for version_str in sorted_versions:
        try:
            version = parse_version(version_str)
            if version.is_prerelease or version.is_devrelease:
                continue

            for release_info in releases[version_str]:
                upload_time_str = release_info.get('upload_time_iso_8601')
                if not upload_time_str:
                    continue

                # Handle different ISO 8601 formats (e.g., 'Z' for UTC)
                if upload_time_str.endswith('Z'):
                    upload_time_str = upload_time_str[:-1] + '+00:00'
                
                # 2. Parse upload_time and convert it to UTC for consistent comparison
                try:
                    upload_time = datetime.fromisoformat(upload_time_str).astimezone(timezone.utc)
                except ValueError as e:
                    # This can happen if the ISO string is malformed or truly has no offset
                    # PyPI generally provides good ISO 8601 strings, but defensive coding is good.
                    print(f"Warning: Malformed upload_time_iso_8601 for {package_name}=={version_str}: '{upload_time_str}' - {e}", file=sys.stderr)
                    continue

                if upload_time <= target_date: # Now this comparison will work
                    # Check Python compatibility
                    if target_python_version_str:
                        requires_python_str = release_info.get('requires_python')
                        if requires_python_str:
                            try:
                                python_specifier = SpecifierSet(requires_python_str)
                                if not python_specifier.contains(target_python_version_str):
                                    continue
                            except Exception as e:
                                print(f"Warning: Could not parse requires_python '{requires_python_str}' for {package_name}=={version_str}: {e}", file=sys.stderr)
                                pass

                    if upload_time > latest_suitable_upload_time:
                        latest_suitable_upload_time = upload_time
                        latest_suitable_version = version_str
                    elif upload_time == latest_suitable_upload_time and parse_version(version_str) > parse_version(latest_suitable_version or '0.0.0'):
                         latest_suitable_version = version_str


        except Exception as e:
            # Catching general exceptions within the loop for robustness, though ValueError is handled above
            print(f"Warning: Could not process version '{version_str}' for {package_name}: {e}", file=sys.stderr)
            continue

    return latest_suitable_version

if __name__ == "__main__":
    # --- Configuration ---
    ORIGINAL_REQUIREMENTS_FILE = 'requirements.txt'
    NEW_REQUIREMENTS_FILE = 'requirements_pinned.txt'
    TARGET_DATE = '2018-06-30' # YYYY-MM-DD: The date you want to "freeze" to
    TARGET_PYTHON_VERSION = '3.6'
    # --- End Configuration ---

    print(f"Attempting to pin dependencies from '{ORIGINAL_REQUIREMENTS_FILE}' to '{TARGET_DATE}'")
    if TARGET_PYTHON_VERSION:
        print(f"Filtering for Python compatibility: {TARGET_PYTHON_VERSION}")

    pinned_dependencies = []
    
    try:
        with open(ORIGINAL_REQUIREMENTS_FILE, 'r') as f_in:
            for line in f_in:
                line = line.strip()
                if not line or line.startswith('#'):
                    if line: # Preserve comments and empty lines
                        pinned_dependencies.append(line)
                    continue

                package_name = line.split('==')[0].split('<')[0].split('>')[0].split('~')[0].strip()
                
                if package_name:
                    print(f"Processing: {package_name}...")
                    time.sleep(0.1) 
                    
                    pinned_version = get_latest_version_before_date(package_name, TARGET_DATE, TARGET_PYTHON_VERSION)
                    if pinned_version:
                        pinned_dependencies.append(f"{package_name}=={pinned_version}")
                        print(f"  Found: {package_name}=={pinned_version}")
                    else:
                        pinned_dependencies.append(line)
                        print(f"  Warning: Could not find suitable version for {package_name} before {TARGET_DATE}. Keeping original line.", file=sys.stderr)
                else:
                    pinned_dependencies.append(line)

    except FileNotFoundError:
        print(f"Error: '{ORIGINAL_REQUIREMENTS_FILE}' not found.", file=sys.stderr)
        sys.exit(1)
    except Exception as e:
        print(f"An unexpected error occurred: {e}", file=sys.stderr)
        sys.exit(1)

    with open(NEW_REQUIREMENTS_FILE, 'w') as f_out:
        for dep_line in pinned_dependencies:
            f_out.write(dep_line + '\n')

    print(f"\nPinned dependencies written to '{NEW_REQUIREMENTS_FILE}'.")
    print("Please review the generated file, especially for warnings about packages where a version couldn't be found.")