Last active
May 7, 2026 21:09
-
-
Save berstend/26932c8cac013c530e20dcdf0eedbc6a to your computer and use it in GitHub Desktop.
Create new requirements.txt file with pinned versions based on the latest version at a specific date
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| import requests | |
| from datetime import datetime, timezone # <-- Add timezone here | |
| from packaging.version import parse as parse_version | |
| from packaging.specifiers import SpecifierSet | |
| import sys | |
| import time | |
| def get_latest_version_before_date(package_name, target_date_str, target_python_version_str=None): | |
| """ | |
| Fetches the latest version of a package available on PyPI before a target date, | |
| optionally checking for Python compatibility. | |
| """ | |
| # 1. Make target_date timezone-aware (UTC) at the beginning of the day | |
| # This assumes your TARGET_DATE (e.g., '2018-06-30') means 2018-06-30 00:00:00 UTC | |
| target_date = datetime.strptime(target_date_str, '%Y-%m-%d').replace(tzinfo=timezone.utc) | |
| url = f"https://pypi.org/pypi/{package_name}/json" | |
| try: | |
| response = requests.get(url, timeout=10) | |
| response.raise_for_status() | |
| data = response.json() | |
| except requests.exceptions.RequestException as e: | |
| print(f"Error fetching {package_name}: {e}", file=sys.stderr) | |
| return None | |
| latest_suitable_version = None | |
| latest_suitable_upload_time = datetime.min.replace(tzinfo=timezone.utc) # Ensure this is also aware | |
| releases = data.get('releases', {}) | |
| sorted_versions = sorted(releases.keys(), key=parse_version, reverse=True) | |
| for version_str in sorted_versions: | |
| try: | |
| version = parse_version(version_str) | |
| if version.is_prerelease or version.is_devrelease: | |
| continue | |
| for release_info in releases[version_str]: | |
| upload_time_str = release_info.get('upload_time_iso_8601') | |
| if not upload_time_str: | |
| continue | |
| # Handle different ISO 8601 formats (e.g., 'Z' for UTC) | |
| if upload_time_str.endswith('Z'): | |
| upload_time_str = upload_time_str[:-1] + '+00:00' | |
| # 2. Parse upload_time and convert it to UTC for consistent comparison | |
| try: | |
| upload_time = datetime.fromisoformat(upload_time_str).astimezone(timezone.utc) | |
| except ValueError as e: | |
| # This can happen if the ISO string is malformed or truly has no offset | |
| # PyPI generally provides good ISO 8601 strings, but defensive coding is good. | |
| print(f"Warning: Malformed upload_time_iso_8601 for {package_name}=={version_str}: '{upload_time_str}' - {e}", file=sys.stderr) | |
| continue | |
| if upload_time <= target_date: # Now this comparison will work | |
| # Check Python compatibility | |
| if target_python_version_str: | |
| requires_python_str = release_info.get('requires_python') | |
| if requires_python_str: | |
| try: | |
| python_specifier = SpecifierSet(requires_python_str) | |
| if not python_specifier.contains(target_python_version_str): | |
| continue | |
| except Exception as e: | |
| print(f"Warning: Could not parse requires_python '{requires_python_str}' for {package_name}=={version_str}: {e}", file=sys.stderr) | |
| pass | |
| if upload_time > latest_suitable_upload_time: | |
| latest_suitable_upload_time = upload_time | |
| latest_suitable_version = version_str | |
| elif upload_time == latest_suitable_upload_time and parse_version(version_str) > parse_version(latest_suitable_version or '0.0.0'): | |
| latest_suitable_version = version_str | |
| except Exception as e: | |
| # Catching general exceptions within the loop for robustness, though ValueError is handled above | |
| print(f"Warning: Could not process version '{version_str}' for {package_name}: {e}", file=sys.stderr) | |
| continue | |
| return latest_suitable_version | |
| if __name__ == "__main__": | |
| # --- Configuration --- | |
| ORIGINAL_REQUIREMENTS_FILE = 'requirements.txt' | |
| NEW_REQUIREMENTS_FILE = 'requirements_pinned.txt' | |
| TARGET_DATE = '2018-06-30' # YYYY-MM-DD: The date you want to "freeze" to | |
| TARGET_PYTHON_VERSION = '3.6' | |
| # --- End Configuration --- | |
| print(f"Attempting to pin dependencies from '{ORIGINAL_REQUIREMENTS_FILE}' to '{TARGET_DATE}'") | |
| if TARGET_PYTHON_VERSION: | |
| print(f"Filtering for Python compatibility: {TARGET_PYTHON_VERSION}") | |
| pinned_dependencies = [] | |
| try: | |
| with open(ORIGINAL_REQUIREMENTS_FILE, 'r') as f_in: | |
| for line in f_in: | |
| line = line.strip() | |
| if not line or line.startswith('#'): | |
| if line: # Preserve comments and empty lines | |
| pinned_dependencies.append(line) | |
| continue | |
| package_name = line.split('==')[0].split('<')[0].split('>')[0].split('~')[0].strip() | |
| if package_name: | |
| print(f"Processing: {package_name}...") | |
| time.sleep(0.1) | |
| pinned_version = get_latest_version_before_date(package_name, TARGET_DATE, TARGET_PYTHON_VERSION) | |
| if pinned_version: | |
| pinned_dependencies.append(f"{package_name}=={pinned_version}") | |
| print(f" Found: {package_name}=={pinned_version}") | |
| else: | |
| pinned_dependencies.append(line) | |
| print(f" Warning: Could not find suitable version for {package_name} before {TARGET_DATE}. Keeping original line.", file=sys.stderr) | |
| else: | |
| pinned_dependencies.append(line) | |
| except FileNotFoundError: | |
| print(f"Error: '{ORIGINAL_REQUIREMENTS_FILE}' not found.", file=sys.stderr) | |
| sys.exit(1) | |
| except Exception as e: | |
| print(f"An unexpected error occurred: {e}", file=sys.stderr) | |
| sys.exit(1) | |
| with open(NEW_REQUIREMENTS_FILE, 'w') as f_out: | |
| for dep_line in pinned_dependencies: | |
| f_out.write(dep_line + '\n') | |
| print(f"\nPinned dependencies written to '{NEW_REQUIREMENTS_FILE}'.") | |
| print("Please review the generated file, especially for warnings about packages where a version couldn't be found.") |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
A fix for some invalid versions: