Collecting maintainer user names from PyPI using a txt file of URLs

maintainer_scrape.py

import requests
from bs4 import BeautifulSoup
import csv
import time

def scrape_user_links(url):
    """
    Scrape all hyperlinks with href starting with /user/ from a PyPI page.
    
    Args:
        url: Full URL to the PyPI package page
        
    Returns:
        List of dictionaries with link text and URLs
    """
    try:
        response = requests.get(url, timeout=10)
        response.raise_for_status()
        
        soup = BeautifulSoup(response.content, 'html.parser')
        
        # Find all links with href starting with /user/
        user_links = []
        for link in soup.find_all('a', href=True):
            href = link['href']
            if href.startswith('/user/'):
                link_text = link.get_text(strip=True)
                full_url = f"https://pypi.org{href}"
                user_links.append({
                    'text': link_text,
                    'href': href,
                    'full_url': full_url
                })
        
        return user_links
        
    except requests.exceptions.RequestException as e:
        print(f"Error fetching {url}: {e}")
        return []

def main():
    # Read URLs from text file
    try:
        with open('urls.txt', 'r') as f:
            urls = [line.strip() for line in f if line.strip()]
    except FileNotFoundError:
        print("Error: urls.txt file not found!")
        print("Please create a urls.txt file with one URL per line.")
        return
    
    if not urls:
        print("No URLs found in urls.txt")
        return
    
    print(f"Loaded {len(urls)} URLs from urls.txt")
    
    all_results = []
    
    for url in urls:
        print(f"Scraping {url}...")
        user_links = scrape_user_links(url)
        
        for link_info in user_links:
            all_results.append({
                'source_url': url,
                'link_text': link_info['text'],
                'href': link_info['href'],
                'full_url': link_info['full_url']
            })
        
        print(f"  Found {len(user_links)} user links")
        
        # Be polite - add a small delay between requests
        time.sleep(1)
    
    # Save results to CSV
    if all_results:
        with open('user_links.csv', 'w', newline='', encoding='utf-8') as f:
            writer = csv.DictWriter(f, fieldnames=['source_url', 'link_text', 'href', 'full_url'])
            writer.writeheader()
            writer.writerows(all_results)
        
        print(f"\nFound {len(all_results)} total user links")
        print("Results saved to user_links.csv")
    else:
        print("No user links found")
    
    # Print unique users found
    unique_users = set(r['link_text'] for r in all_results)
    print(f"Unique users: {len(unique_users)}")

if __name__ == "__main__":
    main()