thedivtagguy · June 3, 2025 12:58
diff --git a/MOVIEOFTHENIGHT_SCRAPER.md b/MOVIEOFTHENIGHT_SCRAPER.md
diff --git a/scraper.py b/scraper.py
 import requests
 import json
 import time
 import os

 # Configuration
 URL = "https://test.movieofthenight.com/api/v2/catalog/bulk"
 HEADERS = {
    "Content-Type": "application/json",
    "X-Motn-Language": "en",
    "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:139.0) Gecko/20100101 Firefox/139.0"
 }

 GENRES = ["action", "adventure", "animation", "comedy", "crime", "documentary", 
          "drama", "family", "fantasy", "history", "horror", "music", "mystery", 
          "reality", "romance", "scifi", "talk", "thriller", "war", "western"]
 SHOW_TYPE = "movie"
 ORDER_BY = "rating"
 CATALOG = "netflix.subscription"
 COUNTRY = "us"
 MIN_YEAR = "2000"
 LIMIT = 100

 def make_request(genre, cursor=None):
    params = {
        "orderBy": ORDER_BY,
        "showType": SHOW_TYPE, 
        "andGenres": genre,
        "catalogs": CATALOG,
        "country": COUNTRY,
        "minYear": MIN_YEAR,
        "limit": str(LIMIT)
    }
    
    if cursor:
        params["cursor"] = cursor
    
    action = {
        "id": os.urandom(16).hex(),
        "type": "searchByFilters", 
        "params": params
    }
    
    try:
        response = requests.post(URL, json={"actions": [action]}, headers=HEADERS, timeout=30)
        if response.status_code == 200:
            return response.json()
        else:
            print(f"Error {response.status_code}")
            return None
    except Exception as e:
        print(f"Request failed: {e}")
        return None

 def scrape_genre(genre):
    all_pages = []
    page_num = 1
    cursor = None
    
    while True:
        print(f"Scraping {genre} page {page_num}...")
        
        data = make_request(genre, cursor)
        if not data:
            break
            
        all_pages.append(data)
        
        # Check if there are more pages
        batch = data[0]
        result = batch["result"]
        has_more = result.get("hasMore", False)
        cursor = result.get("nextCursor")
        
        if not has_more or not cursor:
            break
            
        page_num += 1
        time.sleep(0.5)  # Be nice to the server
    
    return all_pages

 def main():
    all_data = {}
    
    # Scrape each genre
    for genre in GENRES:
        print(f"\n--- Starting {genre} ---")
        all_data[genre] = scrape_genre(genre)
    
    # Save to file
    timestamp = int(time.time())
    filename = f"motn_{SHOW_TYPE}_{CATALOG.split('.')[0]}_{timestamp}.json"
    
    with open(filename, "w") as file:
        json.dump(all_data, file, indent=2)
    
    # Count total items
    total_items = 0
    for genre_data in all_data.values():
        for page in genre_data:
            batch = page[0]
            items = batch["result"]["items"]
            total_items += len(items)
    
    print(f"\nDone! Total {SHOW_TYPE}s: {total_items}")
    print(f"Saved to: {filename}")

 if __name__ == "__main__":
    main()
	import requests
	import json
	import time
	import os

	# Configuration
	URL = "https://test.movieofthenight.com/api/v2/catalog/bulk"
	HEADERS = {
	"Content-Type": "application/json",
	"X-Motn-Language": "en",
	"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:139.0) Gecko/20100101 Firefox/139.0"
	}

	GENRES = ["action", "adventure", "animation", "comedy", "crime", "documentary",
	"drama", "family", "fantasy", "history", "horror", "music", "mystery",
	"reality", "romance", "scifi", "talk", "thriller", "war", "western"]
	SHOW_TYPE = "movie"
	ORDER_BY = "rating"
	CATALOG = "netflix.subscription"
	COUNTRY = "us"
	MIN_YEAR = "2000"
	LIMIT = 100

	def make_request(genre, cursor=None):
	params = {
	"orderBy": ORDER_BY,
	"showType": SHOW_TYPE,
	"andGenres": genre,
	"catalogs": CATALOG,
	"country": COUNTRY,
	"minYear": MIN_YEAR,
	"limit": str(LIMIT)
	}

	if cursor:
	params["cursor"] = cursor

	action = {
	"id": os.urandom(16).hex(),
	"type": "searchByFilters",
	"params": params
	}

	try:
	response = requests.post(URL, json={"actions": [action]}, headers=HEADERS, timeout=30)
	if response.status_code == 200:
	return response.json()
	else:
	print(f"Error {response.status_code}")
	return None
	except Exception as e:
	print(f"Request failed: {e}")
	return None

	def scrape_genre(genre):
	all_pages = []
	page_num = 1
	cursor = None

	while True:
	print(f"Scraping {genre} page {page_num}...")

	data = make_request(genre, cursor)
	if not data:
	break

	all_pages.append(data)

	# Check if there are more pages
	batch = data[0]
	result = batch["result"]
	has_more = result.get("hasMore", False)
	cursor = result.get("nextCursor")

	if not has_more or not cursor:
	break

	page_num += 1
	time.sleep(0.5) # Be nice to the server

	return all_pages

	def main():
	all_data = {}

	# Scrape each genre
	for genre in GENRES:
	print(f"\n--- Starting {genre} ---")
	all_data[genre] = scrape_genre(genre)

	# Save to file
	timestamp = int(time.time())
	filename = f"motn_{SHOW_TYPE}_{CATALOG.split('.')[0]}_{timestamp}.json"

	with open(filename, "w") as file:
	json.dump(all_data, file, indent=2)

	# Count total items
	total_items = 0
	for genre_data in all_data.values():
	for page in genre_data:
	batch = page[0]
	items = batch["result"]["items"]
	total_items += len(items)

	print(f"\nDone! Total {SHOW_TYPE}s: {total_items}")
	print(f"Saved to: {filename}")

	if __name__ == "__main__":
	main()