Download CSV with generation of energy from transparency.entsoe.eu for countries (draft)

scrape-entsoe.py

import requests
import urllib.parse
from bs4 import BeautifulSoup

r = requests.get("https://transparency.entsoe.eu/generation/r2/actualGenerationPerProductionType/show?name=&defaultValue=true&viewType=TABLE&areaType=CTY&atch=true&datepicker-day-offset-select-dv-date-from_input=D&dateTime.dateTime=06.10.2015+00%3A00%7CCET%7CDAYTIMERANGE&dateTime.endDateTime=06.10.2015+00%3A00%7CCET%7CDAYTIMERANGE&area.values=CTY%7C10Y1001A1001A83F!BZN%7C10YDOM-CZ-DE-SKK&productionType.values=B01&productionType.values=B25&productionType.values=B02&productionType.values=B03&productionType.values=B04&productionType.values=B05&productionType.values=B06&productionType.values=B07&productionType.values=B08&productionType.values=B09&productionType.values=B10&productionType.values=B11&productionType.values=B12&productionType.values=B13&productionType.values=B14&productionType.values=B20&productionType.values=B15&productionType.values=B16&productionType.values=B17&productionType.values=B18&productionType.values=B19&dateTime.timezone=CET_CEST&dateTime.timezone_input=CET+(UTC%2B1)+%2F+CEST+(UTC%2B2)&_=1729271269023")

soup = BeautifulSoup(r.content, "html.parser")

countryCheckboxes = soup.select("#dv-market-areas-content .dv-filter-hierarchic-wrapper > .dv-filter-checkbox")

countryToAreaCode = []
for checkbox in countryCheckboxes:
    value = checkbox.find('input')['value']
    areaCode = value.replace('|SINGLE', '')
    country = (checkbox.text.strip(), areaCode)
    #print(country)
    countryToAreaCode.append(country)

#print(countryToAreaCode)

for countryName, areaCode in countryToAreaCode[1:5]:
    print(f"Downloading data for {countryName} ({areaCode})")
    # Download link for 
    # 'https://transparency.entsoe.eu/generation/r2/actualGenerationPerProductionType/export?name=&defaultValue=false&viewType=TABLE&areaType=CTY&atch=true&datepicker-day-offset-select-dv-date-from_input=D&dateTime.dateTime=06.10.2015+00%3A00%7CCET%7CDAYTIMERANGE&dateTime.endDateTime=06.10.2015+00%3A00%7CCET%7CDAYTIMERANGE&area.values=CTY%7C10Y1001A1001A83F!CTY%7C10Y1001A1001A83F&productionType.values=B01&productionType.values=B25&productionType.values=B02&productionType.values=B03&productionType.values=B04&productionType.values=B05&productionType.values=B06&productionType.values=B07&productionType.values=B08&productionType.values=B09&productionType.values=B10&productionType.values=B11&productionType.values=B12&productionType.values=B13&productionType.values=B14&productionType.values=B20&productionType.values=B15&productionType.values=B16&productionType.values=B17&productionType.values=B18&productionType.values=B19&dateTime.timezone=CET_CEST&dateTime.timezone_input=CET+(UTC%2B1)+%2F+CEST+(UTC%2B2)&dataItem=ALL&timeRange=YEAR&exportType=CSV',
    
    areaCodeUriEncoded = urllib.parse.quote(areaCode)
    
    # Fill your values. You can get them from the browser's developer tools after you login
    cookies = {
    'SESSION': '<your-value>', 
    'OAuth_Token_Request_State': '<your-value>',    
    'currentUrl': 'https://transparency.entsoe.eu/generation/r2/actualGenerationPerProductionType/show',    
    'JSESSIONID': '<your-value>',
    }

    headers = {
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
        'Accept-Language': 'en,cs-CZ;q=0.9,cs;q=0.8',
        'Connection': 'keep-alive',
        # 'Cookie': 'Filter|time_zone=CET_CEST; emfip-welcome=true; Filter|MBA=CTY|10YBE----------2!MBA|10YBE----------2; Filter|contract_type_di=A01; SESSION=NmU4M2M5MzUtYzliYy00NTljLWJiYjEtNmVlNzEyZjdiN2Jk; OAuth_Token_Request_State=5abc6fb6-d912-4557-a20d-3c9580849384; Filter|CTA=CTY|10YFR-RTE------C!CTA|10YFR-RTE------C; Filter|CTY=CTY|10Y1001A1001A83F!CTY|10Y1001A1001A83F; currentUrl=https://transparency.entsoe.eu/generation/r2/actualGenerationPerProductionType/show; Filter|date_time=12.10.2016.00.00; Filter|date_time_end=13.10.2016.00.00; Filter|BZN=CTY|10Y1001A1001A39I!BZN|10Y1001A1001A39I; JSESSIONID=6F651873C09917F72600654989E9C4A8',
        'Referer': 'https://transparency.entsoe.eu/generation/r2/actualGenerationPerProductionType/show?name=&defaultValue=true&viewType=TABLE&areaType=CTY&atch=true&datepicker-day-offset-select-dv-date-from_input=D&dateTime.dateTime=06.10.2015+00%3A00%7CCET%7CDAYTIMERANGE&dateTime.endDateTime=06.10.2015+00%3A00%7CCET%7CDAYTIMERANGE&area.values=CTY%7C10Y1001A1001A83F!BZN%7C10YDOM-CZ-DE-SKK&productionType.values=B01&productionType.values=B25&productionType.values=B02&productionType.values=B03&productionType.values=B04&productionType.values=B05&productionType.values=B06&productionType.values=B07&productionType.values=B08&productionType.values=B09&productionType.values=B10&productionType.values=B11&productionType.values=B12&productionType.values=B13&productionType.values=B14&productionType.values=B20&productionType.values=B15&productionType.values=B16&productionType.values=B17&productionType.values=B18&productionType.values=B19&dateTime.timezone=CET_CEST&dateTime.timezone_input=CET+(UTC%2B1)+%2F+CEST+(UTC%2B2)&_=1729271269023',
        'Sec-Fetch-Dest': 'document',
        'Sec-Fetch-Mode': 'navigate',
        'Sec-Fetch-Site': 'same-origin',
        'Sec-Fetch-User': '?1',
        'Upgrade-Insecure-Requests': '1',
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36',
        'sec-ch-ua': '"Google Chrome";v="129", "Not=A?Brand";v="8", "Chromium";v="129"',
        'sec-ch-ua-mobile': '?0',
        'sec-ch-ua-platform': '"Windows"',
    }
    url = f'https://transparency.entsoe.eu/generation/r2/actualGenerationPerProductionType/export?name=&defaultValue=false&viewType=TABLE&areaType=CTY&atch=false&datepicker-day-offset-select-dv-date-from_input=D&dateTime.dateTime=12.10.2016+00%3A00%7CCET%7CDAYTIMERANGE&dateTime.endDateTime=12.10.2016+00%3A00%7CCET%7CDAYTIMERANGE&area.values={areaCodeUriEncoded}!{areaCodeUriEncoded}&productionType.values=B01&productionType.values=B25&productionType.values=B02&productionType.values=B03&productionType.values=B04&productionType.values=B05&productionType.values=B06&productionType.values=B07&productionType.values=B08&productionType.values=B09&productionType.values=B10&productionType.values=B11&productionType.values=B12&productionType.values=B13&productionType.values=B14&productionType.values=B20&productionType.values=B15&productionType.values=B16&productionType.values=B17&productionType.values=B18&productionType.values=B19&dateTime.timezone=CET_CEST&dateTime.timezone_input=CET+(UTC%2B1)+%2F+CEST+(UTC%2B2)&dataItem=ALL&timeRange=YEAR&exportType=CSV'
    print(url)
    r = requests.get(url, cookies=cookies, headers=headers, allow_redirects=True)
    open(f"{countryName}.csv", "wb").write(r.content)