Download MT data

download_mt.py

import requests
import pandas as pd
import geopandas as gpd
import sqlite3
from shapely.geometry import Point

#
#Download data for Trentino Province from meteoreport.
#
#

REGION_ID = "fba93146-7192-4190-adab-605435fdeea1"
GPKG_PATH = "meteo_trentino.gpkg"

VENUES_URL = "https://gitlab.com/tinia-euregio/tinia-website/-/raw/main/data/venues/en/{i}.json"
OBS_URL = "https://meteo.report/var/data/observations/{layout_id}.json"


#
#3000000 == 02:00 to 02:59 A.M.
#
START_LAYOUT_ID = 3000360
STEP_MINUTES = 30
DAYS_BACK = 6
N_STEPS = int(DAYS_BACK* 24 * 60 / STEP_MINUTES)


def as_list(data):
    if isinstance(data, list):
        return data
    if isinstance(data, dict):
        for key in ["venues", "stations", "data", "items"]:
            if key in data and isinstance(data[key], list):
                return data[key]
        return list(data.values())
    return []


def get_coord(station, names):
    for name in names:
        if name in station and station[name] is not None:
            return float(station[name])

    if "coordinates" in station:
        coords = station["coordinates"]
        if isinstance(coords, list) and len(coords) >= 2:
            if "lon" in names or "longitude" in names:
                return float(coords[0])
            return float(coords[1])

    if "geometry" in station and "coordinates" in station["geometry"]:
        coords = station["geometry"]["coordinates"]
        if "lon" in names or "longitude" in names:
            return float(coords[0])
        return float(coords[1])

    return None


def download_stations():
    rows = []

    for i in range(1, 8):
        url = VENUES_URL.format(i=i)
        r = requests.get(url, timeout=30)
        r.raise_for_status()

        data = r.json()

        for s in as_list(data):
            if not isinstance(s, dict):
                continue

            if s.get("id_region") != REGION_ID:
                continue

            station_id = s.get("id")
            if station_id is None:
                continue

            lon = get_coord(s, ["lon", "longitude", "lng", "x"])
            lat = get_coord(s, ["lat", "latitude", "y"])

            rows.append({
                "id": str(station_id),
                "name": s.get("name") or s.get("title") or str(station_id),
                "id_region": s.get("id_region"),
                "elevation": s.get("elevation") or s.get("altitude") or s.get("height"),
                "source_file": f"{i}.json",
                "geometry": Point(lon, lat) if lon is not None and lat is not None else None
            })

    gdf = gpd.GeoDataFrame(rows, geometry="geometry", crs="EPSG:4326")
    gdf = gdf.drop_duplicates(subset=["id"])

    gdf.to_file(GPKG_PATH, layer="stations", driver="GPKG")

    return set(gdf["id"].astype(str))


def download_observations(station_ids):
    rows = []

    for k in range(N_STEPS):
        layout_id = START_LAYOUT_ID - STEP_MINUTES * k
        url = OBS_URL.format(layout_id=layout_id)

        try:
            r = requests.get(url, timeout=30)
            r.raise_for_status()
            data = r.json()
        except Exception as e:
            print(f"Skip {layout_id}: {e}")
            continue

        start = data.get("start")
        end = data.get("end")

        for station_id, values in data.items():
            if station_id in ["start", "end"]:
                continue

            station_id = str(station_id)

            if station_id not in station_ids:
                continue

            if not isinstance(values, dict):
                continue

            for variable, value in values.items():
                rows.append({
                    "layout_id": layout_id,
                    "start_time": start,
                    "end_time": end,
                    "station_id": station_id,
                    "variable": variable,
                    "value": value
                })

    df = pd.DataFrame(rows)

    with sqlite3.connect(GPKG_PATH) as conn:
        df.to_sql("observations", conn, if_exists="replace", index=False)

        conn.execute("""
            CREATE INDEX IF NOT EXISTS idx_observations_station_time
            ON observations (station_id, start_time, variable);
        """)

        conn.execute("""
            CREATE INDEX IF NOT EXISTS idx_observations_variable_time
            ON observations (variable, start_time);
        """)

    return df


if __name__ == "__main__":
    station_ids = download_stations()
    print(f"Stations saved: {len(station_ids)}")

    obs = download_observations(station_ids)
    print(f"Observation records saved: {len(obs)}")