gistfile1.txt

# /// script
# dependencies = [
#     "rp",
#     "nflx-mml-common-fs",
#     "nflx-mml-common-types",
#     "nflx-mml-common-io",
# ]
# ///

import rp
from mml.common.fs import cp

rp.set_current_directory(rp.get_parent_directory(__file__))

csv_path = "maestro_adhoc_dataclient_7ccaec9b-8d4c-4f13-a728-08e3ce9dace0.csv"

csv = rp.load_csv(csv_path, show_progress=True)
urls = csv.studio_algo_baggins_url
paths = list(map(baggins_to_path, urls))


def baggins_to_path(url):
    return url[len("baggins://studio-algo-research/TEST/") :]


def download(pair):
    url, path = pair
    return cp(url, path, recursive=True, threads=4)


pairs = list(zip(urls, paths))

results = rp.load_files(
    download,
    pairs,
    num_threads=32,
    show_progress=True,
    strict=None,
)