Skip to content

Instantly share code, notes, and snippets.

@darwing1210
Last active March 9, 2025 15:50
Show Gist options
  • Save darwing1210/c9ff8e3af8ba832e38e6e6e347d9047a to your computer and use it in GitHub Desktop.
Save darwing1210/c9ff8e3af8ba832e38e6e6e347d9047a to your computer and use it in GitHub Desktop.
Script to download files in a async way, using Python asyncio
import os
import asyncio
import aiohttp # pip install aiohttp
import aiofile # pip install aiofile
REPORTS_FOLDER = "reports"
FILES_PATH = os.path.join(REPORTS_FOLDER, "files")
def download_files_from_report(urls):
os.makedirs(FILES_PATH, exist_ok=True)
sema = asyncio.BoundedSemaphore(5)
async def fetch_file(session, url):
fname = url.split("/")[-1]
async with sema:
async with session.get(url) as resp:
assert resp.status == 200
data = await resp.read()
async with aiofile.async_open(
os.path.join(FILES_PATH, fname), "wb"
) as outfile:
await outfile.write(data)
async def main():
async with aiohttp.ClientSession() as session:
tasks = [fetch_file(session, url) for url in urls]
await asyncio.gather(*tasks)
loop = asyncio.get_event_loop()
loop.run_until_complete(main())
loop.close()
@Sarique-Deepsolv
Copy link

does this not work for video files?I am getting error 403.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment