Skip to content

Instantly share code, notes, and snippets.

@PathToLife
Created April 17, 2020 18:14
Show Gist options
  • Save PathToLife/e7945b11d07c2032b2ed6d5e0b753834 to your computer and use it in GitHub Desktop.
Save PathToLife/e7945b11d07c2032b2ed6d5e0b753834 to your computer and use it in GitHub Desktop.
Simple Python3 Multithreaded Downloading with auto directory creation and incomplete download file marking. Multiprocessing pools
import os
import requests
from time import time as timer
from typing import List, Tuple
from multiprocessing.pool import ThreadPool
import shutil
def download_file(entry: Tuple[str, str]):
path, uri = entry
dirname = os.path.dirname(path)
if not os.path.exists(dirname):
try:
os.makedirs(dirname)
except FileExistsError:
pass
if not os.path.exists(path):
r = requests.get(uri, stream=True)
if r.status_code == 200:
with open(path + '.crdownload', 'wb') as f:
for chunk in r:
f.write(chunk)
shutil.move(path + '.crdownload', path)
return path
def bulk_download(path_url_list: List[Tuple[str, str]]):
start = timer()
print(f"Downloading {len(path_url_list)} files...")
results = ThreadPool(8).imap_unordered(download_file, path_url_list)
for res in results:
print(res)
print(f"Elapsed Time: {timer() - start}")
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment