Skip to content

Instantly share code, notes, and snippets.

@joshainglis
Last active August 29, 2015 14:09
Show Gist options
  • Select an option

  • Save joshainglis/73ed56bf5df5dd195c2f to your computer and use it in GitHub Desktop.

Select an option

Save joshainglis/73ed56bf5df5dd195c2f to your computer and use it in GitHub Desktop.
Modified example from https://docs.python.org/3/library/concurrent.futures.html#threadpoolexecutor-example to wait for all threads to complete.
import concurrent.futures
import urllib.request
URLS = ['http://www.foxnews.com/',
'http://www.cnn.com/',
'http://europe.wsj.com/',
'http://www.bbc.co.uk/',
'http://some-made-up-domain.com/']
# Retrieve a single page and report the url and contents
def load_url(url, timeout):
conn = urllib.request.urlopen(url, timeout=timeout)
return conn.readall()
# We can use a with statement to ensure threads are cleaned up promptly
with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
# Start the load operations and mark each future with its URL
future_to_url = {executor.submit(load_url, url, 60): url for url in URLS}
# returned_futures contains 2 sets: returned_futures.done and returned_futures.not_done
# (not_done should be empty if we wait for ALL_COMPLETED)
returned_futures = concurrent.futures.wait(future_to_url, return_when=concurrent.futures.ALL_COMPLETED)
for future in returned_futures.done:
url = future_to_url[future]
try:
data = future.result()
except Exception as exc:
print('%r generated an exception: %s' % (url, exc))
else:
print('%r page is %d bytes' % (url, len(data)))
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment