Last active
December 22, 2015 00:49
-
-
Save chadgh/6392356 to your computer and use it in GitHub Desktop.
The squid module provides a nice easy way to use threading effectively for I/O bound tasks that you want the results back to do something with.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import threading | |
def tentacle(f, daemon=False): | |
"""Decorator adopted from http://stackoverflow.com/a/14331755/18992 (thanks bj0)""" | |
import Queue | |
def wrapped_f(q, *args, **kwargs): | |
"""Call decorated function and puts the result in a queue.""" | |
ret = f(*args, **kwargs) | |
q.put(ret) | |
def wrap(*args, **kwargs): | |
"""Fire off wrapped_f in a new thread and return the thread object | |
with the result queue attached.""" | |
q = Queue.Queue() | |
t = threading.Thread(target=wrapped_f, args=(q,) + args, kwargs=kwargs) | |
t.daemon = daemon | |
t.start() | |
t.result = q | |
return t | |
return wrap | |
class Squid(object): | |
"""Encapsulate a threading task.""" | |
def __init__(self, action, daemon=False): | |
self.action = action | |
self.daemon = daemon | |
def go(self, call_params): | |
"""Call all the threads with the given params, wait for all to finish, | |
and return the results.""" | |
tentacles = [] | |
for args, kwargs in call_params: | |
func = tentacle(self.action, daemon=self.daemon) | |
tentacles.append(func(*args, **kwargs)) | |
results = [] | |
for t in tentacles: | |
results.append(t.result.get()) | |
return results |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment