process_manager.py

import concurrent.futures


class ProcessManager(object):

    def __init__(self):
        self.pool = concurrent.futures.ProcessPoolExecutor()
        self.futures = {}

    def submit(self, f, *args, **kwargs):
        future = self.pool.submit(f, *args, **kwargs)
        self.futures[future] = (f, args, kwargs)

    def start(self):

        while self.futures:

            res = concurrent.futures.wait(
                self.futures,
                return_when=concurrent.futures.FIRST_EXCEPTION)

            print 'Tasks', len(res.done), len(res.not_done)

            for future in res.done:
                f, args, kwargs = self.futures[future]
                del self.futures[future]

                exc = future.exception()
                if exc is None:
                    self.on_success(future, exc, f, *args, **kwargs)
                else:
                    self.on_error(future, exc, f, *args, **kwargs)
                    print 'Resubmitting'
                    self.submit(f, *args, **kwargs)

    # Overwrite in subclasses

    def on_error(self, future, exc, f, *args, **kwargs):
        print 'Got exception from', future, exc

    def on_success(self, future, exc, f, *args, **kwargs):
        print 'Future finished', future, future.result()


if __name__ == '__main__':
    import random
    import time

    def f(i):

        print 'start'
        while True:
            time.sleep(0.1)
            x = random.random()
            if x < 0.1:
                raise Exception('DEAD')
            elif x < 0.9:
                continue
            else:
                return 'hello world', i

    pm = ProcessManager()
    for i in xrange(5):
        pm.submit(f, i)
    pm.start()