parallelpip demo

core.py

#! -*- Coding: utf-8 -*-

"""
Before running the create a virtualenv (I used version 1.7, so no-site-packages is default), activate virtualenv and install gevent, envoy and try running
"""
import gevent
import time 
from envoy import run
from sys import exit, argv
from gevent import monkey
monkey.patch_all()
import subprocess

def syntax():
    print "Usage:python core.py normal|parallel [libraries]"

def normal_download(lib = None):
    try:
        if lib:
            start = time.time()
            print "normal download started"
            for l in lib:
                print "Trying to install %s"%l
                run("pip install %s"%l)
            return time.time() - start
        else:
            syntax()
            exit()
    except:
        print "Unhandled exception"
        exit()

def parallel_download(lib = None):
    try:
        if lib:
            print "spawning using gevent"
            jobs = [gevent.spawn(pip.call_subprocess, ["pip","install",l]) \
                                                      for l in lib]
            start = time.time()
            print "joined all gevent, d/l started"
            gevent.joinall(jobs)
            for job in jobs:
                print job.value
            return time.time() - start
        else:
            syntax()
            exit()
    except:
        print "unhandled exception"
        exit()

if __name__ == "__main__":
    if argv[1] == 'parallel':
        print(parallel_download(argv[2:])," seconds for parallel d/l")
    elif argv[1] == 'normal':
        print(normal_download(argv[2:]), "seconds for normal d/l")
    else:
        syntax()
        exit()

output.txt

First I used run of envoy then replaced with pip.cal_subprocess.
(parallelpip)kracekumar@python-lover:~/codes/python/asyncpip$ python core.py parallel flask requests
spawning using gevent
joined all gevent, d/l started
Exception in thread Thread-1:
Traceback (most recent call last):
  File "/usr/lib/python2.7/threading.py", line 552, in __bootstrap_inner
    self.run()
  File "/usr/lib/python2.7/threading.py", line 505, in run
    self.__target(*self.__args, **self.__kwargs)
  File "/home/kracekumar/codes/python/asyncpip/parallelpip/local/lib/python2.7/site-packages/envoy/core.py", line 43, in target
    self.out, self.err = self.process.communicate(self.data)
  File "/usr/lib/python2.7/subprocess.py", line 754, in communicate
    return self._communicate(input)
  File "/usr/lib/python2.7/subprocess.py", line 1302, in _communicate
    stdout, stderr = self._communicate_with_poll(input)
  File "/usr/lib/python2.7/subprocess.py", line 1332, in _communicate_with_poll
    poller = select.poll()
AttributeError: 'module' object has no attribute 'poll'

Exception in thread Thread-3:
Traceback (most recent call last):
  File "/usr/lib/python2.7/threading.py", line 552, in __bootstrap_inner
    self.run()
  File "/usr/lib/python2.7/threading.py", line 505, in run
    self.__target(*self.__args, **self.__kwargs)
  File "/home/kracekumar/codes/python/asyncpip/parallelpip/local/lib/python2.7/site-packages/envoy/core.py", line 43, in target
    self.out, self.err = self.process.communicate(self.data)
  File "/usr/lib/python2.7/subprocess.py", line 754, in communicate
    return self._communicate(input)
  File "/usr/lib/python2.7/subprocess.py", line 1302, in _communicate
    stdout, stderr = self._communicate_with_poll(input)
  File "/usr/lib/python2.7/subprocess.py", line 1332, in _communicate_with_poll
    poller = select.poll()
AttributeError: 'module' object has no attribute 'poll'

<Response [pip]>
<Response [pip]>
(0.03878211975097656, ' seconds for parallel d/l')
Exception KeyError: KeyError(3076684868L,) in <module 'threading' from '/usr/lib/python2.7/threading.pyc'> ignored
(parallelpip)kracekumar@python-lover:~/codes/python/asyncpip$ vim core.py (parallelpip)kracekumar@python-lover:~/codes/python/asyncpip$ python core.py parallel flask requests
spawning using gevent
joined all gevent, d/l started
Traceback (most recent call last):
  File "/home/kracekumar/codes/python/asyncpip/parallelpip/local/lib/python2.7/site-packages/gevent/greenlet.py", line 390, in run
    result = self._run(*self.args, **self.kwargs)
  File "/usr/lib/python2.7/subprocess.py", line 679, in __init__
    errread, errwrite)
  File "/usr/lib/python2.7/subprocess.py", line 1239, in _execute_child
    raise child_exception
OSError: [Errno 2] No such file or directory
<Greenlet at 0xb73cb39cL: Popen('pip install flask')> failed with OSError

Traceback (most recent call last):
  File "/home/kracekumar/codes/python/asyncpip/parallelpip/local/lib/python2.7/site-packages/gevent/greenlet.py", line 390, in run
    result = self._run(*self.args, **self.kwargs)
  File "/usr/lib/python2.7/subprocess.py", line 679, in __init__
    errread, errwrite)
  File "/usr/lib/python2.7/subprocess.py", line 1239, in _execute_child
    raise child_exception
OSError: [Errno 2] No such file or directory
<Greenlet at 0xb73cb43cL: Popen('pip install requests')> failed with OSError

None
None
(0.016376972198486328, ' seconds for parallel d/l')
Exception KeyError: KeyError(3074739268L,) in <module 'threading' from '/usr/lib/python2.7/threading.pyc'> ignored
(parallelpip)kracekumar@python-lover:~/codes/python/asyncpip$ vim core.py 
(parallelpip)kracekumar@python-lover:~/codes/python/asyncpip$ python core.py parallel flask requests
spawning using gevent
unhandled exception
Exception KeyError: KeyError(3076037700L,) in <module 'threading' from '/usr/lib/python2.7/threading.pyc'> ignored