Skip to content

Instantly share code, notes, and snippets.

@kracekumar

kracekumar/core.py

Created March 4, 2012 09:13
Download ZIP

Revisions

  1. kracekumar created this gist Mar 4, 2012.
    60 changes: 60 additions & 0 deletions core.py
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,60 @@
    #! -*- Coding: utf-8 -*-

    """
    Before running the create a virtualenv (I used version 1.7, so no-site-packages is default), activate virtualenv and install gevent, envoy and try running
    """
    import gevent
    import time
    from envoy import run
    from sys import exit, argv
    from gevent import monkey
    monkey.patch_all()
    import subprocess

    def syntax():
    print "Usage:python core.py normal|parallel [libraries]"

    def normal_download(lib = None):
    try:
    if lib:
    start = time.time()
    print "normal download started"
    for l in lib:
    print "Trying to install %s"%l
    run("pip install %s"%l)
    return time.time() - start
    else:
    syntax()
    exit()
    except:
    print "Unhandled exception"
    exit()

    def parallel_download(lib = None):
    try:
    if lib:
    print "spawning using gevent"
    jobs = [gevent.spawn(pip.call_subprocess, ["pip","install",l]) \
    for l in lib]
    start = time.time()
    print "joined all gevent, d/l started"
    gevent.joinall(jobs)
    for job in jobs:
    print job.value
    return time.time() - start
    else:
    syntax()
    exit()
    except:
    print "unhandled exception"
    exit()

    if __name__ == "__main__":
    if argv[1] == 'parallel':
    print(parallel_download(argv[2:])," seconds for parallel d/l")
    elif argv[1] == 'normal':
    print(normal_download(argv[2:]), "seconds for normal d/l")
    else:
    syntax()
    exit()

    72 changes: 72 additions & 0 deletions output.txt
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,72 @@
    First I used run of envoy then replaced with pip.cal_subprocess.
    (parallelpip)kracekumar@python-lover:~/codes/python/asyncpip$ python core.py parallel flask requests
    spawning using gevent
    joined all gevent, d/l started
    Exception in thread Thread-1:
    Traceback (most recent call last):
    File "/usr/lib/python2.7/threading.py", line 552, in __bootstrap_inner
    self.run()
    File "/usr/lib/python2.7/threading.py", line 505, in run
    self.__target(*self.__args, **self.__kwargs)
    File "/home/kracekumar/codes/python/asyncpip/parallelpip/local/lib/python2.7/site-packages/envoy/core.py", line 43, in target
    self.out, self.err = self.process.communicate(self.data)
    File "/usr/lib/python2.7/subprocess.py", line 754, in communicate
    return self._communicate(input)
    File "/usr/lib/python2.7/subprocess.py", line 1302, in _communicate
    stdout, stderr = self._communicate_with_poll(input)
    File "/usr/lib/python2.7/subprocess.py", line 1332, in _communicate_with_poll
    poller = select.poll()
    AttributeError: 'module' object has no attribute 'poll'

    Exception in thread Thread-3:
    Traceback (most recent call last):
    File "/usr/lib/python2.7/threading.py", line 552, in __bootstrap_inner
    self.run()
    File "/usr/lib/python2.7/threading.py", line 505, in run
    self.__target(*self.__args, **self.__kwargs)
    File "/home/kracekumar/codes/python/asyncpip/parallelpip/local/lib/python2.7/site-packages/envoy/core.py", line 43, in target
    self.out, self.err = self.process.communicate(self.data)
    File "/usr/lib/python2.7/subprocess.py", line 754, in communicate
    return self._communicate(input)
    File "/usr/lib/python2.7/subprocess.py", line 1302, in _communicate
    stdout, stderr = self._communicate_with_poll(input)
    File "/usr/lib/python2.7/subprocess.py", line 1332, in _communicate_with_poll
    poller = select.poll()
    AttributeError: 'module' object has no attribute 'poll'

    <Response [pip]>
    <Response [pip]>
    (0.03878211975097656, ' seconds for parallel d/l')
    Exception KeyError: KeyError(3076684868L,) in <module 'threading' from '/usr/lib/python2.7/threading.pyc'> ignored
    (parallelpip)kracekumar@python-lover:~/codes/python/asyncpip$ vim core.py (parallelpip)kracekumar@python-lover:~/codes/python/asyncpip$ python core.py parallel flask requests
    spawning using gevent
    joined all gevent, d/l started
    Traceback (most recent call last):
    File "/home/kracekumar/codes/python/asyncpip/parallelpip/local/lib/python2.7/site-packages/gevent/greenlet.py", line 390, in run
    result = self._run(*self.args, **self.kwargs)
    File "/usr/lib/python2.7/subprocess.py", line 679, in __init__
    errread, errwrite)
    File "/usr/lib/python2.7/subprocess.py", line 1239, in _execute_child
    raise child_exception
    OSError: [Errno 2] No such file or directory
    <Greenlet at 0xb73cb39cL: Popen('pip install flask')> failed with OSError

    Traceback (most recent call last):
    File "/home/kracekumar/codes/python/asyncpip/parallelpip/local/lib/python2.7/site-packages/gevent/greenlet.py", line 390, in run
    result = self._run(*self.args, **self.kwargs)
    File "/usr/lib/python2.7/subprocess.py", line 679, in __init__
    errread, errwrite)
    File "/usr/lib/python2.7/subprocess.py", line 1239, in _execute_child
    raise child_exception
    OSError: [Errno 2] No such file or directory
    <Greenlet at 0xb73cb43cL: Popen('pip install requests')> failed with OSError

    None
    None
    (0.016376972198486328, ' seconds for parallel d/l')
    Exception KeyError: KeyError(3074739268L,) in <module 'threading' from '/usr/lib/python2.7/threading.pyc'> ignored
    (parallelpip)kracekumar@python-lover:~/codes/python/asyncpip$ vim core.py
    (parallelpip)kracekumar@python-lover:~/codes/python/asyncpip$ python core.py parallel flask requests
    spawning using gevent
    unhandled exception
    Exception KeyError: KeyError(3076037700L,) in <module 'threading' from '/usr/lib/python2.7/threading.pyc'> ignored