Async http get using aiohttp

async_http_get.py

import aiohttp
import asyncio

def async_http_get(urls, extractor=None, json_response=True):
    tasks = []
    sem = asyncio.Semaphore(32)

    async def fetch(session, url):
        async with session.get(url) as response:
            if json_response:
                response = await response.json()
                if extractor:
                    response = extractor(response)
            else:
                response = await response.text()
            return (url, response)

    async def sem_fetch(sem, session, url):
        async with sem:
            return await fetch(session, url)

    async def run(loop, sem):
        async with aiohttp.ClientSession(loop=loop) as session:
            for url in urls:
                task = asyncio.ensure_future(sem_fetch(sem, session, url))
                tasks.append(task)

            return await asyncio.gather(*tasks)

    loop = asyncio.get_event_loop()
    return loop.run_until_complete(run(loop, sem))