Example async

async_worker.py

import logging
import asyncio
from asyncio import Queue, Event, QueueEmpty

import aiohttp


async def func_worker(taskq, shutdown_signal):
    print('worker: started working')
    while True:
        try:
            host = await taskq.get()
        except QueueEmpty:
            if shutdown_signal.is_set():
                return
            else:
                await asyncio.sleep(0.1)
        else:
            url = 'http://%s/' % host
            logging.debug('Downloading %s' % url)
            async with aiohttp.ClientSession() as sess:
                async with sess.get(url) as res:
                    print(host, res.headers)


async def func_task_generator(taskq):
    with open('var/host.txt') as inp:
        for line in inp:
            host = line.strip()
            await taskq.put(host)


async def manager():
    taskq = Queue(maxsize=10)
    task_gen = asyncio.ensure_future(func_task_generator(taskq))
    shutdown_signal = Event()
    workers_pool = []
    for _ in range(1):
        worker = asyncio.ensure_future(func_worker(taskq, shutdown_signal))
        workers_pool.append(worker)
    while not task_gen.done():
        await asyncio.sleep(0.1)
    print('Check')
    await shutdown_signal.set()
    for worker in workers_pool:
        while not worker.done():
            await asyncio.sleep(0.1)
    print('Done')
    

def main(**kwargs):
    loop = asyncio.get_event_loop()
    #loop.set_debug(True)
    loop.run_until_complete(manager())