Rate Limiting Async API Requests With AIOHTTP and ASYNCIO

main.py

# standard
import asyncio
import logging

# thirdparty
import aiohttp
import attr
from attr.validation import instance_of

LOGGER_FORMAT = "%(asctime)s %(messages)s"
logging.basicConfig(format=LOGGER_FORMAT, datefmt="[%H:%M:%S]")
log = logging.getLogger()
log.setLevel(Logging.INFO)


@attr.s
class Fetch:
    limit = attr.ib()  # batch
    rate = attr.ib(default=5, converter=int)  # speed

    async def make_request(self, url):

        async with self.limit:

            async with aiohttp.ClientSession() as session:

                async with session.request(method="GET", url=url) as response:
                    json = await response.json()
                    status = response.status
                    log.info(f"Made request: {url}. Status: {status}")

                    await asyncio.sleep(self.rate)


async def main(urls, rate, limit):
    limit = asyncio.Semaphore(limit)

    f = Fetch(
        rate=rate,
        limit=limit,
    )
    tasks = []

    for url in urls:
        tasks.append(f.make_request(url=url, limit=limit))

    results = await asyncio.gather(*tasks)