I'm trying to a write super simple task runner with asyncio.
Basically, my goal is to run 100 requests to google as tasks, whilst also being be able to handle tasks being added at a later time.
from abc import ABC, abstractmethod
import asyncio
import socket
import time
import aiohttp
import requests
class MyScheduler:
def __init__(self, wait=2):
self.work = []
self.wait = wait
def set_initial_callback(self, callback, **kwargs):
self.initial_callback = callback
self.initial_callback_args = kwargs
async def add_task(self, callback, **kwargs):
task = asyncio.create_task(callback(**kwargs))
self.work.append(task)
async def _run(self):
while self.work:
task = self.work.pop()
await task
if len(self.work) == 0:
await asyncio.sleep(self.wait)
async def set_things_up(self, callback, **kwargs):
task = asyncio.create_task(callback(**kwargs))
self.work.append(task)
await self._run()
def go(self):
asyncio.run(self.set_things_up(self.initial_callback, **self.initial_callback_args))
async def google(n):
if n == 100:
return None
await s.add_task(google, n=n + 1)
async with aiohttp.ClientSession() as session:
async with session.get('http://h...content-available-to-author-only...n.org/get') as resp:
print(resp.status)
t = time.time()
s = MyScheduler(wait=1)
s.set_initial_callback(google, n=1)
s.go()
print(time.time() - t)
I benchmarked this against sequentally running requests, and I did see a massive speed up. It's still super rough, but I'd love some pointers on how I could improve my code in terms of readability/exploiting async stuff better.