From f026777f08daa3be4eda7a2cce8301612e4ac16a Mon Sep 17 00:00:00 2001
From: Miro Hrončok <miro@hroncok.cz>
Date: Aug 25 2018 15:18:19 +0000
Subject: find-orphaned-packages: Throttle the requests with asyncio.Semaphore


---

diff --git a/find-orphaned-packages b/find-orphaned-packages
index a6fb6c8..25d3208 100755
--- a/find-orphaned-packages
+++ b/find-orphaned-packages
@@ -13,9 +13,9 @@ async def all_orphanned():
             yield pkg
 
 
-async def print_if_not_retired(pkg, sleep=1):
+async def print_if_not_retired(pkg, semaphore, sleep=1):
     url = f'https://src.fedoraproject.org/rpms/{pkg}/blob/master/f/dead.package'
-    async with aiohttp.ClientSession() as session:
+    async with semaphore, aiohttp.ClientSession() as session:
         try:
             async with session.head(url) as resp:
                 if resp.status == 404:
@@ -26,13 +26,14 @@ async def print_if_not_retired(pkg, sleep=1):
             if sleep > 15 * 60:
                 raise
             await asyncio.sleep(sleep)
-            return await print_if_not_retired(pkg, sleep*2)
+            return await print_if_not_retired(pkg, semaphore, sleep*2)
 
 
 async def main():
     tasks = []
+    semaphore = asyncio.Semaphore(512)
     async for pkg in all_orphanned():
-        tasks.append(asyncio.create_task(print_if_not_retired(pkg)))
+        tasks.append(asyncio.create_task(print_if_not_retired(pkg, semaphore)))
     await asyncio.gather(*tasks)