Handle HTTP error 419

update-poms.py

@@ -31,6 +31,10 @@ done_lock = asyncio.Lock()
 num_workers = 50
 
 
+class TooManyRequestsException(Exception):
+    pass
+
+
 class PackagePOM:
     def __init__(self, package: 'Package', pom: str):
         logger.debug(f'{package}: Parsing POM')
@@ -153,7 +157,7 @@ class Package:
                         return await response.text()
                         break
                     elif response.status == 429:
-                        logger.error(f'{self}: HTTP error 429 (Too many requests). Retry after {response.headers["Retry-After"]}')
+                        raise TooManyRequestsException()
                     else:
                         logger.error(f'{self}: HTTP error {response.status} from mirror {mirror}')
             else:
@@ -205,6 +209,8 @@ class Package:
                             logger.warning(f'{self}: No matching packages found')
 
                         self._verified = False
+                elif response.status == 429:
+                    raise TooManyRequestsException()
                 else:
                     self._verified = False
                     logger.error(f'{self}: HTTP error {response.status} downloading pom')
@@ -270,8 +276,15 @@ async def download(package: Package, queue: asyncio.Queue) -> None:
 async def worker(queue: asyncio.Queue) -> None:
     while True:
         package = await queue.get()
-        await download(package, queue)
-        await asyncio.sleep(random.random())
+
+        while True:
+            try:
+                await download(package, queue)
+                break
+            except TooManyRequestsException:
+                logger.debug('Too many requests. Delaying next attempt')
+                await asyncio.sleep(3*random.random())
+
         queue.task_done()