Handle HTTP error 419

update-poms.py

@@ -1,6 +1,7 @@
 #!/bin/python3
 
 import re
+import random
 import argparse
 import logging
 import asyncio
@@ -30,6 +31,10 @@ done_lock = asyncio.Lock()
 num_workers = 50
 
 
+class TooManyRequestsException(Exception):
+    pass
+
+
 class PackagePOM:
     def __init__(self, package: 'Package', pom: str):
         logger.debug(f'{package}: Parsing POM')
@@ -199,6 +204,8 @@ class Package:
                         logger.debug(f'{self}: {extension} downloaded')
                         return await response.text()
                         break
+                    elif response.status == 429:
+                        raise TooManyRequestsException()
                     else:
                         logger.debug(f'{self}: HTTP error {response.status} from mirror {mirror}')
             else:
@@ -250,6 +257,8 @@ class Package:
                             logger.warning(f'{self}: No matching packages found')
 
                         self._verified = False
+                elif response.status == 429:
+                    raise TooManyRequestsException()
                 else:
                     self._verified = False
                     logger.warning(f'{self}: HTTP error {response.status} downloading pom')
@@ -322,7 +331,15 @@ async def download(package: Package, queue: asyncio.Queue) -> None:
 async def worker(queue: asyncio.Queue) -> None:
     while True:
         package = await queue.get()
-        await download(package, queue)
+
+        while True:
+            try:
+                await download(package, queue)
+                break
+            except TooManyRequestsException:
+                logger.info('Too many requests. Delaying next attempt')
+                await asyncio.sleep(3*random.random() + 0.2)
+
         queue.task_done()