Add mirrors to pom fetch

pom.xml

@@ -20,6 +20,7 @@
     <module>poms/org.apache.maven.plugins:maven-failsafe-plugin:3.0.0-M7/pom.xml</module>
     <module>poms/org.apache.maven.plugins:maven-shade-plugin:3.3.0/pom.xml</module>
     <module>poms/org.apache.maven.plugins:maven-surefire-plugin:3.0.0-M7/pom.xml</module>
+    <module>poms/org.asciidoctor:asciidoctorj:2.5.4/pom.xml</module>
     <module>poms/org.asciidoctor:asciidoctorj-api:2.5.4/pom.xml</module>
     <module>poms/org.asciidoctor:asciidoctorj-diagram:2.2.3/pom.xml</module>
     <module>poms/org.asciidoctor:asciidoctorj-diagram-plantuml:1.2022.5/pom.xml</module>

poms/org.asciidoctor:asciidoctorj:2.5.4/pom.xml

@@ -0,0 +1,14 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0                          https://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <groupId>tmp.org.asciidoctor</groupId>
+  <artifactId>placeholder-asciidoctorj</artifactId>
+  <version>2.5.4</version>
+  <name>Package asciidoctorj</name>
+  <dependencies>
+    <dependency>
+      <groupId>org.asciidoctor</groupId>
+      <artifactId>asciidoctorj</artifactId>
+      <version>2.5.4</version>
+    </dependency>
+  </dependencies>
+</project>

update-poms.py

@@ -15,6 +15,15 @@ ET.register_namespace('', 'http://maven.apache.org/POM/4.0.0')
 
 baseurl = 'https://search.maven.org'
 base_pom_path = Path('poms')
+mirrors = [
+    "https://repo.maven.apache.org/maven2",
+    "https://repo1.maven.org/maven2",
+    "https://oss.sonatype.org/content/repositories/snapshots",
+    "https://packages.confluent.io/maven",
+    "https://registry.quarkus.io/maven",
+    "https://plugins.gradle.org/m2",
+]
+
 done: set[str] = set()
 done_lock = asyncio.Lock()
 num_workers = 50
@@ -67,7 +76,6 @@ class PackagePOM:
 
     def get_property(self, prop: str):
         elem = self.raw_root.find(f'.//properties/{prop}')
-        import pdb; pdb.set_trace()
         if elem is not None:
             return elem.text
         else:
@@ -141,17 +149,21 @@ class Package:
         group_path = self.groupId.replace(".", "/")
         pom_path = f'{self.artifactId}-{self.version}.pom'
         filepath = f'{group_path}/{self.artifactId}/{self.version}/{pom_path}'
-        pom_url = f'{baseurl}/remotecontent?filepath={filepath}'
-
-        logger.debug(f'{self}: Downloading pom from {pom_url}')
 
         async with aiohttp.ClientSession() as session:
-            async with session.get(pom_url) as response:
-                if response.status == 200:
-                    logger.debug(f'{self}: POM downloaded')
-                    self._pom = PackagePOM(self, await response.text())
-                else:
-                    logger.warning(f'{self}: HTTP error {response.status} downloading pom')
+            for mirror in mirrors:
+                pom_url = f'{mirror}/{filepath}'
+                logger.debug(f'{self}: Downloading pom from {pom_url}')
+
+                async with session.get(pom_url) as response:
+                    if response.status == 200:
+                        logger.debug(f'{self}: POM downloaded')
+                        self._pom = PackagePOM(self, await response.text())
+                        break
+                    else:
+                        logger.debug(f'{self}: HTTP error {response.status} from mirror {mirror}')
+            else:
+                logger.warning(f'{self}: Failed for all mirrors')
 
         return self._pom
 
@@ -258,6 +270,7 @@ async def main() -> None:
 
     load_package_list(Path('package-list.txt'), queue)
 
+    logger.debug(f'Starting {num_workers} workers')
     for i in range(num_workers):
         tasks.append(
             asyncio.create_task(
@@ -266,11 +279,14 @@ async def main() -> None:
         )
 
     await queue.join()
+
+    logger.debug('Queue is empty. Cancelling workers')
     for task in tasks:
         task.cancel()
 
     await asyncio.gather(*tasks, return_exceptions=True)
 
+    logger.info('Generating master POM')
     subprocess.call(['sh', 'generate_master_pom.sh'])
 
 
@@ -278,6 +294,7 @@ logger = logging.getLogger(__name__)
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser()
+    parser.add_argument('-w', '--workers', type=int, default=num_workers)
     parser.add_argument('-v', '--verbose', dest='verbosity', action='count', default=0)
     args = parser.parse_args()
 
@@ -290,4 +307,6 @@ if __name__ == '__main__':
 
     logging.basicConfig(level=log_level)
 
+    num_workers = args.workers
+
     asyncio.run(main())