Add a dataclass for package

sync/src/config.py

@@ -8,13 +8,23 @@ from pathlib import Path
 logger = logging.getLogger(__name__)
 
 
+@dataclass
+class Package:
+    group_id: str
+    artifact_id: str
+    version: str
+
+    def __str__(self):
+        return f'{self.group_id}:{self.artifact_id}:{self.version}'
+
+
 @dataclass
 class Configuration:
     name: str
     kotlin_version: str
     gradle_version: Optional[str]
     plugins: dict[str, str]
-    packages: list[str]
+    packages: list[Package]
 
 
 @dataclass
@@ -23,39 +33,44 @@ class Config:
     mirrors: list[str]
 
 
-def handle_packages(section) -> list[str]:
+def handle_packages(section) -> list[Package]:
     ignore = ['_versions']
-    result = []
+    result: list[Package] = []
 
     for entry in section:
         if entry not in ignore:
             if ':' in entry:
-                versions = section[entry]
-                if isinstance(versions, str):
-                    result.append(f'{entry}:{versions}')
-                elif isinstance(versions, list):
-                    for version in versions:
-                        result.append(f'{entry}:{version}')
+                try:
+                    group_id, artifact_id = entry.split(':')
+                except ValueError:
+                    logger.exception(f'Illegal package identifier "{entry}". Should be on the format "groupId:artifactId"')
+                    continue
+
+                value = section[entry]
+                if isinstance(value, str):
+                    result.append(Package(group_id, artifact_id, value))
+                elif isinstance(value, list):
+                    for version in value:
+                        result.append(Package(group_id, artifact_id, version))
                 else:
-                    logger.warning(f'Invalid version "{versions}" for "{entry}". Should be a string or list.')
+                    logger.warning(f'Invalid version "{value}" for "{entry}". Should be a string or list.')
             elif isinstance(section[entry], dict):
+                group_id = entry
                 group_section = section[entry]
                 default_versions = group_section.get('_versions', [])
 
-                for artifact in group_section:
-                    if artifact not in ignore:
-                        versions = group_section[artifact]
-
-                        if not versions and default_versions:
-                            versions = default_versions
+                for artifact_id, value in group_section.items():
+                    if artifact_id not in ignore:
+                        if not value and default_versions:
+                            value = default_versions
 
-                        if isinstance(versions, str):
-                            result.append(f'{entry}:{artifact}:{versions}')
-                        elif isinstance(versions, list) and versions:
-                            for version in versions:
-                                result.append(f'{entry}:{artifact}:{version}')
+                        if isinstance(value, str):
+                            result.append(Package(group_id, artifact_id, value))
+                        elif isinstance(value, list):
+                            for version in value:
+                                result.append(Package(group_id, artifact_id, version))
                         else:
-                            logger.warning(f'Invalid versions "{versions}" for "{entry}:{artifact}"')
+                            logger.warning(f'Invalid versions "{value}" for "{group_id}:{artifact_id}"')
             else:
                 logger.warning(f'Invalid package spec "{entry}". Should be a full spec or a group ID')
 

sync/src/gradle.py

@@ -1,3 +1,6 @@
+from config import Package
+
+
 def create_gradle_settings(repo: str) -> str:
     return """// Generated, do not edit
 rootProject.name = "gradle sync job"
@@ -13,7 +16,7 @@ pluginManagement {
 """
 
 
-def create_gradle_build(kotlin_version: str, plugins: dict[str, str], packages: list[str], repo: str) -> str:
+def create_gradle_build(kotlin_version: str, plugins: dict[str, str], packages: list[Package], repo: str) -> str:
     return """// Generated, do not edit
 plugins {
     """ + f'kotlin("jvm") version "{kotlin_version}"' + """
@@ -29,7 +32,7 @@ repositories {
 }
 
 val deps = listOf<String>(
-    """ + ',\n    '.join(f'"{dep}"' for dep in sorted(packages)) + """
+    """ + ',\n    '.join(f'"{dep}"' for dep in sorted([str(p) for p in packages])) + """
 ).flatMap {
     listOf(it, it + ":sources", it + ":javadoc")
 }.map {

sync/src/maven/fetch.py

@@ -3,6 +3,7 @@ import asyncio
 from aiohttp import ClientSession
 from typing import Optional, Iterable
 
+from config import Package
 from pom import PackagePOM
 
 logger = logging.getLogger(__name__)
@@ -12,8 +13,13 @@ class TooManyRequestsException(Exception):
     pass
 
 
-async def fetch_pom(package: str, mirrors: Iterable[str]) -> Optional[PackagePOM]:
-    pom = await fetch_maven_file(package, 'pom', mirrors)
+def group_url(group_id: str) -> str:
+    return f'{group_id.replace(".", "/")}'
+
+
+async def fetch_pom(package: Package, mirrors: Iterable[str]) -> Optional[PackagePOM]:
+    url = f'{group_url(package.group_id)}/{package.artifact_id}/{package.version}/{package.artifact_id}-{package.version}.pom'
+    pom = await fetch_maven_file(url, mirrors)
     return PackagePOM(package, pom) if pom else None
 
 
@@ -27,10 +33,8 @@ async def fetch_from_mirror(session: ClientSession, url: str) -> str | int:
             return response.status
 
 
-async def fetch_maven_file(package: str, extension: str, mirrors: Iterable[str]) -> Optional[str]:
-    group_id, artifact_id, version = package.split(':')
-    url = f'{group_id.replace(".", "/")}/{artifact_id}/{version}/{artifact_id}-{version}.{extension}'
-    logger.debug(f'{package}: Downloading {extension} from {url}')
+async def fetch_maven_file(url: str, mirrors: Iterable[str]) -> Optional[str]:
+    logger.debug(f'Downloading {url}')
 
     async with ClientSession() as session:
         # Retry up to 5 times
@@ -46,20 +50,20 @@ async def fetch_maven_file(package: str, extension: str, mirrors: Iterable[str])
                     result = await fetch_from_mirror(session, f'{mirror}/{url}')
 
                     if isinstance(result, str):
-                        logger.debug(f'{package}: {extension} downloaded')
+                        logger.debug(f'{url} downloaded')
                         return result
                     else:
-                        logger.debug(f'{package}: HTTP error {result} from mirror {mirror}')
+                        logger.warning(f'HTTP error {result} from {mirror}/{url}. Trying next mirror.')
                 except TooManyRequestsException:
-                    logger.info(f'{package}: Received Too Many Requests error. Trying other mirror.')
+                    logger.info(f'Received Too Many Requests error from {mirror}/{url}. Trying other mirror.')
                     retry_mirrors.append(mirror)
 
             if retry_mirrors:
-                logger.info(f'{package}: Backing off, then trying again')
+                logger.info(f"Could not find {url}, but some mirrors didn't respond. Backing off, then trying again")
                 mirrors = retry_mirrors
                 await asyncio.sleep(0.1)
             else:
                 break
 
-        logger.warning(f'{package}: File download of {extension} failed for all mirrors')
+        logger.error(f'Download of {url} failed for all mirrors')
         return None

sync/src/maven/packages.py

@@ -1,13 +1,14 @@
 import logging
 from typing import Optional, Iterable
 
+from config import Package
 from maven.fetch import fetch_pom
 from pom import PropertyMissing, Properties
 
 logger = logging.getLogger(__name__)
 
 
-async def get_effective_packages(package: str, mirrors: Iterable[str]) -> list[str]:
+async def get_effective_packages(package: Package, mirrors: Iterable[str]) -> list[Package]:
     """
     Get a list of packages that is required for Gradle to fetch this package.
 
@@ -42,7 +43,7 @@ async def get_effective_packages(package: str, mirrors: Iterable[str]) -> list[s
     return packages
 
 
-async def get_parent_props(parent: Optional[str], mirrors: Iterable[str]) -> Properties:
+async def get_parent_props(parent: Optional[Package], mirrors: Iterable[str]) -> Properties:
     if parent:
         if pom := await fetch_pom(parent, mirrors):
             pom.set_properties(await get_parent_props(pom.parent, mirrors))

sync/src/pom.py

@@ -3,6 +3,7 @@ import re
 from typing import Optional, TypeAlias
 from xml.etree import ElementTree
 
+from config import Package
 from xmlutils import pom_namespace as ns, find_tag_text
 
 logger = logging.getLogger(__name__)
@@ -22,30 +23,20 @@ class PackageError(Exception):
 class PackagePOM:
     name: str
     is_bom: bool
-    parent: Optional[str]
-    group_id: Optional[str]
-    artifact_id: Optional[str]
-    version: Optional[str]
+    package: Package
+    parent: Optional[Package]
     properties: Properties
 
     _raw_root: ElementTree.Element
 
-    def __init__(self, name: str, pom: str):
-        logger.debug(f'{name}: Parsing POM')
-        self.name = name
+    def __init__(self, package: Package, pom: str):
+        self.package = package
+        self.name = str(package)
+        logger.debug(f'{self.name}: Parsing POM')
 
         self._raw_root = ElementTree.fromstring(pom)
 
-        self.group_id, self.artifact_id, self.version = name.split(':')
-
-        if self.group_id is None or self.artifact_id is None or self.version is None:
-            logger.warning(
-                f'{name}: One of groupId={self.group_id}, artifactId={self.artifact_id}, version={self.version} was '
-                f'None. This can cause issues with dependency resolution'
-            )
-
         self.parent = None
-
         if (parent_tag := self._raw_root.find('parent', ns)) is not None:
             parent_group = find_tag_text(parent_tag, 'groupId')
             parent_artifact = find_tag_text(parent_tag, 'artifactId')
@@ -54,7 +45,7 @@ class PackagePOM:
             if parent_group is None or parent_artifact is None or parent_version is None:
                 raise PackageError(f'Invalid parent {parent_group}:{parent_artifact}:{parent_version}')
             else:
-                self.parent = f'{parent_group}:{parent_artifact}:{parent_version}'
+                self.parent = Package(parent_group, parent_artifact, parent_version)
 
         if (packaging := self._raw_root.find('packaging', ns)) is not None:
             self.is_bom = packaging == 'pom'
@@ -64,7 +55,7 @@ class PackagePOM:
         self.set_properties({})
 
     @property
-    def dependency_management(self) -> list[str]:
+    def dependency_management(self) -> list[Package]:
         return [
             self._package_from_xml_dep(dep)
             for dep in self._raw_root.find('dependencyManagement/dependencies', ns) or []
@@ -99,11 +90,11 @@ class PackagePOM:
             prop = match.group(1)
 
             if prop == 'project.groupId':
-                value = str(self.group_id)
+                value = str(self.package.group_id)
             elif prop == 'project.artifactId':
-                value = str(self.artifact_id)
+                value = str(self.package.artifact_id)
             elif prop == 'project.version':
-                value = str(self.version)
+                value = str(self.package.version)
             elif prop.startswith('project.build') or prop.startswith('env.') or prop.startswith('maven.'):
                 value = None
             elif prop in ['project.basedir', 'basedir', 'user.home', 'debug.port']:
@@ -126,10 +117,14 @@ class PackagePOM:
             text,
         )
 
-    def _package_from_xml_dep(self, dep: ElementTree.Element) -> str:
+    def _package_from_xml_dep(self, dep: ElementTree.Element) -> Package:
         def prop_replace_tag(tag) -> str:
             return self._prop_replace(
                 elem.text or '' if (elem := dep.find(tag, ns)) is not None else '',
             )
 
-        return f"{prop_replace_tag('groupId')}:{prop_replace_tag('artifactId')}:{prop_replace_tag('version')}"
+        return Package(
+            prop_replace_tag('groupId'),
+            prop_replace_tag('artifactId'),
+            prop_replace_tag('version'),
+        )