Add support for resolving latest or stable version of packages

package-list.yaml

@@ -33,7 +33,7 @@ configurations:
     packages:
       org.jetbrains.kotlinx:kotlinx-datetime: "0.4.0"
       com.expediagroup:
-        _versions: ["7.0.0-alpha.6", "6.5.2"]
-        graphql-kotlin-ktor-server: ["7.0.0-alpha.6"]
+        _versions: ["latest", "stable"]
+        graphql-kotlin-ktor-server: ["latest"]
         graphql-kotlin-client: []
         graphql-kotlin-client-generator: []

sync/src/config.py

@@ -17,6 +17,9 @@ class Package:
     def __str__(self):
         return f'{self.group_id}:{self.artifact_id}:{self.version}'
 
+    def __hash__(self):
+        return hash(str(self))
+
 
 @dataclass
 class Configuration:

sync/src/gradle.py

@@ -1,3 +1,5 @@
+from typing import Iterable
+
 from config import Package
 
 
@@ -16,7 +18,7 @@ pluginManagement {
 """
 
 
-def create_gradle_build(kotlin_version: str, plugins: dict[str, str], packages: list[Package], repo: str) -> str:
+def create_gradle_build(kotlin_version: str, plugins: dict[str, str], packages: Iterable[Package], repo: str) -> str:
     return """// Generated, do not edit
 plugins {
     """ + f'kotlin("jvm") version "{kotlin_version}"' + """

sync/src/main.py

@@ -11,11 +11,11 @@ logger = logging.getLogger(__name__)
 
 
 async def resolve_kotlin(ident: str, configuration: Configuration, output_dir: Path, mirrors: list[str], gradle_repo: str):
-    resolved_packages = [
+    resolved_packages = set([
         resolved
         for package in configuration.packages
         for resolved in await get_effective_packages(package, mirrors)
-    ]
+    ])
 
     gradle_path = configuration.gradle_version or "default"
     path = output_dir / f"{gradle_path}/{ident}-kotlin-{configuration.kotlin_version}"

sync/src/maven/fetch.py

@@ -4,6 +4,7 @@ from aiohttp import ClientSession
 from typing import Optional, Iterable
 
 from config import Package
+from maven.metadata import MavenMetadata
 from pom import PackagePOM
 
 logger = logging.getLogger(__name__)
@@ -17,6 +18,12 @@ def group_url(group_id: str) -> str:
     return f'{group_id.replace(".", "/")}'
 
 
+async def fetch_metadata(package: Package, mirrors: Iterable[str]) -> Optional[MavenMetadata]:
+    url = f'{group_url(package.group_id)}/{package.artifact_id}/maven-metadata.xml'
+    metadata = await fetch_maven_file(url, mirrors)
+    return MavenMetadata(package, metadata) if metadata else None
+
+
 async def fetch_pom(package: Package, mirrors: Iterable[str]) -> Optional[PackagePOM]:
     url = f'{group_url(package.group_id)}/{package.artifact_id}/{package.version}/{package.artifact_id}-{package.version}.pom'
     pom = await fetch_maven_file(url, mirrors)

sync/src/maven/metadata.py

@@ -0,0 +1,58 @@
+import logging
+from dataclasses import dataclass
+from xml.etree import ElementTree
+
+from config import Package
+from xmlutils import find_tag_text
+
+logger = logging.getLogger(__name__)
+
+
+class MetadataError(Exception):
+    pass
+
+
+@dataclass
+class MavenMetadata:
+    latest: str
+    release: str
+    stable: str
+    versions: list[str]
+
+    def __init__(self, package: Package, xml: str):
+        root = ElementTree.fromstring(xml)
+
+        versioning = root.find('versioning')
+        if versioning is None:
+            logger.error(f'Maven metadata for {package} does not contain "<versioning>"')
+            raise MetadataError(str(package))
+
+        versions_tag = versioning.find('versions')
+        if versions_tag is None:
+            logger.error(f'Maven metadata for {package} does not contain any versions')
+            raise MetadataError(str(package))
+
+        self.versions = [v.text for v in versions_tag if v is not None and v.text]
+        self.versions.reverse()
+
+        if (latest := find_tag_text(versioning, 'latest')) is not None:
+            self.latest = latest
+        else:
+            logger.error(f'Maven metadata for {package} does not contain <latest>')
+            raise MetadataError(str(package))
+
+        if (release := find_tag_text(versioning, 'release')) is not None:
+            self.release = release
+        else:
+            logger.error(f'Maven metadata for {package} does not contain <release>')
+            raise MetadataError(str(package))
+
+        if '-' in self.release:
+            release_index = self.versions.index(self.release)
+            stable_list = [v for v in self.versions[release_index:] if '-' not in v]
+            try:
+                self.stable = stable_list[0]
+            except IndexError:
+                self.stable = self.release
+        else:
+            self.stable = self.release

sync/src/maven/packages.py

@@ -1,8 +1,10 @@
+import dataclasses
 import logging
 from typing import Optional, Iterable
 
 from config import Package
 from maven.fetch import fetch_pom
+from maven.version import resolve_version
 from pom import PropertyMissing, Properties
 
 logger = logging.getLogger(__name__)
@@ -15,13 +17,30 @@ async def get_effective_packages(package: Package, mirrors: Iterable[str]) -> li
     For most packages, this just returns the package name. However, for BOMs, the full list of packages included
     in the BOM is returned (including the BOM itself). This is because Gradle doesn't fetch all packages of a BOM.
 
+    If the version is 'latest' or 'stable', the maven metadata is queried. For 'latest', the <latest> tag is used.
+    For 'stable' the <release> tag is used. If the <release> tag contains a minus (-) the version is interpreted as
+    an alpha, beta or RC, and the newest version without a minus will also be fetched.
+
     This requires querying Maven and parsing the POM to check if the package is a BOM.
     """
+    versions = [
+        dataclasses.replace(package, version=version)
+        for version in await resolve_version(package, mirrors)
+    ]
+
+    deps = [
+        dep
+        for v in versions
+        for dep in await get_dependencies(v, mirrors)
+    ]
+
+    return versions + deps
+
+
+async def get_dependencies(package: Package, mirrors: Iterable[str]) -> list[Package]:
     packages = []
 
     if pom := await fetch_pom(package, mirrors):
-        packages.append(package)
-
         if pom.is_bom:
             try:
                 packages.extend(pom.dependency_management)

sync/src/maven/version.py

@@ -0,0 +1,27 @@
+import logging
+from typing import Iterable
+
+from config import Package
+from maven.fetch import fetch_metadata
+
+logger = logging.getLogger(__name__)
+
+
+async def resolve_version(package: Package, mirrors: Iterable[str]) -> list[str]:
+    versions = []
+
+    if package.version in ['latest', 'stable']:
+        if metadata := await fetch_metadata(package, mirrors):
+            if package.version == 'latest':
+                versions.append(metadata.latest)
+            elif package.version == 'stable':
+                versions.append(metadata.release)
+
+                if metadata.stable != metadata.release:
+                    versions.append(metadata.stable)
+        else:
+            logger.error(f'{package}: Could not find package metadata')
+    else:
+        versions.append(package.version)
+
+    return versions

sync/src/pom.py

@@ -4,12 +4,18 @@ from typing import Optional, TypeAlias
 from xml.etree import ElementTree
 
 from config import Package
-from xmlutils import pom_namespace as ns, find_tag_text
+from xmlutils import find_tag_text
 
 logger = logging.getLogger(__name__)
 
 Properties: TypeAlias = dict[str, str]
 
+ns = {'': 'http://maven.apache.org/POM/4.0.0'}
+
+
+def pom_find_tag_text(parent: ElementTree.Element, tag: str) -> str | None:
+    return find_tag_text(parent, tag, ns)
+
 
 class PropertyMissing(Exception):
     def __init__(self, prop: str):
@@ -38,9 +44,9 @@ class PackagePOM:
 
         self.parent = None
         if (parent_tag := self._raw_root.find('parent', ns)) is not None:
-            parent_group = find_tag_text(parent_tag, 'groupId')
-            parent_artifact = find_tag_text(parent_tag, 'artifactId')
-            parent_version = find_tag_text(parent_tag, 'version')
+            parent_group = pom_find_tag_text(parent_tag, 'groupId')
+            parent_artifact = pom_find_tag_text(parent_tag, 'artifactId')
+            parent_version = pom_find_tag_text(parent_tag, 'version')
 
             if parent_group is None or parent_artifact is None or parent_version is None:
                 raise PackageError(f'Invalid parent {parent_group}:{parent_artifact}:{parent_version}')

sync/src/xmlutils.py

@@ -1,6 +1,7 @@
-pom_namespace = {'': 'http://maven.apache.org/POM/4.0.0'}
+from typing import Optional
+from xml.etree import ElementTree
 
 
-def find_tag_text(parent, tag) -> str | None:
-    elem = parent.find(tag, pom_namespace)
+def find_tag_text(parent: ElementTree.Element, tag: str, namespace: Optional[dict[str, str]] = None) -> str | None:
+    elem = parent.find(tag, namespace)
     return elem.text if elem is not None else None