From 15aa4ea177e36bcea231982253b48cd42a495528 Mon Sep 17 00:00:00 2001 From: Sindre Stephansen Date: Mon, 31 Jul 2023 14:56:10 +0200 Subject: [PATCH] Add support for resolving latest or stable version of packages --- package-list.yaml | 4 +-- sync/src/config.py | 3 ++ sync/src/gradle.py | 4 ++- sync/src/main.py | 4 +-- sync/src/maven/fetch.py | 7 +++++ sync/src/maven/metadata.py | 58 ++++++++++++++++++++++++++++++++++++++ sync/src/maven/packages.py | 23 +++++++++++++-- sync/src/maven/version.py | 27 ++++++++++++++++++ sync/src/pom.py | 14 ++++++--- sync/src/xmlutils.py | 7 +++-- 10 files changed, 137 insertions(+), 14 deletions(-) create mode 100644 sync/src/maven/metadata.py create mode 100644 sync/src/maven/version.py diff --git a/package-list.yaml b/package-list.yaml index 4abf054..f75938f 100644 --- a/package-list.yaml +++ b/package-list.yaml @@ -33,7 +33,7 @@ configurations: packages: org.jetbrains.kotlinx:kotlinx-datetime: "0.4.0" com.expediagroup: - _versions: ["7.0.0-alpha.6", "6.5.2"] - graphql-kotlin-ktor-server: ["7.0.0-alpha.6"] + _versions: ["latest", "stable"] + graphql-kotlin-ktor-server: ["latest"] graphql-kotlin-client: [] graphql-kotlin-client-generator: [] diff --git a/sync/src/config.py b/sync/src/config.py index c16abd5..dab5814 100644 --- a/sync/src/config.py +++ b/sync/src/config.py @@ -17,6 +17,9 @@ class Package: def __str__(self): return f'{self.group_id}:{self.artifact_id}:{self.version}' + def __hash__(self): + return hash(str(self)) + @dataclass class Configuration: diff --git a/sync/src/gradle.py b/sync/src/gradle.py index 5fc5984..791d554 100644 --- a/sync/src/gradle.py +++ b/sync/src/gradle.py @@ -1,3 +1,5 @@ +from typing import Iterable + from config import Package @@ -16,7 +18,7 @@ pluginManagement { """ -def create_gradle_build(kotlin_version: str, plugins: dict[str, str], packages: list[Package], repo: str) -> str: +def create_gradle_build(kotlin_version: str, plugins: dict[str, str], packages: Iterable[Package], repo: str) -> str: return """// Generated, do not edit plugins { """ + f'kotlin("jvm") version "{kotlin_version}"' + """ diff --git a/sync/src/main.py b/sync/src/main.py index 19c4b1f..9fa17ae 100644 --- a/sync/src/main.py +++ b/sync/src/main.py @@ -11,11 +11,11 @@ logger = logging.getLogger(__name__) async def resolve_kotlin(ident: str, configuration: Configuration, output_dir: Path, mirrors: list[str], gradle_repo: str): - resolved_packages = [ + resolved_packages = set([ resolved for package in configuration.packages for resolved in await get_effective_packages(package, mirrors) - ] + ]) gradle_path = configuration.gradle_version or "default" path = output_dir / f"{gradle_path}/{ident}-kotlin-{configuration.kotlin_version}" diff --git a/sync/src/maven/fetch.py b/sync/src/maven/fetch.py index 7aa69d5..20335df 100644 --- a/sync/src/maven/fetch.py +++ b/sync/src/maven/fetch.py @@ -4,6 +4,7 @@ from aiohttp import ClientSession from typing import Optional, Iterable from config import Package +from maven.metadata import MavenMetadata from pom import PackagePOM logger = logging.getLogger(__name__) @@ -17,6 +18,12 @@ def group_url(group_id: str) -> str: return f'{group_id.replace(".", "/")}' +async def fetch_metadata(package: Package, mirrors: Iterable[str]) -> Optional[MavenMetadata]: + url = f'{group_url(package.group_id)}/{package.artifact_id}/maven-metadata.xml' + metadata = await fetch_maven_file(url, mirrors) + return MavenMetadata(package, metadata) if metadata else None + + async def fetch_pom(package: Package, mirrors: Iterable[str]) -> Optional[PackagePOM]: url = f'{group_url(package.group_id)}/{package.artifact_id}/{package.version}/{package.artifact_id}-{package.version}.pom' pom = await fetch_maven_file(url, mirrors) diff --git a/sync/src/maven/metadata.py b/sync/src/maven/metadata.py new file mode 100644 index 0000000..68316b9 --- /dev/null +++ b/sync/src/maven/metadata.py @@ -0,0 +1,58 @@ +import logging +from dataclasses import dataclass +from xml.etree import ElementTree + +from config import Package +from xmlutils import find_tag_text + +logger = logging.getLogger(__name__) + + +class MetadataError(Exception): + pass + + +@dataclass +class MavenMetadata: + latest: str + release: str + stable: str + versions: list[str] + + def __init__(self, package: Package, xml: str): + root = ElementTree.fromstring(xml) + + versioning = root.find('versioning') + if versioning is None: + logger.error(f'Maven metadata for {package} does not contain ""') + raise MetadataError(str(package)) + + versions_tag = versioning.find('versions') + if versions_tag is None: + logger.error(f'Maven metadata for {package} does not contain any versions') + raise MetadataError(str(package)) + + self.versions = [v.text for v in versions_tag if v is not None and v.text] + self.versions.reverse() + + if (latest := find_tag_text(versioning, 'latest')) is not None: + self.latest = latest + else: + logger.error(f'Maven metadata for {package} does not contain ') + raise MetadataError(str(package)) + + if (release := find_tag_text(versioning, 'release')) is not None: + self.release = release + else: + logger.error(f'Maven metadata for {package} does not contain ') + raise MetadataError(str(package)) + + if '-' in self.release: + release_index = self.versions.index(self.release) + stable_list = [v for v in self.versions[release_index:] if '-' not in v] + try: + self.stable = stable_list[0] + except IndexError: + self.stable = self.release + else: + self.stable = self.release diff --git a/sync/src/maven/packages.py b/sync/src/maven/packages.py index b9f3fee..46f13db 100644 --- a/sync/src/maven/packages.py +++ b/sync/src/maven/packages.py @@ -1,8 +1,10 @@ +import dataclasses import logging from typing import Optional, Iterable from config import Package from maven.fetch import fetch_pom +from maven.version import resolve_version from pom import PropertyMissing, Properties logger = logging.getLogger(__name__) @@ -15,13 +17,30 @@ async def get_effective_packages(package: Package, mirrors: Iterable[str]) -> li For most packages, this just returns the package name. However, for BOMs, the full list of packages included in the BOM is returned (including the BOM itself). This is because Gradle doesn't fetch all packages of a BOM. + If the version is 'latest' or 'stable', the maven metadata is queried. For 'latest', the tag is used. + For 'stable' the tag is used. If the tag contains a minus (-) the version is interpreted as + an alpha, beta or RC, and the newest version without a minus will also be fetched. + This requires querying Maven and parsing the POM to check if the package is a BOM. """ + versions = [ + dataclasses.replace(package, version=version) + for version in await resolve_version(package, mirrors) + ] + + deps = [ + dep + for v in versions + for dep in await get_dependencies(v, mirrors) + ] + + return versions + deps + + +async def get_dependencies(package: Package, mirrors: Iterable[str]) -> list[Package]: packages = [] if pom := await fetch_pom(package, mirrors): - packages.append(package) - if pom.is_bom: try: packages.extend(pom.dependency_management) diff --git a/sync/src/maven/version.py b/sync/src/maven/version.py new file mode 100644 index 0000000..1bd442e --- /dev/null +++ b/sync/src/maven/version.py @@ -0,0 +1,27 @@ +import logging +from typing import Iterable + +from config import Package +from maven.fetch import fetch_metadata + +logger = logging.getLogger(__name__) + + +async def resolve_version(package: Package, mirrors: Iterable[str]) -> list[str]: + versions = [] + + if package.version in ['latest', 'stable']: + if metadata := await fetch_metadata(package, mirrors): + if package.version == 'latest': + versions.append(metadata.latest) + elif package.version == 'stable': + versions.append(metadata.release) + + if metadata.stable != metadata.release: + versions.append(metadata.stable) + else: + logger.error(f'{package}: Could not find package metadata') + else: + versions.append(package.version) + + return versions diff --git a/sync/src/pom.py b/sync/src/pom.py index cb00aa3..0928f73 100644 --- a/sync/src/pom.py +++ b/sync/src/pom.py @@ -4,12 +4,18 @@ from typing import Optional, TypeAlias from xml.etree import ElementTree from config import Package -from xmlutils import pom_namespace as ns, find_tag_text +from xmlutils import find_tag_text logger = logging.getLogger(__name__) Properties: TypeAlias = dict[str, str] +ns = {'': 'http://maven.apache.org/POM/4.0.0'} + + +def pom_find_tag_text(parent: ElementTree.Element, tag: str) -> str | None: + return find_tag_text(parent, tag, ns) + class PropertyMissing(Exception): def __init__(self, prop: str): @@ -38,9 +44,9 @@ class PackagePOM: self.parent = None if (parent_tag := self._raw_root.find('parent', ns)) is not None: - parent_group = find_tag_text(parent_tag, 'groupId') - parent_artifact = find_tag_text(parent_tag, 'artifactId') - parent_version = find_tag_text(parent_tag, 'version') + parent_group = pom_find_tag_text(parent_tag, 'groupId') + parent_artifact = pom_find_tag_text(parent_tag, 'artifactId') + parent_version = pom_find_tag_text(parent_tag, 'version') if parent_group is None or parent_artifact is None or parent_version is None: raise PackageError(f'Invalid parent {parent_group}:{parent_artifact}:{parent_version}') diff --git a/sync/src/xmlutils.py b/sync/src/xmlutils.py index 1c2488c..972c9f0 100644 --- a/sync/src/xmlutils.py +++ b/sync/src/xmlutils.py @@ -1,6 +1,7 @@ -pom_namespace = {'': 'http://maven.apache.org/POM/4.0.0'} +from typing import Optional +from xml.etree import ElementTree -def find_tag_text(parent, tag) -> str | None: - elem = parent.find(tag, pom_namespace) +def find_tag_text(parent: ElementTree.Element, tag: str, namespace: Optional[dict[str, str]] = None) -> str | None: + elem = parent.find(tag, namespace) return elem.text if elem is not None else None