From 66e8cba3e7fb9644b7ce9435acbbd221d9085b15 Mon Sep 17 00:00:00 2001 From: Sindre Stephansen Date: Mon, 31 Jul 2023 09:47:55 +0200 Subject: [PATCH] Add a dataclass for package --- sync/src/config.py | 59 ++++++++++++++++++++++++-------------- sync/src/gradle.py | 7 +++-- sync/src/maven/fetch.py | 26 ++++++++++------- sync/src/maven/packages.py | 5 ++-- sync/src/pom.py | 41 ++++++++++++-------------- 5 files changed, 78 insertions(+), 60 deletions(-) diff --git a/sync/src/config.py b/sync/src/config.py index 1661abc..c16abd5 100644 --- a/sync/src/config.py +++ b/sync/src/config.py @@ -8,13 +8,23 @@ from pathlib import Path logger = logging.getLogger(__name__) +@dataclass +class Package: + group_id: str + artifact_id: str + version: str + + def __str__(self): + return f'{self.group_id}:{self.artifact_id}:{self.version}' + + @dataclass class Configuration: name: str kotlin_version: str gradle_version: Optional[str] plugins: dict[str, str] - packages: list[str] + packages: list[Package] @dataclass @@ -23,39 +33,44 @@ class Config: mirrors: list[str] -def handle_packages(section) -> list[str]: +def handle_packages(section) -> list[Package]: ignore = ['_versions'] - result = [] + result: list[Package] = [] for entry in section: if entry not in ignore: if ':' in entry: - versions = section[entry] - if isinstance(versions, str): - result.append(f'{entry}:{versions}') - elif isinstance(versions, list): - for version in versions: - result.append(f'{entry}:{version}') + try: + group_id, artifact_id = entry.split(':') + except ValueError: + logger.exception(f'Illegal package identifier "{entry}". Should be on the format "groupId:artifactId"') + continue + + value = section[entry] + if isinstance(value, str): + result.append(Package(group_id, artifact_id, value)) + elif isinstance(value, list): + for version in value: + result.append(Package(group_id, artifact_id, version)) else: - logger.warning(f'Invalid version "{versions}" for "{entry}". Should be a string or list.') + logger.warning(f'Invalid version "{value}" for "{entry}". Should be a string or list.') elif isinstance(section[entry], dict): + group_id = entry group_section = section[entry] default_versions = group_section.get('_versions', []) - for artifact in group_section: - if artifact not in ignore: - versions = group_section[artifact] - - if not versions and default_versions: - versions = default_versions + for artifact_id, value in group_section.items(): + if artifact_id not in ignore: + if not value and default_versions: + value = default_versions - if isinstance(versions, str): - result.append(f'{entry}:{artifact}:{versions}') - elif isinstance(versions, list) and versions: - for version in versions: - result.append(f'{entry}:{artifact}:{version}') + if isinstance(value, str): + result.append(Package(group_id, artifact_id, value)) + elif isinstance(value, list): + for version in value: + result.append(Package(group_id, artifact_id, version)) else: - logger.warning(f'Invalid versions "{versions}" for "{entry}:{artifact}"') + logger.warning(f'Invalid versions "{value}" for "{group_id}:{artifact_id}"') else: logger.warning(f'Invalid package spec "{entry}". Should be a full spec or a group ID') diff --git a/sync/src/gradle.py b/sync/src/gradle.py index 97048c1..5fc5984 100644 --- a/sync/src/gradle.py +++ b/sync/src/gradle.py @@ -1,3 +1,6 @@ +from config import Package + + def create_gradle_settings(repo: str) -> str: return """// Generated, do not edit rootProject.name = "gradle sync job" @@ -13,7 +16,7 @@ pluginManagement { """ -def create_gradle_build(kotlin_version: str, plugins: dict[str, str], packages: list[str], repo: str) -> str: +def create_gradle_build(kotlin_version: str, plugins: dict[str, str], packages: list[Package], repo: str) -> str: return """// Generated, do not edit plugins { """ + f'kotlin("jvm") version "{kotlin_version}"' + """ @@ -29,7 +32,7 @@ repositories { } val deps = listOf( - """ + ',\n '.join(f'"{dep}"' for dep in sorted(packages)) + """ + """ + ',\n '.join(f'"{dep}"' for dep in sorted([str(p) for p in packages])) + """ ).flatMap { listOf(it, it + ":sources", it + ":javadoc") }.map { diff --git a/sync/src/maven/fetch.py b/sync/src/maven/fetch.py index dd5cf21..7aa69d5 100644 --- a/sync/src/maven/fetch.py +++ b/sync/src/maven/fetch.py @@ -3,6 +3,7 @@ import asyncio from aiohttp import ClientSession from typing import Optional, Iterable +from config import Package from pom import PackagePOM logger = logging.getLogger(__name__) @@ -12,8 +13,13 @@ class TooManyRequestsException(Exception): pass -async def fetch_pom(package: str, mirrors: Iterable[str]) -> Optional[PackagePOM]: - pom = await fetch_maven_file(package, 'pom', mirrors) +def group_url(group_id: str) -> str: + return f'{group_id.replace(".", "/")}' + + +async def fetch_pom(package: Package, mirrors: Iterable[str]) -> Optional[PackagePOM]: + url = f'{group_url(package.group_id)}/{package.artifact_id}/{package.version}/{package.artifact_id}-{package.version}.pom' + pom = await fetch_maven_file(url, mirrors) return PackagePOM(package, pom) if pom else None @@ -27,10 +33,8 @@ async def fetch_from_mirror(session: ClientSession, url: str) -> str | int: return response.status -async def fetch_maven_file(package: str, extension: str, mirrors: Iterable[str]) -> Optional[str]: - group_id, artifact_id, version = package.split(':') - url = f'{group_id.replace(".", "/")}/{artifact_id}/{version}/{artifact_id}-{version}.{extension}' - logger.debug(f'{package}: Downloading {extension} from {url}') +async def fetch_maven_file(url: str, mirrors: Iterable[str]) -> Optional[str]: + logger.debug(f'Downloading {url}') async with ClientSession() as session: # Retry up to 5 times @@ -46,20 +50,20 @@ async def fetch_maven_file(package: str, extension: str, mirrors: Iterable[str]) result = await fetch_from_mirror(session, f'{mirror}/{url}') if isinstance(result, str): - logger.debug(f'{package}: {extension} downloaded') + logger.debug(f'{url} downloaded') return result else: - logger.debug(f'{package}: HTTP error {result} from mirror {mirror}') + logger.warning(f'HTTP error {result} from {mirror}/{url}. Trying next mirror.') except TooManyRequestsException: - logger.info(f'{package}: Received Too Many Requests error. Trying other mirror.') + logger.info(f'Received Too Many Requests error from {mirror}/{url}. Trying other mirror.') retry_mirrors.append(mirror) if retry_mirrors: - logger.info(f'{package}: Backing off, then trying again') + logger.info(f"Could not find {url}, but some mirrors didn't respond. Backing off, then trying again") mirrors = retry_mirrors await asyncio.sleep(0.1) else: break - logger.warning(f'{package}: File download of {extension} failed for all mirrors') + logger.error(f'Download of {url} failed for all mirrors') return None diff --git a/sync/src/maven/packages.py b/sync/src/maven/packages.py index 7d294ab..b9f3fee 100644 --- a/sync/src/maven/packages.py +++ b/sync/src/maven/packages.py @@ -1,13 +1,14 @@ import logging from typing import Optional, Iterable +from config import Package from maven.fetch import fetch_pom from pom import PropertyMissing, Properties logger = logging.getLogger(__name__) -async def get_effective_packages(package: str, mirrors: Iterable[str]) -> list[str]: +async def get_effective_packages(package: Package, mirrors: Iterable[str]) -> list[Package]: """ Get a list of packages that is required for Gradle to fetch this package. @@ -42,7 +43,7 @@ async def get_effective_packages(package: str, mirrors: Iterable[str]) -> list[s return packages -async def get_parent_props(parent: Optional[str], mirrors: Iterable[str]) -> Properties: +async def get_parent_props(parent: Optional[Package], mirrors: Iterable[str]) -> Properties: if parent: if pom := await fetch_pom(parent, mirrors): pom.set_properties(await get_parent_props(pom.parent, mirrors)) diff --git a/sync/src/pom.py b/sync/src/pom.py index b4ddca1..997859d 100644 --- a/sync/src/pom.py +++ b/sync/src/pom.py @@ -3,6 +3,7 @@ import re from typing import Optional, TypeAlias from xml.etree import ElementTree +from config import Package from xmlutils import pom_namespace as ns, find_tag_text logger = logging.getLogger(__name__) @@ -22,30 +23,20 @@ class PackageError(Exception): class PackagePOM: name: str is_bom: bool - parent: Optional[str] - group_id: Optional[str] - artifact_id: Optional[str] - version: Optional[str] + package: Package + parent: Optional[Package] properties: Properties _raw_root: ElementTree.Element - def __init__(self, name: str, pom: str): - logger.debug(f'{name}: Parsing POM') - self.name = name + def __init__(self, package: Package, pom: str): + self.package = package + self.name = str(package) + logger.debug(f'{self.name}: Parsing POM') self._raw_root = ElementTree.fromstring(pom) - self.group_id, self.artifact_id, self.version = name.split(':') - - if self.group_id is None or self.artifact_id is None or self.version is None: - logger.warning( - f'{name}: One of groupId={self.group_id}, artifactId={self.artifact_id}, version={self.version} was ' - f'None. This can cause issues with dependency resolution' - ) - self.parent = None - if (parent_tag := self._raw_root.find('parent', ns)) is not None: parent_group = find_tag_text(parent_tag, 'groupId') parent_artifact = find_tag_text(parent_tag, 'artifactId') @@ -54,7 +45,7 @@ class PackagePOM: if parent_group is None or parent_artifact is None or parent_version is None: raise PackageError(f'Invalid parent {parent_group}:{parent_artifact}:{parent_version}') else: - self.parent = f'{parent_group}:{parent_artifact}:{parent_version}' + self.parent = Package(parent_group, parent_artifact, parent_version) if (packaging := self._raw_root.find('packaging', ns)) is not None: self.is_bom = packaging == 'pom' @@ -64,7 +55,7 @@ class PackagePOM: self.set_properties({}) @property - def dependency_management(self) -> list[str]: + def dependency_management(self) -> list[Package]: return [ self._package_from_xml_dep(dep) for dep in self._raw_root.find('dependencyManagement/dependencies', ns) or [] @@ -99,11 +90,11 @@ class PackagePOM: prop = match.group(1) if prop == 'project.groupId': - value = str(self.group_id) + value = str(self.package.group_id) elif prop == 'project.artifactId': - value = str(self.artifact_id) + value = str(self.package.artifact_id) elif prop == 'project.version': - value = str(self.version) + value = str(self.package.version) elif prop.startswith('project.build') or prop.startswith('env.') or prop.startswith('maven.'): value = None elif prop in ['project.basedir', 'basedir', 'user.home', 'debug.port']: @@ -126,10 +117,14 @@ class PackagePOM: text, ) - def _package_from_xml_dep(self, dep: ElementTree.Element) -> str: + def _package_from_xml_dep(self, dep: ElementTree.Element) -> Package: def prop_replace_tag(tag) -> str: return self._prop_replace( elem.text or '' if (elem := dep.find(tag, ns)) is not None else '', ) - return f"{prop_replace_tag('groupId')}:{prop_replace_tag('artifactId')}:{prop_replace_tag('version')}" + return Package( + prop_replace_tag('groupId'), + prop_replace_tag('artifactId'), + prop_replace_tag('version'), + )