diff --git a/package-list.yaml b/package-list.yaml index 65e6509..a5bf366 100644 --- a/package-list.yaml +++ b/package-list.yaml @@ -1,14 +1,3 @@ -maven: - package-mirrors: - - "https://repo.maven.apache.org/maven2" - - "https://repo1.maven.org/maven2" - - "https://oss.sonatype.org/content/repositories/snapshots" - - "https://packages.confluent.io/maven" - - "https://registry.quarkus.io/maven" - - plugin-mirrors: - - "https://plugins.gradle.org/m2" - configurations: # Specify configurations where the packages/plugins depend on a specific kotlin and/or gradle version. - kotlin-version: ["latest", "1.8.20", "1.7.0"] diff --git a/sync/resolve-deps.sh b/sync/resolve-deps.sh index 9058a92..d22d6af 100755 --- a/sync/resolve-deps.sh +++ b/sync/resolve-deps.sh @@ -43,7 +43,7 @@ mkdir -p "${GRADLE_VERSIONS_DIR}" echo "Resolving packages and generating gradle config" python3 --version -if ! python3 src/main.py --repo="repo:80" --output-dir "$PROJECTS_DIR" /package-list.yaml; then +if ! python3 src/main.py --repo="http://repo:80/releases" --output-dir "$PROJECTS_DIR" /package-list.yaml; then echo "Gradle generation failed" exit 255 fi diff --git a/sync/src/config/__init__.py b/sync/src/config/__init__.py index f24bdab..b23386e 100644 --- a/sync/src/config/__init__.py +++ b/sync/src/config/__init__.py @@ -1,3 +1,3 @@ from .packages import Package from .plugins import Plugin -from .parse import Configuration, Config, MavenMirrors, parse_config +from .parse import Configuration, parse_config diff --git a/sync/src/config/parse.py b/sync/src/config/parse.py index 7c06f13..c8d33e3 100644 --- a/sync/src/config/parse.py +++ b/sync/src/config/parse.py @@ -19,19 +19,7 @@ class Configuration: packages: list[Package] -@dataclass -class MavenMirrors: - package_mirrors: list[str] - plugin_mirrors: list[str] - - -@dataclass -class Config: - configurations: list[Configuration] - mirrors: MavenMirrors - - -def parse_config(path: Path) -> Optional[Config]: +def parse_config(path: Path) -> Optional[list[Configuration]]: with path.open('r') as f: data = yaml.safe_load(f) @@ -70,13 +58,7 @@ def parse_config(path: Path) -> Optional[Config]: return None else: - return Config( - [ - Configuration(kotlin, gradle, plugins, packages) - for (kotlin, gradle), (plugins, packages) in configurations.items() - ], - MavenMirrors( - data.get('maven', {}).get('package-mirrors', []), - data.get('maven', {}).get('plugin-mirrors', []), - ), - ) + return [ + Configuration(kotlin, gradle, plugins, packages) + for (kotlin, gradle), (plugins, packages) in configurations.items() + ] diff --git a/sync/src/gradle/build.py b/sync/src/gradle/build.py index 3aa2b5e..b907e7b 100644 --- a/sync/src/gradle/build.py +++ b/sync/src/gradle/build.py @@ -16,7 +16,7 @@ plugins { """ + (""" repositories { maven { - url=uri("http://""" + repo + """/releases") + url=uri(\"""" + repo + """\") isAllowInsecureProtocol=true } } diff --git a/sync/src/gradle/settings.py b/sync/src/gradle/settings.py index b2e106b..a9f5725 100644 --- a/sync/src/gradle/settings.py +++ b/sync/src/gradle/settings.py @@ -10,7 +10,7 @@ rootProject.name = "gradle sync job" pluginManagement { repositories { maven { - url=uri("http://""" + repo + """/releases") + url=uri(\"""" + repo + """\") isAllowInsecureProtocol=true } } diff --git a/sync/src/main.py b/sync/src/main.py index f1f752d..d6e3112 100644 --- a/sync/src/main.py +++ b/sync/src/main.py @@ -4,17 +4,17 @@ import dataclasses import logging from pathlib import Path -from config import parse_config, Configuration, MavenMirrors, Package +from config import parse_config, Configuration, Package from gradle import generate_gradle from maven import get_effective_packages, get_effective_plugins, resolve_version logger = logging.getLogger(__name__) -async def resolve_kotlin(configuration: Configuration, mirrors: MavenMirrors) -> list[Configuration]: +async def resolve_kotlin(configuration: Configuration, repo: str) -> list[Configuration]: kotlin_versions = await resolve_version( Package('org.jetbrains.kotlin', 'kotlin-stdlib', configuration.kotlin_version), - mirrors.package_mirrors, + repo, ) if not kotlin_versions: @@ -24,13 +24,13 @@ async def resolve_kotlin(configuration: Configuration, mirrors: MavenMirrors) -> packages = [ resolved for package in configuration.packages - for resolved in await get_effective_packages(package, mirrors) + for resolved in await get_effective_packages(package, repo) ] plugins = [ resolved for plugin in configuration.plugins - for resolved in await get_effective_plugins(plugin, mirrors) + for resolved in await get_effective_plugins(plugin, repo) ] return [ @@ -39,25 +39,19 @@ async def resolve_kotlin(configuration: Configuration, mirrors: MavenMirrors) -> ] -async def main(package_list: Path, output_dir: Path, gradle_repo: str) -> None: - config = parse_config(package_list) +async def main(package_list: Path, output_dir: Path, repo: str) -> None: + configurations = parse_config(package_list) - if not config: - print('Error in configuration') - elif not config.configurations: - print('No configurations defined, nothing to do.') - elif not config.mirrors.package_mirrors and [p for c in config.configurations for p in c.packages]: - print('No package mirrors defined. Add maven.package-mirrors in the config file.') - elif not config.mirrors.plugin_mirrors and [p for c in config.configurations for p in c.plugins]: - print('No plugin mirrors defined. Add maven.plugin-mirrors in the config file.') - else: - for i, configuration in enumerate(config.configurations): + if configurations: + for configuration in configurations: logger.info(f'Resolving configuration kotlin={configuration.kotlin_version}, gradle={configuration.gradle_version}') try: - for c in await resolve_kotlin(configuration, config.mirrors): - generate_gradle(c, gradle_repo, output_dir) + for c in await resolve_kotlin(configuration, repo): + generate_gradle(c, repo, output_dir) except: logger.exception(f'Error resolving kotlin version {configuration.kotlin_version}') + else: + print('Error in configuration') if __name__ == '__main__': diff --git a/sync/src/maven/fetch.py b/sync/src/maven/fetch.py index 3f02878..35dee09 100644 --- a/sync/src/maven/fetch.py +++ b/sync/src/maven/fetch.py @@ -1,7 +1,7 @@ import logging -import asyncio +from typing import Optional + from aiohttp import ClientSession -from typing import Optional, Iterable from config import Package from .metadata import MavenMetadata @@ -10,67 +10,29 @@ from .pom import PackagePOM logger = logging.getLogger(__name__) -class TooManyRequestsException(Exception): - pass - - def group_url(group_id: str) -> str: return f'{group_id.replace(".", "/")}' -async def fetch_metadata(package: Package, mirrors: Iterable[str]) -> Optional[MavenMetadata]: +async def fetch_metadata(package: Package, repo: str) -> Optional[MavenMetadata]: url = f'{group_url(package.group_id)}/{package.artifact_id}/maven-metadata.xml' - metadata = await fetch_maven_file(url, mirrors) + metadata = await fetch_maven_file(f'{repo}/{url}') return MavenMetadata(package, metadata) if metadata else None -async def fetch_pom(package: Package, mirrors: Iterable[str]) -> Optional[PackagePOM]: +async def fetch_pom(package: Package, repo) -> Optional[PackagePOM]: url = f'{group_url(package.group_id)}/{package.artifact_id}/{package.version}/{package.artifact_id}-{package.version}.pom' - pom = await fetch_maven_file(url, mirrors) + pom = await fetch_maven_file(f'{repo}/{url}') return PackagePOM(package, pom) if pom else None -async def fetch_from_mirror(session: ClientSession, url: str) -> str | int: - async with session.get(url) as response: - if response.status == 200: - return await response.text() - elif response.status == 429: - raise TooManyRequestsException() - else: - return response.status - - -async def fetch_maven_file(url: str, mirrors: Iterable[str]) -> Optional[str]: +async def fetch_maven_file(url: str) -> Optional[str]: logger.debug(f'Downloading {url}') async with ClientSession() as session: - # Retry up to 5 times - for _ in range(5): - # Go through the mirrors, trying to find the package. - # If the package doesn't exist in a mirror, that mirror is removed from the list. - # If another error occurs the mirror is kept. - # After trying all mirrors, if the package isn't found, the remaining mirrors are retried. - - retry_mirrors = [] - for mirror in mirrors: - try: - result = await fetch_from_mirror(session, f'{mirror}/{url}') - - if isinstance(result, str): - logger.debug(f'{url} downloaded') - return result - else: - logger.warning(f'HTTP error {result} from {mirror}/{url}. Trying next mirror.') - except TooManyRequestsException: - logger.info(f'Received Too Many Requests error from {mirror}/{url}. Trying other mirror.') - retry_mirrors.append(mirror) - - if retry_mirrors: - logger.info(f"Could not find {url}, but some mirrors didn't respond. Backing off, then trying again") - mirrors = retry_mirrors - await asyncio.sleep(0.1) + async with session.get(url) as response: + if response.status == 200: + return await response.text() else: - break - - logger.error(f'Download of {url} failed for all mirrors') - return None + logger.warning(f'HTTP error {response.status} from {url}.') + return None diff --git a/sync/src/maven/packages.py b/sync/src/maven/packages.py index bc20590..f96fa4f 100644 --- a/sync/src/maven/packages.py +++ b/sync/src/maven/packages.py @@ -2,7 +2,7 @@ import dataclasses import logging from typing import Optional, Iterable -from config import Package, MavenMirrors +from config import Package from .fetch import fetch_pom from .version import resolve_version from .pom import PropertyMissing, Properties @@ -10,7 +10,7 @@ from .pom import PropertyMissing, Properties logger = logging.getLogger(__name__) -async def get_effective_packages(package: Package, mirrors: MavenMirrors) -> list[Package]: +async def get_effective_packages(package: Package, repo: str) -> list[Package]: """ Get a list of packages that is required for Gradle to fetch this package. @@ -25,13 +25,13 @@ async def get_effective_packages(package: Package, mirrors: MavenMirrors) -> lis """ versions = [ dataclasses.replace(package, version=version) - for version in await resolve_version(package, mirrors.package_mirrors) + for version in await resolve_version(package, repo) ] deps = [ dep for v in versions - for dep in await get_dependencies(v, mirrors.package_mirrors) + for dep in await get_dependencies(v, repo) ] return versions + deps diff --git a/sync/src/maven/plugins.py b/sync/src/maven/plugins.py index de619fc..f9482ee 100644 --- a/sync/src/maven/plugins.py +++ b/sync/src/maven/plugins.py @@ -1,14 +1,14 @@ import dataclasses -from config import Plugin, MavenMirrors +from config import Plugin from .version import resolve_version -async def get_effective_plugins(plugin: Plugin, mirrors: MavenMirrors) -> list[Plugin]: +async def get_effective_plugins(plugin: Plugin, repo: str) -> list[Plugin]: if plugin.version: return [ dataclasses.replace(plugin, version=version) - for version in await resolve_version(plugin.package, mirrors.plugin_mirrors) + for version in await resolve_version(plugin.package, repo) ] else: return [plugin] diff --git a/sync/src/maven/version.py b/sync/src/maven/version.py index cde54d9..cdc7d21 100644 --- a/sync/src/maven/version.py +++ b/sync/src/maven/version.py @@ -1,5 +1,4 @@ import logging -from typing import Iterable from config import Package from .fetch import fetch_metadata @@ -7,11 +6,11 @@ from .fetch import fetch_metadata logger = logging.getLogger(__name__) -async def resolve_version(package: Package, mirrors: Iterable[str]) -> list[str]: +async def resolve_version(package: Package, repo: str) -> list[str]: versions = [] if package.version in ['latest', 'stable']: - if metadata := await fetch_metadata(package, mirrors): + if metadata := await fetch_metadata(package, repo): if package.version == 'latest': versions.append(metadata.latest) elif package.version == 'stable':