Use reposilite as mirror for everything

package-list.yaml

@@ -1,14 +1,3 @@
-maven:
-  package-mirrors:
-    - "https://repo.maven.apache.org/maven2"
-    - "https://repo1.maven.org/maven2"
-    - "https://oss.sonatype.org/content/repositories/snapshots"
-    - "https://packages.confluent.io/maven"
-    - "https://registry.quarkus.io/maven"
-
-  plugin-mirrors:
-    - "https://plugins.gradle.org/m2"
-
 configurations:
   # Specify configurations where the packages/plugins depend on a specific kotlin and/or gradle version.
   - kotlin-version: ["latest", "1.8.20", "1.7.0"]

sync/resolve-deps.sh

@@ -43,7 +43,7 @@ mkdir -p "${GRADLE_VERSIONS_DIR}"
 
 echo "Resolving packages and generating gradle config"
 python3 --version
-if ! python3 src/main.py --repo="repo:80" --output-dir "$PROJECTS_DIR" /package-list.yaml; then
+if ! python3 src/main.py --repo="http://repo:80/releases" --output-dir "$PROJECTS_DIR" /package-list.yaml; then
     echo "Gradle generation failed"
     exit 255
 fi

sync/src/config/__init__.py

@@ -1,3 +1,3 @@
 from .packages import Package
 from .plugins import Plugin
-from .parse import Configuration, Config, MavenMirrors, parse_config
+from .parse import Configuration, parse_config

sync/src/config/parse.py

@@ -19,19 +19,7 @@ class Configuration:
     packages: list[Package]
 
 
-@dataclass
-class MavenMirrors:
-    package_mirrors: list[str]
-    plugin_mirrors: list[str]
-
-
-@dataclass
-class Config:
-    configurations: list[Configuration]
-    mirrors: MavenMirrors
-
-
-def parse_config(path: Path) -> Optional[Config]:
+def parse_config(path: Path) -> Optional[list[Configuration]]:
     with path.open('r') as f:
         data = yaml.safe_load(f)
 
@@ -70,13 +58,7 @@ def parse_config(path: Path) -> Optional[Config]:
 
             return None
         else:
-            return Config(
-                [
-                    Configuration(kotlin, gradle, plugins, packages)
-                    for (kotlin, gradle), (plugins, packages) in configurations.items()
-                ],
-                MavenMirrors(
-                    data.get('maven', {}).get('package-mirrors', []),
-                    data.get('maven', {}).get('plugin-mirrors', []),
-                ),
-            )
+            return [
+                Configuration(kotlin, gradle, plugins, packages)
+                for (kotlin, gradle), (plugins, packages) in configurations.items()
+            ]

sync/src/gradle/build.py

@@ -16,7 +16,7 @@ plugins {
 """ + ("""
 repositories {
     maven {
-        url=uri("http://""" + repo + """/releases")
+        url=uri(\"""" + repo + """\")
         isAllowInsecureProtocol=true
     }
 }

sync/src/gradle/settings.py

@@ -10,7 +10,7 @@ rootProject.name = "gradle sync job"
 pluginManagement {
     repositories {
         maven {
-            url=uri("http://""" + repo + """/releases")
+            url=uri(\"""" + repo + """\")
             isAllowInsecureProtocol=true
         }
     }

sync/src/main.py

@@ -4,17 +4,17 @@ import dataclasses
 import logging
 from pathlib import Path
 
-from config import parse_config, Configuration, MavenMirrors, Package
+from config import parse_config, Configuration, Package
 from gradle import generate_gradle
 from maven import get_effective_packages, get_effective_plugins, resolve_version
 
 logger = logging.getLogger(__name__)
 
 
-async def resolve_kotlin(configuration: Configuration, mirrors: MavenMirrors) -> list[Configuration]:
+async def resolve_kotlin(configuration: Configuration, repo: str) -> list[Configuration]:
     kotlin_versions = await resolve_version(
         Package('org.jetbrains.kotlin', 'kotlin-stdlib', configuration.kotlin_version),
-        mirrors.package_mirrors,
+        repo,
     )
 
     if not kotlin_versions:
@@ -24,13 +24,13 @@ async def resolve_kotlin(configuration: Configuration, mirrors: MavenMirrors) ->
     packages = [
         resolved
         for package in configuration.packages
-        for resolved in await get_effective_packages(package, mirrors)
+        for resolved in await get_effective_packages(package, repo)
     ]
 
     plugins = [
         resolved
         for plugin in configuration.plugins
-        for resolved in await get_effective_plugins(plugin, mirrors)
+        for resolved in await get_effective_plugins(plugin, repo)
     ]
 
     return [
@@ -39,25 +39,19 @@ async def resolve_kotlin(configuration: Configuration, mirrors: MavenMirrors) ->
     ]
 
 
-async def main(package_list: Path, output_dir: Path, gradle_repo: str) -> None:
-    config = parse_config(package_list)
+async def main(package_list: Path, output_dir: Path, repo: str) -> None:
+    configurations = parse_config(package_list)
 
-    if not config:
-        print('Error in configuration')
-    elif not config.configurations:
-        print('No configurations defined, nothing to do.')
-    elif not config.mirrors.package_mirrors and [p for c in config.configurations for p in c.packages]:
-        print('No package mirrors defined. Add maven.package-mirrors in the config file.')
-    elif not config.mirrors.plugin_mirrors and [p for c in config.configurations for p in c.plugins]:
-        print('No plugin mirrors defined. Add maven.plugin-mirrors in the config file.')
-    else:
-        for i, configuration in enumerate(config.configurations):
+    if configurations:
+        for configuration in configurations:
             logger.info(f'Resolving configuration kotlin={configuration.kotlin_version}, gradle={configuration.gradle_version}')
             try:
-                for c in await resolve_kotlin(configuration, config.mirrors):
-                    generate_gradle(c, gradle_repo, output_dir)
+                for c in await resolve_kotlin(configuration, repo):
+                    generate_gradle(c, repo, output_dir)
             except:
                 logger.exception(f'Error resolving kotlin version {configuration.kotlin_version}')
+    else:
+        print('Error in configuration')
 
 
 if __name__ == '__main__':

sync/src/maven/fetch.py

@@ -1,7 +1,7 @@
 import logging
-import asyncio
+from typing import Optional
+
 from aiohttp import ClientSession
-from typing import Optional, Iterable
 
 from config import Package
 from .metadata import MavenMetadata
@@ -10,67 +10,29 @@ from .pom import PackagePOM
 logger = logging.getLogger(__name__)
 
 
-class TooManyRequestsException(Exception):
-    pass
-
-
 def group_url(group_id: str) -> str:
     return f'{group_id.replace(".", "/")}'
 
 
-async def fetch_metadata(package: Package, mirrors: Iterable[str]) -> Optional[MavenMetadata]:
+async def fetch_metadata(package: Package, repo: str) -> Optional[MavenMetadata]:
     url = f'{group_url(package.group_id)}/{package.artifact_id}/maven-metadata.xml'
-    metadata = await fetch_maven_file(url, mirrors)
+    metadata = await fetch_maven_file(f'{repo}/{url}')
     return MavenMetadata(package, metadata) if metadata else None
 
 
-async def fetch_pom(package: Package, mirrors: Iterable[str]) -> Optional[PackagePOM]:
+async def fetch_pom(package: Package, repo) -> Optional[PackagePOM]:
     url = f'{group_url(package.group_id)}/{package.artifact_id}/{package.version}/{package.artifact_id}-{package.version}.pom'
-    pom = await fetch_maven_file(url, mirrors)
+    pom = await fetch_maven_file(f'{repo}/{url}')
     return PackagePOM(package, pom) if pom else None
 
 
-async def fetch_from_mirror(session: ClientSession, url: str) -> str | int:
-    async with session.get(url) as response:
-        if response.status == 200:
-            return await response.text()
-        elif response.status == 429:
-            raise TooManyRequestsException()
-        else:
-            return response.status
-
-
-async def fetch_maven_file(url: str, mirrors: Iterable[str]) -> Optional[str]:
+async def fetch_maven_file(url: str) -> Optional[str]:
     logger.debug(f'Downloading {url}')
 
     async with ClientSession() as session:
-        # Retry up to 5 times
-        for _ in range(5):
-            # Go through the mirrors, trying to find the package.
-            # If the package doesn't exist in a mirror, that mirror is removed from the list.
-            # If another error occurs the mirror is kept.
-            # After trying all mirrors, if the package isn't found, the remaining mirrors are retried.
-
-            retry_mirrors = []
-            for mirror in mirrors:
-                try:
-                    result = await fetch_from_mirror(session, f'{mirror}/{url}')
-
-                    if isinstance(result, str):
-                        logger.debug(f'{url} downloaded')
-                        return result
-                    else:
-                        logger.warning(f'HTTP error {result} from {mirror}/{url}. Trying next mirror.')
-                except TooManyRequestsException:
-                    logger.info(f'Received Too Many Requests error from {mirror}/{url}. Trying other mirror.')
-                    retry_mirrors.append(mirror)
-
-            if retry_mirrors:
-                logger.info(f"Could not find {url}, but some mirrors didn't respond. Backing off, then trying again")
-                mirrors = retry_mirrors
-                await asyncio.sleep(0.1)
+        async with session.get(url) as response:
+            if response.status == 200:
+                return await response.text()
             else:
-                break
-
-        logger.error(f'Download of {url} failed for all mirrors')
-        return None
+                logger.warning(f'HTTP error {response.status} from {url}.')
+                return None

sync/src/maven/packages.py

@@ -2,7 +2,7 @@ import dataclasses
 import logging
 from typing import Optional, Iterable
 
-from config import Package, MavenMirrors
+from config import Package
 from .fetch import fetch_pom
 from .version import resolve_version
 from .pom import PropertyMissing, Properties
@@ -10,7 +10,7 @@ from .pom import PropertyMissing, Properties
 logger = logging.getLogger(__name__)
 
 
-async def get_effective_packages(package: Package, mirrors: MavenMirrors) -> list[Package]:
+async def get_effective_packages(package: Package, repo: str) -> list[Package]:
     """
     Get a list of packages that is required for Gradle to fetch this package.
 
@@ -25,13 +25,13 @@ async def get_effective_packages(package: Package, mirrors: MavenMirrors) -> lis
     """
     versions = [
         dataclasses.replace(package, version=version)
-        for version in await resolve_version(package, mirrors.package_mirrors)
+        for version in await resolve_version(package, repo)
     ]
 
     deps = [
         dep
         for v in versions
-        for dep in await get_dependencies(v, mirrors.package_mirrors)
+        for dep in await get_dependencies(v, repo)
     ]
 
     return versions + deps

sync/src/maven/plugins.py

@@ -1,14 +1,14 @@
 import dataclasses
 
-from config import Plugin, MavenMirrors
+from config import Plugin
 from .version import resolve_version
 
 
-async def get_effective_plugins(plugin: Plugin, mirrors: MavenMirrors) -> list[Plugin]:
+async def get_effective_plugins(plugin: Plugin, repo: str) -> list[Plugin]:
     if plugin.version:
         return [
             dataclasses.replace(plugin, version=version)
-            for version in await resolve_version(plugin.package, mirrors.plugin_mirrors)
+            for version in await resolve_version(plugin.package, repo)
         ]
     else:
         return [plugin]

sync/src/maven/version.py

@@ -1,5 +1,4 @@
 import logging
-from typing import Iterable
 
 from config import Package
 from .fetch import fetch_metadata
@@ -7,11 +6,11 @@ from .fetch import fetch_metadata
 logger = logging.getLogger(__name__)
 
 
-async def resolve_version(package: Package, mirrors: Iterable[str]) -> list[str]:
+async def resolve_version(package: Package, repo: str) -> list[str]:
     versions = []
 
     if package.version in ['latest', 'stable']:
-        if metadata := await fetch_metadata(package, mirrors):
+        if metadata := await fetch_metadata(package, repo):
             if package.version == 'latest':
                 versions.append(metadata.latest)
             elif package.version == 'stable':