Rewrite to support multiple kotlin versions (#27)

* WIP Try to add variant support

* Rewrite package resolver

The new version adds support for multiple kotlin versions, and direct support for plugins. The total runtime is lower because the resolver does less work.

* Update README

README.md

@@ -11,13 +11,48 @@ Run the following command:
 This starts the reposilite server, then starts maven and makes download all packages defined in the pom's in the `poms/` folder, including their dependencies. These packages are then cached by reposilite in `data/`. The relevant folders are turned into tarballs and saved in the root directory.
 
 ## Adding packages
-### Automatically
-New packages can be added to the `package-list.txt` file. The group ID and artifact ID are required, but the version is optional. If the version number is omitted the latest version is found. Run `./update-poms.sh` to generate the pom files from the package file.
-The root `pom.xml` file is updated automatically.
+New packages are added to `package-list.yaml`.
 
-### Manually
-Pom files can be added manually by creating a folder with a `pom.xml` file in the `poms/` folder.
+Example config:
+```yaml
+maven:
+  # List of mirrors to query for package details.
+  mirrors:
+    - "https://repo.maven.apache.org/maven2"
 
-After the `pom.xml` is created, run `./generate_master_pom.xml` to update the root `pom.xml` file, and commit the changes.
+# Specify which kotlin versions to fetch.
+kotlin:
+  1.8.20:
+    # Plugins specific to this kotlin version. See `general` section for details.
+    plugins:
+      com.expediagroup.graphql: ["7.0.0-alpha.5", "6.5.2"]
+    # Packages specific to this kotlin version. See `general` section for details.
+    packages:
+      org.slf4j:slf4j-api: "2.1.0"
+  # A kotlin version without specific plugins and packages.
+  1.8.0: {}
+  1.7.20:
+    packages:
+      org.slf4j:slf4j-api: "2.0.7"
 
-This structure is necessary to make dependabot find all dependencies.
+# Packages and plugins that doesn't require a specific kotlin version.
+general:
+  # The kotlin version to use (doesn't matter much, use the newest stable version).
+  _kotlin-version: "1.8.20"
+  
+  plugins:
+    # Plugins are listed with their plugin ID, not the full package name.
+    # Plugin versions can be a single version as a string, or a list of strings to fetch multiple versions.
+    org.panteleyev.jpackageplugin: "1.5.2"
+  packages:
+    # Packages are listed either with their full name as a key, or split into group and artifact.
+    # Package versions can be a single version as a string, or a list of strings to fetch multiple versions.
+    org.jetbrains.kotlinx:kotlinx-datetime: "0.4.0"
+    com.expediagroup:
+      # When splitting into group and artifact, the default versions for the group can be specified.
+      # Using an empty list as the version for an artifact implies that the default versions should be used.
+      _versions: ["7.0.0-alpha.6", "6.5.2"]
+      graphql-kotlin-ktor-server: ["7.0.0-alpha.6"] # Uses only the specified version
+      graphql-kotlin-client: [] # Uses the default versions
+      graphql-kotlin-client-generator: [] # Uses the default versions
+```

docker-compose.yml

@@ -19,4 +19,4 @@ services:
     depends_on:
       - repo
     volumes:
-      - ./package-list.txt:/package-list.txt
+      - ./package-list.yaml:/package-list.yaml

package-list.txt

@@ -1,70 +0,0 @@
-org.jetbrains.kotlin:kotlin-bom:1.7.20
-org.jetbrains.kotlinx:kotlinx-coroutines-core:1.6.4
-org.jetbrains.kotlinx:kotlinx-datetime:0.4.0
-org.jetbrains.kotlin:kotlin-serialization:1.7.20
-org.jetbrains.kotlin:kotlin-maven-serialization:1.6.21
-org.jetbrains.kotlin:kotlin-maven-serialization
-org.jetbrains.kotlin:kotlin-maven-plugin:1.7.20
-org.jetbrains.kotlin:kotlin-maven-allopen:1.7.20
-org.jetbrains.kotlin:kotlin-maven-noarg:1.7.20
-org.jetbrains.kotlin:kotlin-gradle-plugin:1.7.20
-org.jetbrains.kotlin:kotlin-gradle-plugin-idea:1.7.20
-org.jetbrains.kotlin.jvm:org.jetbrains.kotlin.jvm.gradle.plugin:1.7.20
-org.jetbrains.kotlin.jvm:org.jetbrains.kotlin.jvm.gradle.plugin:1.7.20
-org.jetbrains.kotlin.plugin.serialization:org.jetbrains.kotlin.plugin.serialization.gradle.plugin:1.7.20
-org.jetbrains.exposed:exposed-bom:0.41.1
-org.slf4j:slf4j-api:2.0.6
-org.junit:junit-bom:5.9.1
-com.github.node-gradle.node:com.github.node-gradle.node.gradle.plugin:3.5.1
-io.kotest:kotest-bom:5.5.4
-io.kotest:kotest-runner-junit5:5.5.4
-org.asciidoctor:asciidoctor-maven-plugin
-org.asciidoctor:asciidoctorj:2.5.7
-org.asciidoctor:asciidoctorj-api:2.5.7
-org.asciidoctor:asciidoctorj-test-support:2.5.7
-org.asciidoctor:asciidoctorj-revealjs:4.1.0
-org.asciidoctor:asciidoctorj-diagram-plantuml
-org.asciidoctor:asciidoctorj-pdf:2.3.4
-org.asciidoctor:asciidoctorj-diagram:2.2.3
-com.vladmihalcea:hibernate-types-55
-org.apache.maven.plugins:maven-surefire-plugin
-org.apache.maven.plugins:maven-failsafe-plugin
-org.apache.maven.plugins:maven-shade-plugin
-org.apache.maven.plugins:maven-compiler-plugin
-io.insert-koin:koin-core:3.3.0
-io.insert-koin:koin-core:3.2.2
-io.insert-koin:koin-test:3.3.0
-io.insert-koin:koin-test:3.2.2
-io.insert-koin:koin-test-core:2.2.3
-io.insert-koin:koin-test-junit5:3.3.0
-io.insert-koin:koin-test-junit5:3.2.2
-io.insert-koin:koin-ktor:3.2.2
-org.kodein.di:kodein-di
-org.kodein.di:kodein-di:7.16.0
-org.kodein.di:kodein-di-framework-ktor-server-jvm
-org.kodein.di:kodein-di-framework-ktor-server-jvm:7.16.0
-io.ktor:ktor-bom
-io.ktor:ktor-bom:2.2.1
-io.mockk:mockk
-io.mockk:mockk:1.13.3
-io.quarkiverse.mockk:quarkus-junit5-mockk
-org.glassfish.corba:glassfish-corba
-org.glassfish.corba:glassfish-corba-tests
-org.glassfish.corba:glassfish-corba-internal-api
-org.glassfish.corba:glassfish-corba-omgapi
-org.glassfish.corba:glassfish-corba-orb
-org.glassfish.corba:glassfish-corba-asm
-org.glassfish.corba:glassfish-corba-csiv2-idl
-org.glassfish.corba:exception-annotation-processor
-org.glassfish.corba:idlj
-ch.qos.logback:logback-classic:1.2.11
-org.snmp4j:snmp4j:3.7.4
-io.github.microutils:kotlin-logging:3.0.4
-io.github.microutils:kotlin-logging
-org.apache.sshd:sshd:2.9.2
-org.apache.sshd:sshd-core:2.9.2
-org.apache.sshd:sshd-common:2.9.2
-org.apache.sshd:sshd-sftp:2.9.2
-org.apache.sshd:sshd-scp:2.9.2
-org.apache.sshd:sshd-ldap:2.9.2
-org.hibernate.reactive:hibernate-reactive-core

package-list.yaml

@@ -0,0 +1,29 @@
+maven:
+  mirrors:
+    - "https://repo.maven.apache.org/maven2"
+    - "https://repo1.maven.org/maven2"
+    - "https://oss.sonatype.org/content/repositories/snapshots"
+    - "https://packages.confluent.io/maven"
+    - "https://registry.quarkus.io/maven"
+    - "https://plugins.gradle.org/m2"
+
+kotlin:
+  1.8.20:
+    plugins:
+      com.expediagroup.graphql: ["7.0.0-alpha.5", "6.5.2"]
+  1.8.0: {}
+  1.7.20:
+    packages:
+      org.slf4j:slf4j-api: "2.0.7"
+
+general:
+  _kotlin-version: "1.8.20"
+  plugins:
+    org.panteleyev.jpackageplugin: "1.5.2"
+  packages:
+    org.jetbrains.kotlinx:kotlinx-datetime: "0.4.0"
+    com.expediagroup:
+      _versions: ["7.0.0-alpha.6", "6.5.2"]
+      graphql-kotlin-ktor-server: ["7.0.0-alpha.6"]
+      graphql-kotlin-client: []
+      graphql-kotlin-client-generator: []

sync/.dockerignore

@@ -1,4 +1,4 @@
 **
 !resolve-deps.sh
-!generate-gradle.py
+!src
 !requirements.txt

sync/Dockerfile

@@ -8,6 +8,6 @@ COPY requirements.txt .
 RUN python3 -m pip install -r requirements.txt
 
 COPY resolve-deps.sh .
-COPY generate-gradle.py .
+COPY src ./src
 
 CMD [ "/bin/sh", "./resolve-deps.sh" ]

sync/generate-gradle.py

@@ -1,579 +0,0 @@
-#!/bin/python3
-
-import re
-import copy
-import random
-import argparse
-import logging
-import asyncio
-import subprocess
-import copy
-import aiohttp
-from pathlib import Path
-from xml.etree import ElementTree as ET
-
-
-ns = {'': 'http://maven.apache.org/POM/4.0.0'}
-ET.register_namespace('', ns[''])
-
-baseurl = 'https://search.maven.org'
-base_pom_path = Path('poms')
-mirrors = [
-    "https://repo.maven.apache.org/maven2",
-    "https://repo1.maven.org/maven2",
-    "https://oss.sonatype.org/content/repositories/snapshots",
-    "https://packages.confluent.io/maven",
-    "https://registry.quarkus.io/maven",
-    "https://plugins.gradle.org/m2",
-]
-
-done: set[str] = set()
-done_lock = asyncio.Lock()
-in_progress: set[str] = set()
-in_progress_lock = asyncio.Lock()
-gradle_packages: set[str] = set()
-gradle_packages_lock = asyncio.Lock()
-
-global_properties: dict[str, dict[str, str]] = {}
-
-
-class TooManyRequestsException(Exception):
-    pass
-
-
-class PackageError(Exception):
-    pass
-
-class WaitForPackage(Exception):
-    def __init__(self, package):
-        self.package = package
-
-def find_tag_text(parent, tag) -> str | None:
-    elem = parent.find(tag, ns)
-    return elem.text if elem is not None else None
-
-class PackagePOM:
-    def __init__(self, package: 'Package', pom: str):
-        self._package = package
-
-        logger.debug(f'{package}: Parsing POM')
-        self.raw_root = ET.fromstring(pom)
-
-        self.parent: Package | None = None
-
-        if (parent_tag := self.raw_root.find('parent', ns)) is not None:
-            parent_group = find_tag_text(parent_tag, 'groupId')
-            parent_artifact = find_tag_text(parent_tag, 'artifactId')
-            parent_version = find_tag_text(parent_tag, 'version')
-
-            logger.debug(f'{package}: Parsing parent {parent_group}:{parent_artifact}:{parent_version}')
-
-            if parent_group is not None and parent_artifact is not None and parent_version is not None:
-                parent = Package(
-                    parent_group,
-                    parent_artifact,
-                    parent_version,
-                )
-
-                if str(parent) in done:
-                    self.parent = parent
-                else:
-                    raise WaitForPackage(parent)
-            else:
-                raise PackageError(f'Invalid parent {parent_group}:{parent_artifact}:{parent_version}')
-
-        logger.debug(f'{package}: Parsing properties')
-        parent_props: dict[str, str] = {} if self.parent is None else global_properties[str(self.parent)]
-        self.properties = self.resolve_props(parent_props)
-        global_properties[str(package)] = self.properties
-
-        logger.debug(f'{package}: Parsing packaging')
-        if (packaging := self.raw_root.find('packaging', ns)) is not None:
-            self.packaging = packaging.text
-        else:
-            self.packaging = '??'
-
-        self.is_bom = self.packaging == 'pom'
-
-        self.gradle_packages = [str(package)]
-
-        if self.is_bom:
-            logger.debug(f'{package}: Parsing dependencyManagement')
-
-            if (dependencyManagement := self.raw_root.find('dependencyManagement', ns)):
-                if (dependencies := dependencyManagement.find('dependencies', ns)):
-                    packages = []
-
-                    for dep in dependencies.findall('dependency', ns):
-                        groupId = find_tag_text(dep, 'groupId')
-                        artifactId = find_tag_text(dep, 'artifactId')
-                        version = find_tag_text(dep, 'version')
-
-
-                        if groupId is not None and artifactId is not None and version is not None:
-                            groupId = self.prop_replace(groupId)
-                            artifactId = self.prop_replace(artifactId)
-                            version = self.prop_replace(version)
-                            packages.append(f'{groupId}:{artifactId}:{version}')
-
-                    logger.debug(f'{package}: Adding {len(packages)} package(s) from dependencyManagement')
-
-                    self.gradle_packages.extend(packages)
-                else:
-                    logger.warn(f'{package}: dependencyManagement has no dependencies')
-            else:
-                logger.warn(f'{package}: BOM has no dependencyManagement')
-
-        logger.debug(f'{package}: POM parsed')
-
-    def resolve_props(self, initial: dict[str, str]):
-        props = initial
-
-        for prop_tag in self.raw_root.findall('.//properties/*', ns):
-            prop = prop_tag.tag.replace(f'{{{ns[""]}}}', '')
-            value = prop_tag.text if prop_tag.text is not None else ''
-            logger.debug(f'{self._package}: Setting prop {prop}={value}')
-            props[prop] = value
-
-        changed = True
-        while changed:
-            changed = False
-
-            for prop, value in props.items():
-                new_value = self.prop_replace(value, props)
-
-                if new_value != value:
-                    changed = True
-                    logger.debug(f'{self._package}: Setting prop {prop}={new_value}')
-                    props[prop] = new_value
-
-        return props
-
-    def prop_replace(self, text, props: dict[str, str] | None = None) -> str:
-        def lookup_prop(match) -> str:
-            prop = match.group(1)
-
-            if prop == 'project.groupId':
-                value = str(self._package.groupId)
-            elif prop == 'project.artifactId':
-                value = str(self._package.artifactId)
-            elif prop == 'project.version':
-                value = str(self._package.version)
-            elif prop.startswith('project.build') or prop.startswith('env.') or prop.startswith('maven.'):
-                value = ''
-            elif prop in ['project.basedir', 'basedir', 'user.home', 'debug.port']:
-                value = ''
-            else:
-                try:
-                    value = props[prop] if props is not None else self.properties[prop]
-                except KeyError:
-                    logger.error(f'{self._package}: Could not find property {prop}. Setting it to ""')
-                    value = ''
-
-            logger.debug(f'{self._package}: Replacing property {prop} with {value}')
-            return value
-
-        return re.sub(
-            r'\$\{([^\}]*)\}',
-            lookup_prop,
-            text,
-        )
-
-    def _package_from_xml_dep(self, dep: ET.Element) -> 'Package':
-        def prop_replace_tag(tag) -> str:
-            return self.prop_replace(
-                elem.text or '' if (elem := dep.find(tag, ns)) is not None else '',
-            )
-
-        return Package(
-            groupId=prop_replace_tag('groupId'),
-            artifactId=prop_replace_tag('artifactId'),
-            version=prop_replace_tag('version'),
-        )
-
-    @property
-    def dependency_management(self) -> list['Package']:
-        dependencies: list[Package] = []
-
-        for dep in self.raw_root.find('dependencyManagement/dependencies', ns) or []:
-            package = self._package_from_xml_dep(dep)
-            dependencies.append(package)
-
-        return dependencies
-
-
-class Package:
-    _pom: PackagePOM | None = None
-    _verified: bool = False
-
-    def __init__(self, groupId: str, artifactId: str, version: str | None = None, implicit: bool = False):
-        self.groupId = groupId
-        self.artifactId = artifactId
-        self.version = version if version and not version.isspace() else None
-        self.implicit = implicit
-
-    def __str__(self) -> str:
-        return f'{self.groupId}:{self.artifactId}:{self.version or "----"}'
-
-    def __eq__(self, other) -> bool:
-        return (
-            self.groupId == other.groupId
-            and self.artifactId == other.artifactId
-            and self.version == other.version
-        )
-
-    def __hash__(self) -> int:
-        return hash((self.groupId, self.artifactId, self.version))
-
-    @property
-    def dir_path(self):
-        group_path = self.groupId.replace(".", "/")
-        return f'{group_path}/{self.artifactId}/{self.version}'
-
-    @property
-    def base_filename(self):
-        return f'{self.artifactId}-{self.version}'
-
-    async def download_file(self, extension):
-        filepath = f'{self.dir_path}/{self.base_filename}.{extension}'
-
-        async with aiohttp.ClientSession() as session:
-            for mirror in mirrors:
-                pom_url = f'{mirror}/{filepath}'
-                logger.debug(f'{self}: Downloading {extension} from {pom_url}')
-
-                async with session.get(pom_url) as response:
-                    if response.status == 200:
-                        logger.debug(f'{self}: {extension} downloaded')
-                        return await response.text()
-                        break
-                    elif response.status == 429:
-                        raise TooManyRequestsException()
-                    else:
-                        logger.debug(f'{self}: HTTP error {response.status} from mirror {mirror}')
-            else:
-                logger.warning(f'{self}: File download of {extension} failed for all mirrors')
-                return None
-
-    @property
-    async def pom(self) -> PackagePOM:
-        if self._pom is not None:
-            return self._pom
-
-        if self.version is None:
-            await self._query_maven()
-
-        self._pom = PackagePOM(self, await self.download_file('pom'))
-
-        return self._pom
-
-    @property
-    def _urlquery(self) -> str:
-        q = f'g:{self.groupId}+AND+a:{self.artifactId}'
-
-        if self.version is not None:
-            q += f'+AND+v:{self.version}'
-
-        return q
-
-    async def _query_maven(self) -> None:
-        self._verified = False
-
-        async with aiohttp.ClientSession() as session:
-            for mirror in mirrors:
-                url = f'{mirror}/{self.groupId.replace(".", "/")}/{self.artifactId}/maven-metadata.xml'
-                logger.debug(f'{self}: Querying maven at url {url}')
-
-                async with session.get(url) as response:
-                    if response.status == 200:
-                        response_text = await response.text()
-                        metadata = ET.fromstring(response_text)
-
-                        if metadata is not None:
-                            logger.debug(f'{self}: Metadata found')
-
-                            if self.version is None:
-                                release_tag = metadata.find('./versioning/release')
-                                latest_tag = metadata.find('./versioning/latest')
-                                version = release_tag.text if release_tag is not None else latest_tag.text if latest_tag is not None else None
-
-                                if version is not None:
-                                    logger.debug(f'{self}: Using newest version {version}')
-                                    self.version = version
-                                    self._verified = True
-                                    return
-                                else:
-                                    logger.info(f'{self}: Could not find latest version in metadata from mirror {mirror}')
-                            else:
-                                if metadata.find(f'./versioning/versions/version[.="{self.version}"]') is not None:
-                                    logger.debug(f'{self}: Version {self.version} is valid')
-                                    self._verified = True
-                                    return
-                                else:
-                                    logger.info(f'{self}: Could not find version {self.version} in metadata from mirror {mirror}')
-                        else:
-                            logger.warning('{self}: Invalid XML for maven metadata: {response_text}')
-                    elif response.status == 429:
-                        raise TooManyRequestsException()
-                    else:
-                        logger.info(f'{self}: HTTP error {response.status} downloading maven metadata from {url}')
-            else:
-                if self.implicit:
-                    logger.info(f'{self}: Package not found in any mirror')
-                else:
-                    logger.warning(f'{self}: Package not found in any mirror')
-
-    async def verify(self) -> bool:
-        if not self._verified:
-            await self._query_maven()
-        return self._verified
-
-
-def load_package_list(list_path: Path, queue: asyncio.Queue) -> None:
-    logger.info(f'Parsing {list_path}')
-
-    with list_path.open('r') as f:
-        for line in f.readlines():
-            sections = line.strip().split(':')
-
-            if len(sections) < 2 or len(sections) > 3:
-                logger.warning(f'Invalid package format "{line}". It should be "groupID:artifactID" or "groupID:artifactID:version"')
-                continue
-
-            package = Package(
-                sections[0],
-                sections[1],
-                sections[2] if len(sections) == 3 else None,
-            )
-
-            queue.put_nowait(package)
-
-            continue
-            if not package.artifactId.endswith('-jvm'):
-                queue.put_nowait(
-                    Package(
-                        package.groupId,
-                        f'{package.artifactId}-jvm',
-                        package.version,
-                        True,
-                    )
-                )
-
-def create_gradle_build(packages, repo) -> str:
-    return """// Generated, do not edit
-plugins {
-    kotlin("jvm") version "1.7.20"
-}
-
-repositories {
-    maven {
-        url=uri("http://""" + repo + """/releases")
-        isAllowInsecureProtocol=true
-    }
-}
-
-val deps = listOf<String>(
-    """ + ',\n    '.join(f'"{dep}"' for dep in sorted(packages)) + """
-).flatMap {
-    listOf(it, it + ":sources", it + ":javadoc")
-}.map {
-    configurations.create(it.replace(':', '_')) {
-        isCanBeResolved = true
-        isCanBeConsumed = false
-    } to it
-}
-
-dependencies {
-    deps.forEach { (conf, dep) ->
-        conf(dep)
-    }
-}
-
-tasks.register("downloadDependencies") {
-    val logger = getLogger()
-
-    doLast {
-        deps.forEach { (conf, dep) ->
-            try {
-                conf.files
-            } catch (e: Exception) {
-                if (dep.endsWith(":sources")) {
-                    logger.warn("Package '$dep' has no sources")
-                } else if (dep.endsWith(":javadoc")) {
-                    logger.warn("Package '$dep' has no javadoc")
-                } else {
-                    logger.warn("Error while fetching '$dep': $e")
-                }
-            }
-        }
-    }
-}
-"""
-
-
-def create_gradle_settings(repo: str) -> str:
-    return """// Generated, do not edit
-rootProject.name = "gradle sync job"
-
-pluginManagement {
-    repositories {
-        maven {
-            url=uri("http://""" + repo + """/releases")
-            isAllowInsecureProtocol=true
-        }
-    }
-}
-"""
-
-async def download(package: Package, queue: asyncio.Queue) -> None:
-    async with done_lock:
-        is_done = str(package) in done
-
-    async with in_progress_lock:
-        is_in_progress = str(package) in in_progress
-
-    if is_done:
-        logger.info(f'{package}: Already downloaded. Skipping.')
-    elif is_in_progress:
-        logger.info(f'{package}: Already in progress. Skipping.')
-    else:
-        async with in_progress_lock:
-            in_progress.add(str(package))
-
-        for _ in range(50):
-            try:
-                verified = await package.verify()
-                break
-            except TooManyRequestsException:
-                logger.info(f'{package}: Too many requests. Delaying next attempt')
-                await asyncio.sleep(3*random.random() + 0.2)
-        else:
-            logger.error(f'{package}: Verification failed after 50 tries')
-            exit(1)
-
-        if verified:
-            for _ in range(50):
-                try:
-                    pom = await package.pom
-                    break
-                except TooManyRequestsException:
-                    logger.info(f'{package}: Too many requests. Delaying next attempt')
-                    await asyncio.sleep(3*random.random() + 0.2)
-                except WaitForPackage as e:
-                    logger.info(f'{package}: Waiting for {e.package}')
-
-                    async with in_progress_lock:
-                        if str(package) in in_progress:
-                            in_progress.remove(str(package))
-
-                        if str(e.package) not in in_progress:
-                            await queue.put(e.package)
-
-                        await queue.put(package)
-
-                    return
-            else:
-                logger.error(f'{package}: POM parsing failed after 50 tries')
-                exit(1)
-
-            if not pom:
-                logger.warn(f'{package}: No pom')
-                return
-
-            async with gradle_packages_lock:
-                gradle_packages.update(pom.gradle_packages)
-
-            if not pom.is_bom:
-                for dep in pom.dependency_management:
-                    logger.info(f'{package}: Handling transitive dependency {dep}')
-                    await queue.put(dep)
-
-        async with done_lock:
-            logger.debug(f'{package}: Marking done')
-            p = copy.copy(package)
-            p.version = None
-            done.add(str(package))
-            done.add(str(p))
-
-        async with in_progress_lock:
-            if str(package) in in_progress:
-                in_progress.remove(str(package))
-            else:
-                p = copy.copy(package)
-                p.version = None
-                if str(p) in in_progress:
-                    in_progress.remove(str(p))
-                else:
-                    logger.warning(f'{package}: Package is done, but not marked as in progress')
-
-async def worker(queue: asyncio.Queue) -> None:
-    while True:
-        package = await queue.get()
-
-        while True:
-            try:
-                await download(package, queue)
-                break
-            except PackageError:
-                logger.exception(f'{package}: Error while processing package')
-                break
-            except Exception:
-                logger.exception(f'{package}: Unknown error while processing package')
-                break
-
-
-        queue.task_done()
-
-
-async def main(package_list: Path, output_dir: Path, num_workers: int, gradle_repo: str) -> None:
-    queue: asyncio.Queue = asyncio.Queue()
-    tasks = []
-
-    load_package_list(package_list, queue)
-
-    logger.debug(f'Starting {num_workers} workers')
-    for i in range(num_workers):
-        tasks.append(
-            asyncio.create_task(
-                worker(queue)
-            )
-        )
-
-    await queue.join()
-
-    logger.debug('Queue is empty. Cancelling workers')
-    for task in tasks:
-        task.cancel()
-
-    await asyncio.gather(*tasks, return_exceptions=True)
-
-    async with gradle_packages_lock:
-        logger.info('Generating build.gradle.kts')
-        (output_dir / 'build.gradle.kts').write_text(create_gradle_build(gradle_packages, gradle_repo))
-
-    logger.info('Generating settings.gradle.kts')
-    (output_dir / 'settings.gradle.kts').write_text(create_gradle_settings(gradle_repo))
-
-logger = logging.getLogger(__name__)
-
-if __name__ == '__main__':
-    parser = argparse.ArgumentParser()
-    parser.add_argument('-w', '--workers', type=int, default=20)
-    parser.add_argument('-v', '--verbose', dest='verbosity', action='count', default=0)
-    parser.add_argument('--repo', type=str, help="The repository gradle should use", required=True)
-    parser.add_argument('--output_dir', type=Path, help="The directory to put the generated gradle files in", default=Path('.'), required=False)
-    parser.add_argument('package_list', type=Path, help="The list of packages to download")
-    args = parser.parse_args()
-
-    if args.verbosity == 0:
-        log_level = 'WARNING'
-    elif args.verbosity == 1:
-        log_level = 'INFO'
-    else:
-        log_level = 'DEBUG'
-
-    logging.basicConfig(level=log_level)
-
-    asyncio.run(
-        main(args.package_list, args.output_dir, args.workers, args.repo)
-    )

sync/requirements.txt

@@ -1 +1,2 @@
-aiohttp==3.8.4
+aiohttp~=3.8
+pyyaml~=6.0

sync/resolve-deps.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 
 is_reposilite_up ()
 {
@@ -24,25 +24,34 @@ wait_for_reposilite ()
 
 # Main scipt
 
-if [[ ! -f /package-list.txt ]]; then
-    echo "No /package-list.txt file. Aborting"
+if [[ ! -f /package-list.yaml ]]; then
+    echo "No /package-list.yaml file. Aborting"
     exit 255
 fi
 
+export PROJECTS_DIR=/gradle-projects
+mkdir -p "$PROJECTS_DIR"
+
 echo "Resolving packages and generating gradle config"
-if ! python3 ./generate-gradle.py --repo="repo:80" /package-list.txt; then
+mkdir -p /gradle
+python3 --version
+if ! python3 src/main.py --repo="repo:80" --output-dir "$PROJECTS_DIR" /package-list.yaml; then
     echo "Gradle generation failed"
     exit 255
 fi
 
-cat build.gradle.kts
-
 echo "Check that reposilite is running"
 wait_for_reposilite
 
 if [ $? -lt 30 ]; then
-    echo "Running gradle"
-    gradle -Dorg.gradle.jvmargs=-Xms4096m downloadDependencies
+  for project in "$PROJECTS_DIR"/*; do
+    echo "-----------------------Config for $project-------------------------------"
+    echo "Running $project"
+    cat "$project/build.gradle.kts"
+
+    echo "-----------------------Running gradle------------------------------------"
+    (cd "$project" && gradle -Dorg.gradle.jvmargs=-Xms4096m downloadDependencies --info)
+  done
 else
     echo "Can't connect to repository"
     exit 255

sync/src/config.py

@@ -0,0 +1,119 @@
+import yaml
+import logging
+from dataclasses import dataclass
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class KotlinVersion:
+    version: str
+    name: str
+    plugins: dict[str, str]
+    packages: list[str]
+
+
+@dataclass
+class Config:
+    kotlin_versions: list[KotlinVersion]
+    mirrors: list[str]
+
+
+def handle_packages(section) -> list[str]:
+    ignore = ["_kotlin-version", '_versions']
+    result = []
+
+    for entry in section:
+        if entry not in ignore:
+            if ':' in entry:
+                versions = section[entry]
+                if isinstance(versions, str):
+                    result.append(f'{entry}:{versions}')
+                elif isinstance(versions, list):
+                    for version in versions:
+                        result.append(f'{entry}:{version}')
+                else:
+                    logger.warning(f'Invalid version "{versions}" for "{entry}". Should be a string or list.')
+            elif isinstance(section[entry], dict):
+                group_section = section[entry]
+                default_versions = group_section['_versions']
+
+                for artifact in group_section:
+                    if artifact not in ignore:
+                        versions = group_section[artifact]
+
+                        if not versions and default_versions:
+                            versions = default_versions
+
+                        if isinstance(versions, str):
+                            result.append(f'{entry}:{artifact}:{versions}')
+                        elif isinstance(versions, list) and versions:
+                            for version in versions:
+                                result.append(f'{entry}:{artifact}:{version}')
+                        else:
+                            logger.warning(f'Invalid versions "{versions}" for "{entry}:{artifact}"')
+            else:
+                logger.warning(f'Invalid package spec "{entry}". Should be a full spec or a group ID')
+
+    return result
+
+
+def handle_plugins(section) -> dict[str, list[str]]:
+    result = {}
+
+    for entry in section:
+        versions = section[entry]
+
+        if isinstance(versions, str):
+            result[entry] = [versions]
+        elif isinstance(versions, list):
+            result[entry] = versions
+        else:
+            logger.warning(f'Invalid plugin version "{versions}" for {entry}')
+
+    return result
+
+
+def parse_config(path: Path) -> Config:
+    with path.open('r') as f:
+        data = yaml.safe_load(f)
+
+        parsed = {}
+
+        for version, section in data['kotlin'].items():
+            plugins = handle_plugins(section.get('plugins', {}))
+            packages = handle_packages(section.get('packages', {}))
+            parsed[version] = {
+                'plugins': plugins,
+                'packages': packages,
+            }
+
+        # Merge the general plugins and packages with that of the default version
+        if general := data.get('general'):
+            if default_version := general.get('_kotlin-version'):
+                parsed.setdefault(default_version, {})
+
+                parsed[default_version]['plugins'].update(
+                    handle_plugins(general.get('plugins', {}))
+                )
+
+                parsed[default_version]['packages'].extend(
+                    handle_packages(general.get('packages', {}))
+                )
+            else:
+                logger.warning(f'"general" section exists, but "_kotlin-version" is not defined. Skipping.')
+
+        kotlin_versions = []
+
+        for version, section in parsed.items():
+            kotlin_versions.append(KotlinVersion(version, f'version-packages', {}, section['packages']))
+
+            for plugin, plugin_versions in section['plugins'].items():
+                for plugin_version in plugin_versions:
+                    kotlin_versions.append(KotlinVersion(version, f'version-plugins-{plugin}', {plugin: plugin_version}, []))
+
+        return Config(
+            kotlin_versions,
+            data.get('maven', {}).get('mirrors', []),
+        )

sync/src/gradle.py

@@ -0,0 +1,68 @@
+def create_gradle_settings(repo: str) -> str:
+    return """// Generated, do not edit
+rootProject.name = "gradle sync job"
+
+pluginManagement {
+    repositories {
+        maven {
+            url=uri("http://""" + repo + """/releases")
+            isAllowInsecureProtocol=true
+        }
+    }
+}
+"""
+
+
+def create_gradle_build(kotlin_version: str, plugins: dict[str, str], packages: list[str], repo: str) -> str:
+    return """// Generated, do not edit
+plugins {
+    """ + f'kotlin("jvm") version "{kotlin_version}"' + """
+    
+    """ + '\n    '.join(f'id("{name}") version "{version}"' for name, version in plugins.items()) + """
+}
+
+repositories {
+    maven {
+        url=uri("http://""" + repo + """/releases")
+        isAllowInsecureProtocol=true
+    }
+}
+
+val deps = listOf<String>(
+    """ + ',\n    '.join(f'"{dep}"' for dep in sorted(packages)) + """
+).flatMap {
+    listOf(it, it + ":sources", it + ":javadoc")
+}.map {
+    configurations.create(it.replace(':', '_')) {
+        isCanBeResolved = true
+        isCanBeConsumed = false
+    } to it
+}
+
+dependencies {
+    deps.forEach { (conf, dep) ->
+        conf(dep)
+    }
+}
+
+tasks.register("downloadDependencies") {
+    val logger = getLogger()
+
+    doLast {
+        deps.forEach { (conf, dep) ->
+            logger.warn("$conf")
+            try {
+                conf.files
+            } catch (e: Exception) {
+                if (dep.endsWith(":sources")) {
+                    logger.info("Package '$dep' has no sources")
+                } else if (dep.endsWith(":javadoc")) {
+                    logger.info("Package '$dep' has no javadoc")
+                } else {
+                    logger.info("Error while fetching '$dep': $e")
+                }
+            }
+        }
+    }
+}
+"""

sync/src/main.py

@@ -0,0 +1,69 @@
+import argparse
+import asyncio
+import logging
+from pathlib import Path
+
+from maven import get_effective_packages
+from config import parse_config, KotlinVersion
+from gradle import create_gradle_build, create_gradle_settings
+
+logger = logging.getLogger(__name__)
+
+
+async def resolve_kotlin(ident: str, kotlin: KotlinVersion, output_dir: Path, mirrors: list[str], gradle_repo: str):
+    resolved_packages = [
+        resolved
+        for package in kotlin.packages
+        for resolved in await get_effective_packages(package, mirrors)
+    ]
+
+    path = output_dir / f"{ident}-kotlin-{kotlin.version}"
+    path.mkdir()
+    logger.debug(f'Creating {path}')
+
+    logger.debug('Generating build.gradle.kts')
+    gradle_build = create_gradle_build(kotlin.version, kotlin.plugins, resolved_packages, gradle_repo)
+    (path / 'build.gradle.kts').write_text(gradle_build)
+
+    logger.debug('Generating settings.gradle.kts')
+    gradle_settings = create_gradle_settings(gradle_repo)
+    (path / 'settings.gradle.kts').write_text(gradle_settings)
+
+
+async def main(package_list: Path, output_dir: Path, gradle_repo: str) -> None:
+    config = parse_config(package_list)
+
+    if not config.kotlin_versions:
+        print('No packages defined, nothing to do.')
+    elif not config.mirrors:
+        print('No mirrors defined. Add maven.mirrors in the config file.')
+    else:
+        for i, kotlin in enumerate(config.kotlin_versions):
+            logger.info(f'Resolving kotlin {kotlin.version}')
+            try:
+                await resolve_kotlin(str(i), kotlin, output_dir, config.mirrors, gradle_repo)
+            except:
+                logger.exception(f'Error resolving kotlin version {kotlin.version}')
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-v', '--verbose', dest='verbosity', action='count', default=0)
+    parser.add_argument('--repo', type=str, help="The repository gradle should use", required=True)
+    parser.add_argument('--output-dir', type=Path, help="The directory to put the generated gradle files in",
+                        default=Path('..'), required=False)
+    parser.add_argument('package_list', type=Path, help="The list of packages to download")
+    args = parser.parse_args()
+
+    if args.verbosity == 0:
+        log_level = 'WARNING'
+    elif args.verbosity == 1:
+        log_level = 'INFO'
+    else:
+        log_level = 'DEBUG'
+
+    logging.basicConfig(level=log_level)
+
+    asyncio.run(
+        main(args.package_list, args.output_dir, args.repo)
+    )

sync/src/maven.py

@@ -0,0 +1,111 @@
+import asyncio
+import logging
+from typing import Optional, Iterable
+from aiohttp import ClientSession
+
+from pom import PropertyMissing, PackagePOM, Properties
+
+logger = logging.getLogger(__name__)
+
+
+class TooManyRequestsException(Exception):
+    pass
+
+
+async def get_effective_packages(package: str, mirrors: Iterable[str]) -> list[str]:
+    """
+    Get a list of packages that is required for Gradle to fetch this package.
+
+    For most packages, this just returns the package name. However, for BOMs, the full list of packages included
+    in the BOM is returned (including the BOM itself). This is because Gradle doesn't fetch all packages of a BOM.
+
+    This requires querying Maven and parsing the POM to check if the package is a BOM.
+    """
+    packages = []
+
+    if pom := await fetch_pom(package, mirrors):
+        packages.append(package)
+
+        if pom.is_bom:
+            try:
+                packages.extend(pom.dependency_management)
+            except PropertyMissing as e1:
+                parent_props = await get_parent_props(pom.parent, mirrors)
+                if parent_props:
+                    pom.set_properties(parent_props)
+                    try:
+                        packages.extend(pom.dependency_management)
+                    except PropertyMissing as e2:
+                        logger.warning(
+                            f'{package}: Could not resolve property {e2.prop}, which is necessary for resolving dependencies'
+                        )
+                else:
+                    logger.warning(
+                        f'{package}: Could not resolve property {e1.prop}, and could not get properties from parent'
+                    )
+
+    return packages
+
+
+async def get_parent_props(parent: Optional[str], mirrors: Iterable[str]) -> Properties:
+    if parent:
+        if pom := await fetch_pom(parent, mirrors):
+            pom.set_properties(await get_parent_props(pom.parent, mirrors))
+            return pom.properties
+        else:
+            logger.warning(f'{parent}: Error fetching pom')
+
+    return {}
+
+
+async def fetch_pom(package: str, mirrors: Iterable[str]) -> Optional[PackagePOM]:
+    pom = await fetch_maven_file(package, 'pom', mirrors)
+    return PackagePOM(package, pom) if pom else None
+
+
+async def fetch_from_mirror(session: ClientSession, url: str) -> str | int:
+    async with session.get(url) as response:
+        if response.status == 200:
+            return await response.text()
+        elif response.status == 429:
+            raise TooManyRequestsException()
+        else:
+            return response.status
+
+
+async def fetch_maven_file(package: str, extension: str, mirrors: Iterable[str]) -> Optional[str]:
+    group_id, artifact_id, version = package.split(':')
+    url = f'{group_id.replace(".", "/")}/{artifact_id}/{version}/{artifact_id}-{version}.{extension}'
+    logger.debug(f'{package}: Downloading {extension} from {url}')
+
+    async with ClientSession() as session:
+        # Retry up to 5 times
+        for _ in range(5):
+            # Go through the mirrors, trying to find the package.
+            # If the package doesn't exist in a mirror, that mirror is removed from the list.
+            # If another error occurs the mirror is kept.
+            # After trying all mirrors, if the package isn't found, the remaining mirrors are retried.
+
+            retry_mirrors = []
+            for mirror in mirrors:
+                try:
+                    result = await fetch_from_mirror(session, f'{mirror}/{url}')
+
+                    if isinstance(result, str):
+                        logger.debug(f'{package}: {extension} downloaded')
+                        return result
+                    else:
+                        logger.debug(f'{package}: HTTP error {result} from mirror {mirror}')
+                except TooManyRequestsException:
+                    logger.info(f'{package}: Received Too Many Requests error. Trying other mirror.')
+                    retry_mirrors.append(mirror)
+
+            if retry_mirrors:
+                logger.info(f'{package}: Backing off, then trying again')
+                mirrors = retry_mirrors
+                await asyncio.sleep(0.1)
+            else:
+                break
+
+        logger.warning(f'{package}: File download of {extension} failed for all mirrors')
+        return None

sync/src/pom.py

@@ -0,0 +1,135 @@
+import logging
+import re
+from typing import Optional, TypeAlias
+from xml.etree import ElementTree
+
+from xmlutils import pom_namespace as ns, find_tag_text
+
+logger = logging.getLogger(__name__)
+
+Properties: TypeAlias = dict[str, Optional[str]]
+
+
+class PropertyMissing(Exception):
+    def __init__(self, prop: str):
+        self.prop = prop
+
+
+class PackageError(Exception):
+    pass
+
+
+class PackagePOM:
+    name: str
+    is_bom: bool
+    parent: Optional[str]
+    group_id: Optional[str]
+    artifact_id: Optional[str]
+    version: Optional[str]
+    properties: Properties
+
+    _raw_root: ElementTree.Element
+
+    def __init__(self, name: str, pom: str):
+        logger.debug(f'{name}: Parsing POM')
+        self.name = name
+
+        self._raw_root = ElementTree.fromstring(pom)
+
+        self.group_id, self.artifact_id, self.version = name.split(':')
+
+        if self.group_id is None or self.artifact_id is None or self.version is None:
+            logger.warning(
+                f'{name}: One of groupId={self.group_id}, artifactId={self.artifact_id}, version={self.version} was '
+                f'None. This can cause issues with dependency resolution'
+            )
+
+        self.parent = None
+
+        if (parent_tag := self._raw_root.find('parent', ns)) is not None:
+            parent_group = find_tag_text(parent_tag, 'groupId')
+            parent_artifact = find_tag_text(parent_tag, 'artifactId')
+            parent_version = find_tag_text(parent_tag, 'version')
+
+            if parent_group is None or parent_artifact is None or parent_version is None:
+                raise PackageError(f'Invalid parent {parent_group}:{parent_artifact}:{parent_version}')
+            else:
+                self.parent = f'{parent_group}:{parent_artifact}:{parent_version}'
+
+        if (packaging := self._raw_root.find('packaging', ns)) is not None:
+            self.is_bom = packaging == 'pom'
+        else:
+            self.is_bom = False
+
+        self.set_properties({})
+
+    @property
+    def dependency_management(self) -> list[str]:
+        return [
+            self._package_from_xml_dep(dep)
+            for dep in self._raw_root.find('dependencyManagement/dependencies', ns) or []
+        ]
+
+    def set_properties(self, parent_properties: Optional[Properties]):
+        logger.debug(f'{self.name}: Parsing properties')
+        props: Properties = parent_properties or {}
+
+        for prop_tag in self._raw_root.findall('.//properties/*', ns):
+            prop = prop_tag.tag.replace(f'{{{ns[""]}}}', '')
+            value = prop_tag.text if prop_tag.text is not None else ''
+            logger.debug(f'{self.name}: Setting prop {prop}={value}')
+            props[prop] = value
+
+        changed = True
+        while changed:
+            changed = False
+
+            for prop, value in props.items():
+                new_value = self._prop_replace(value, props, True)
+
+                if new_value != value:
+                    changed = True
+                    logger.debug(f'{self.name}: Setting prop {prop}={new_value}')
+                    props[prop] = new_value
+
+        self.properties = props
+
+    def _prop_replace(self, text, props: Optional[Properties] = None, quiet: bool = False) -> str:
+        def lookup_prop(match) -> str:
+            prop = match.group(1)
+
+            if prop == 'project.groupId':
+                value = str(self.group_id)
+            elif prop == 'project.artifactId':
+                value = str(self.artifact_id)
+            elif prop == 'project.version':
+                value = str(self.version)
+            elif prop.startswith('project.build') or prop.startswith('env.') or prop.startswith('maven.'):
+                value = None
+            elif prop in ['project.basedir', 'basedir', 'user.home', 'debug.port']:
+                value = None
+            else:
+                try:
+                    value = props[prop] if props is not None else self.properties[prop]
+                except KeyError:
+                    value = None
+
+            if value is None and not quiet:
+                raise PropertyMissing(prop)
+            else:
+                logger.debug(f'{self.name}: Replacing property {prop} with {value}')
+                return value
+
+        return re.sub(
+            r'\$\{([^}]*)}',
+            lookup_prop,
+            text,
+        )
+
+    def _package_from_xml_dep(self, dep: ElementTree.Element) -> str:
+        def prop_replace_tag(tag) -> str:
+            return self._prop_replace(
+                elem.text or '' if (elem := dep.find(tag, ns)) is not None else '',
+            )
+
+        return f"{prop_replace_tag('groupId')}:{prop_replace_tag('artifactId')}:{prop_replace_tag('version')}"

sync/src/xmlutils.py

@@ -0,0 +1,6 @@
+pom_namespace = {'': 'http://maven.apache.org/POM/4.0.0'}
+
+
+def find_tag_text(parent, tag) -> str | None:
+    elem = parent.find(tag, pom_namespace)
+    return elem.text if elem is not None else None