diff --git a/README.md b/README.md index c03a660..f31152a 100644 --- a/README.md +++ b/README.md @@ -11,13 +11,48 @@ Run the following command: This starts the reposilite server, then starts maven and makes download all packages defined in the pom's in the `poms/` folder, including their dependencies. These packages are then cached by reposilite in `data/`. The relevant folders are turned into tarballs and saved in the root directory. ## Adding packages -### Automatically -New packages can be added to the `package-list.txt` file. The group ID and artifact ID are required, but the version is optional. If the version number is omitted the latest version is found. Run `./update-poms.sh` to generate the pom files from the package file. -The root `pom.xml` file is updated automatically. +New packages are added to `package-list.yaml`. -### Manually -Pom files can be added manually by creating a folder with a `pom.xml` file in the `poms/` folder. +Example config: +```yaml +maven: + # List of mirrors to query for package details. + mirrors: + - "https://repo.maven.apache.org/maven2" -After the `pom.xml` is created, run `./generate_master_pom.xml` to update the root `pom.xml` file, and commit the changes. +# Specify which kotlin versions to fetch. +kotlin: + 1.8.20: + # Plugins specific to this kotlin version. See `general` section for details. + plugins: + com.expediagroup.graphql: ["7.0.0-alpha.5", "6.5.2"] + # Packages specific to this kotlin version. See `general` section for details. + packages: + org.slf4j:slf4j-api: "2.1.0" + # A kotlin version without specific plugins and packages. + 1.8.0: {} + 1.7.20: + packages: + org.slf4j:slf4j-api: "2.0.7" -This structure is necessary to make dependabot find all dependencies. +# Packages and plugins that doesn't require a specific kotlin version. +general: + # The kotlin version to use (doesn't matter much, use the newest stable version). + _kotlin-version: "1.8.20" + + plugins: + # Plugins are listed with their plugin ID, not the full package name. + # Plugin versions can be a single version as a string, or a list of strings to fetch multiple versions. + org.panteleyev.jpackageplugin: "1.5.2" + packages: + # Packages are listed either with their full name as a key, or split into group and artifact. + # Package versions can be a single version as a string, or a list of strings to fetch multiple versions. + org.jetbrains.kotlinx:kotlinx-datetime: "0.4.0" + com.expediagroup: + # When splitting into group and artifact, the default versions for the group can be specified. + # Using an empty list as the version for an artifact implies that the default versions should be used. + _versions: ["7.0.0-alpha.6", "6.5.2"] + graphql-kotlin-ktor-server: ["7.0.0-alpha.6"] # Uses only the specified version + graphql-kotlin-client: [] # Uses the default versions + graphql-kotlin-client-generator: [] # Uses the default versions +``` diff --git a/docker-compose.yml b/docker-compose.yml index d33df49..708c04d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -19,4 +19,4 @@ services: depends_on: - repo volumes: - - ./package-list.txt:/package-list.txt + - ./package-list.yaml:/package-list.yaml diff --git a/package-list.txt b/package-list.txt deleted file mode 100644 index 3102275..0000000 --- a/package-list.txt +++ /dev/null @@ -1,70 +0,0 @@ -org.jetbrains.kotlin:kotlin-bom:1.7.20 -org.jetbrains.kotlinx:kotlinx-coroutines-core:1.6.4 -org.jetbrains.kotlinx:kotlinx-datetime:0.4.0 -org.jetbrains.kotlin:kotlin-serialization:1.7.20 -org.jetbrains.kotlin:kotlin-maven-serialization:1.6.21 -org.jetbrains.kotlin:kotlin-maven-serialization -org.jetbrains.kotlin:kotlin-maven-plugin:1.7.20 -org.jetbrains.kotlin:kotlin-maven-allopen:1.7.20 -org.jetbrains.kotlin:kotlin-maven-noarg:1.7.20 -org.jetbrains.kotlin:kotlin-gradle-plugin:1.7.20 -org.jetbrains.kotlin:kotlin-gradle-plugin-idea:1.7.20 -org.jetbrains.kotlin.jvm:org.jetbrains.kotlin.jvm.gradle.plugin:1.7.20 -org.jetbrains.kotlin.jvm:org.jetbrains.kotlin.jvm.gradle.plugin:1.7.20 -org.jetbrains.kotlin.plugin.serialization:org.jetbrains.kotlin.plugin.serialization.gradle.plugin:1.7.20 -org.jetbrains.exposed:exposed-bom:0.41.1 -org.slf4j:slf4j-api:2.0.6 -org.junit:junit-bom:5.9.1 -com.github.node-gradle.node:com.github.node-gradle.node.gradle.plugin:3.5.1 -io.kotest:kotest-bom:5.5.4 -io.kotest:kotest-runner-junit5:5.5.4 -org.asciidoctor:asciidoctor-maven-plugin -org.asciidoctor:asciidoctorj:2.5.7 -org.asciidoctor:asciidoctorj-api:2.5.7 -org.asciidoctor:asciidoctorj-test-support:2.5.7 -org.asciidoctor:asciidoctorj-revealjs:4.1.0 -org.asciidoctor:asciidoctorj-diagram-plantuml -org.asciidoctor:asciidoctorj-pdf:2.3.4 -org.asciidoctor:asciidoctorj-diagram:2.2.3 -com.vladmihalcea:hibernate-types-55 -org.apache.maven.plugins:maven-surefire-plugin -org.apache.maven.plugins:maven-failsafe-plugin -org.apache.maven.plugins:maven-shade-plugin -org.apache.maven.plugins:maven-compiler-plugin -io.insert-koin:koin-core:3.3.0 -io.insert-koin:koin-core:3.2.2 -io.insert-koin:koin-test:3.3.0 -io.insert-koin:koin-test:3.2.2 -io.insert-koin:koin-test-core:2.2.3 -io.insert-koin:koin-test-junit5:3.3.0 -io.insert-koin:koin-test-junit5:3.2.2 -io.insert-koin:koin-ktor:3.2.2 -org.kodein.di:kodein-di -org.kodein.di:kodein-di:7.16.0 -org.kodein.di:kodein-di-framework-ktor-server-jvm -org.kodein.di:kodein-di-framework-ktor-server-jvm:7.16.0 -io.ktor:ktor-bom -io.ktor:ktor-bom:2.2.1 -io.mockk:mockk -io.mockk:mockk:1.13.3 -io.quarkiverse.mockk:quarkus-junit5-mockk -org.glassfish.corba:glassfish-corba -org.glassfish.corba:glassfish-corba-tests -org.glassfish.corba:glassfish-corba-internal-api -org.glassfish.corba:glassfish-corba-omgapi -org.glassfish.corba:glassfish-corba-orb -org.glassfish.corba:glassfish-corba-asm -org.glassfish.corba:glassfish-corba-csiv2-idl -org.glassfish.corba:exception-annotation-processor -org.glassfish.corba:idlj -ch.qos.logback:logback-classic:1.2.11 -org.snmp4j:snmp4j:3.7.4 -io.github.microutils:kotlin-logging:3.0.4 -io.github.microutils:kotlin-logging -org.apache.sshd:sshd:2.9.2 -org.apache.sshd:sshd-core:2.9.2 -org.apache.sshd:sshd-common:2.9.2 -org.apache.sshd:sshd-sftp:2.9.2 -org.apache.sshd:sshd-scp:2.9.2 -org.apache.sshd:sshd-ldap:2.9.2 -org.hibernate.reactive:hibernate-reactive-core diff --git a/package-list.yaml b/package-list.yaml new file mode 100644 index 0000000..c808aec --- /dev/null +++ b/package-list.yaml @@ -0,0 +1,29 @@ +maven: + mirrors: + - "https://repo.maven.apache.org/maven2" + - "https://repo1.maven.org/maven2" + - "https://oss.sonatype.org/content/repositories/snapshots" + - "https://packages.confluent.io/maven" + - "https://registry.quarkus.io/maven" + - "https://plugins.gradle.org/m2" + +kotlin: + 1.8.20: + plugins: + com.expediagroup.graphql: ["7.0.0-alpha.5", "6.5.2"] + 1.8.0: {} + 1.7.20: + packages: + org.slf4j:slf4j-api: "2.0.7" + +general: + _kotlin-version: "1.8.20" + plugins: + org.panteleyev.jpackageplugin: "1.5.2" + packages: + org.jetbrains.kotlinx:kotlinx-datetime: "0.4.0" + com.expediagroup: + _versions: ["7.0.0-alpha.6", "6.5.2"] + graphql-kotlin-ktor-server: ["7.0.0-alpha.6"] + graphql-kotlin-client: [] + graphql-kotlin-client-generator: [] diff --git a/sync/.dockerignore b/sync/.dockerignore index dcc4d72..d9a7dec 100644 --- a/sync/.dockerignore +++ b/sync/.dockerignore @@ -1,4 +1,4 @@ ** !resolve-deps.sh -!generate-gradle.py +!src !requirements.txt diff --git a/sync/Dockerfile b/sync/Dockerfile index f4c4113..7c47abe 100644 --- a/sync/Dockerfile +++ b/sync/Dockerfile @@ -8,6 +8,6 @@ COPY requirements.txt . RUN python3 -m pip install -r requirements.txt COPY resolve-deps.sh . -COPY generate-gradle.py . +COPY src ./src CMD [ "/bin/sh", "./resolve-deps.sh" ] diff --git a/sync/generate-gradle.py b/sync/generate-gradle.py deleted file mode 100755 index 8951e41..0000000 --- a/sync/generate-gradle.py +++ /dev/null @@ -1,579 +0,0 @@ -#!/bin/python3 - -import re -import copy -import random -import argparse -import logging -import asyncio -import subprocess -import copy -import aiohttp -from pathlib import Path -from xml.etree import ElementTree as ET - - -ns = {'': 'http://maven.apache.org/POM/4.0.0'} -ET.register_namespace('', ns['']) - -baseurl = 'https://search.maven.org' -base_pom_path = Path('poms') -mirrors = [ - "https://repo.maven.apache.org/maven2", - "https://repo1.maven.org/maven2", - "https://oss.sonatype.org/content/repositories/snapshots", - "https://packages.confluent.io/maven", - "https://registry.quarkus.io/maven", - "https://plugins.gradle.org/m2", -] - -done: set[str] = set() -done_lock = asyncio.Lock() -in_progress: set[str] = set() -in_progress_lock = asyncio.Lock() -gradle_packages: set[str] = set() -gradle_packages_lock = asyncio.Lock() - -global_properties: dict[str, dict[str, str]] = {} - - -class TooManyRequestsException(Exception): - pass - - -class PackageError(Exception): - pass - -class WaitForPackage(Exception): - def __init__(self, package): - self.package = package - -def find_tag_text(parent, tag) -> str | None: - elem = parent.find(tag, ns) - return elem.text if elem is not None else None - -class PackagePOM: - def __init__(self, package: 'Package', pom: str): - self._package = package - - logger.debug(f'{package}: Parsing POM') - self.raw_root = ET.fromstring(pom) - - self.parent: Package | None = None - - if (parent_tag := self.raw_root.find('parent', ns)) is not None: - parent_group = find_tag_text(parent_tag, 'groupId') - parent_artifact = find_tag_text(parent_tag, 'artifactId') - parent_version = find_tag_text(parent_tag, 'version') - - logger.debug(f'{package}: Parsing parent {parent_group}:{parent_artifact}:{parent_version}') - - if parent_group is not None and parent_artifact is not None and parent_version is not None: - parent = Package( - parent_group, - parent_artifact, - parent_version, - ) - - if str(parent) in done: - self.parent = parent - else: - raise WaitForPackage(parent) - else: - raise PackageError(f'Invalid parent {parent_group}:{parent_artifact}:{parent_version}') - - logger.debug(f'{package}: Parsing properties') - parent_props: dict[str, str] = {} if self.parent is None else global_properties[str(self.parent)] - self.properties = self.resolve_props(parent_props) - global_properties[str(package)] = self.properties - - logger.debug(f'{package}: Parsing packaging') - if (packaging := self.raw_root.find('packaging', ns)) is not None: - self.packaging = packaging.text - else: - self.packaging = '??' - - self.is_bom = self.packaging == 'pom' - - self.gradle_packages = [str(package)] - - if self.is_bom: - logger.debug(f'{package}: Parsing dependencyManagement') - - if (dependencyManagement := self.raw_root.find('dependencyManagement', ns)): - if (dependencies := dependencyManagement.find('dependencies', ns)): - packages = [] - - for dep in dependencies.findall('dependency', ns): - groupId = find_tag_text(dep, 'groupId') - artifactId = find_tag_text(dep, 'artifactId') - version = find_tag_text(dep, 'version') - - - if groupId is not None and artifactId is not None and version is not None: - groupId = self.prop_replace(groupId) - artifactId = self.prop_replace(artifactId) - version = self.prop_replace(version) - packages.append(f'{groupId}:{artifactId}:{version}') - - logger.debug(f'{package}: Adding {len(packages)} package(s) from dependencyManagement') - - self.gradle_packages.extend(packages) - else: - logger.warn(f'{package}: dependencyManagement has no dependencies') - else: - logger.warn(f'{package}: BOM has no dependencyManagement') - - logger.debug(f'{package}: POM parsed') - - def resolve_props(self, initial: dict[str, str]): - props = initial - - for prop_tag in self.raw_root.findall('.//properties/*', ns): - prop = prop_tag.tag.replace(f'{{{ns[""]}}}', '') - value = prop_tag.text if prop_tag.text is not None else '' - logger.debug(f'{self._package}: Setting prop {prop}={value}') - props[prop] = value - - changed = True - while changed: - changed = False - - for prop, value in props.items(): - new_value = self.prop_replace(value, props) - - if new_value != value: - changed = True - logger.debug(f'{self._package}: Setting prop {prop}={new_value}') - props[prop] = new_value - - return props - - def prop_replace(self, text, props: dict[str, str] | None = None) -> str: - def lookup_prop(match) -> str: - prop = match.group(1) - - if prop == 'project.groupId': - value = str(self._package.groupId) - elif prop == 'project.artifactId': - value = str(self._package.artifactId) - elif prop == 'project.version': - value = str(self._package.version) - elif prop.startswith('project.build') or prop.startswith('env.') or prop.startswith('maven.'): - value = '' - elif prop in ['project.basedir', 'basedir', 'user.home', 'debug.port']: - value = '' - else: - try: - value = props[prop] if props is not None else self.properties[prop] - except KeyError: - logger.error(f'{self._package}: Could not find property {prop}. Setting it to ""') - value = '' - - logger.debug(f'{self._package}: Replacing property {prop} with {value}') - return value - - return re.sub( - r'\$\{([^\}]*)\}', - lookup_prop, - text, - ) - - def _package_from_xml_dep(self, dep: ET.Element) -> 'Package': - def prop_replace_tag(tag) -> str: - return self.prop_replace( - elem.text or '' if (elem := dep.find(tag, ns)) is not None else '', - ) - - return Package( - groupId=prop_replace_tag('groupId'), - artifactId=prop_replace_tag('artifactId'), - version=prop_replace_tag('version'), - ) - - @property - def dependency_management(self) -> list['Package']: - dependencies: list[Package] = [] - - for dep in self.raw_root.find('dependencyManagement/dependencies', ns) or []: - package = self._package_from_xml_dep(dep) - dependencies.append(package) - - return dependencies - - -class Package: - _pom: PackagePOM | None = None - _verified: bool = False - - def __init__(self, groupId: str, artifactId: str, version: str | None = None, implicit: bool = False): - self.groupId = groupId - self.artifactId = artifactId - self.version = version if version and not version.isspace() else None - self.implicit = implicit - - def __str__(self) -> str: - return f'{self.groupId}:{self.artifactId}:{self.version or "----"}' - - def __eq__(self, other) -> bool: - return ( - self.groupId == other.groupId - and self.artifactId == other.artifactId - and self.version == other.version - ) - - def __hash__(self) -> int: - return hash((self.groupId, self.artifactId, self.version)) - - @property - def dir_path(self): - group_path = self.groupId.replace(".", "/") - return f'{group_path}/{self.artifactId}/{self.version}' - - @property - def base_filename(self): - return f'{self.artifactId}-{self.version}' - - async def download_file(self, extension): - filepath = f'{self.dir_path}/{self.base_filename}.{extension}' - - async with aiohttp.ClientSession() as session: - for mirror in mirrors: - pom_url = f'{mirror}/{filepath}' - logger.debug(f'{self}: Downloading {extension} from {pom_url}') - - async with session.get(pom_url) as response: - if response.status == 200: - logger.debug(f'{self}: {extension} downloaded') - return await response.text() - break - elif response.status == 429: - raise TooManyRequestsException() - else: - logger.debug(f'{self}: HTTP error {response.status} from mirror {mirror}') - else: - logger.warning(f'{self}: File download of {extension} failed for all mirrors') - return None - - @property - async def pom(self) -> PackagePOM: - if self._pom is not None: - return self._pom - - if self.version is None: - await self._query_maven() - - self._pom = PackagePOM(self, await self.download_file('pom')) - - return self._pom - - @property - def _urlquery(self) -> str: - q = f'g:{self.groupId}+AND+a:{self.artifactId}' - - if self.version is not None: - q += f'+AND+v:{self.version}' - - return q - - async def _query_maven(self) -> None: - self._verified = False - - async with aiohttp.ClientSession() as session: - for mirror in mirrors: - url = f'{mirror}/{self.groupId.replace(".", "/")}/{self.artifactId}/maven-metadata.xml' - logger.debug(f'{self}: Querying maven at url {url}') - - async with session.get(url) as response: - if response.status == 200: - response_text = await response.text() - metadata = ET.fromstring(response_text) - - if metadata is not None: - logger.debug(f'{self}: Metadata found') - - if self.version is None: - release_tag = metadata.find('./versioning/release') - latest_tag = metadata.find('./versioning/latest') - version = release_tag.text if release_tag is not None else latest_tag.text if latest_tag is not None else None - - if version is not None: - logger.debug(f'{self}: Using newest version {version}') - self.version = version - self._verified = True - return - else: - logger.info(f'{self}: Could not find latest version in metadata from mirror {mirror}') - else: - if metadata.find(f'./versioning/versions/version[.="{self.version}"]') is not None: - logger.debug(f'{self}: Version {self.version} is valid') - self._verified = True - return - else: - logger.info(f'{self}: Could not find version {self.version} in metadata from mirror {mirror}') - else: - logger.warning('{self}: Invalid XML for maven metadata: {response_text}') - elif response.status == 429: - raise TooManyRequestsException() - else: - logger.info(f'{self}: HTTP error {response.status} downloading maven metadata from {url}') - else: - if self.implicit: - logger.info(f'{self}: Package not found in any mirror') - else: - logger.warning(f'{self}: Package not found in any mirror') - - async def verify(self) -> bool: - if not self._verified: - await self._query_maven() - return self._verified - - -def load_package_list(list_path: Path, queue: asyncio.Queue) -> None: - logger.info(f'Parsing {list_path}') - - with list_path.open('r') as f: - for line in f.readlines(): - sections = line.strip().split(':') - - if len(sections) < 2 or len(sections) > 3: - logger.warning(f'Invalid package format "{line}". It should be "groupID:artifactID" or "groupID:artifactID:version"') - continue - - package = Package( - sections[0], - sections[1], - sections[2] if len(sections) == 3 else None, - ) - - queue.put_nowait(package) - - continue - if not package.artifactId.endswith('-jvm'): - queue.put_nowait( - Package( - package.groupId, - f'{package.artifactId}-jvm', - package.version, - True, - ) - ) - -def create_gradle_build(packages, repo) -> str: - return """// Generated, do not edit -plugins { - kotlin("jvm") version "1.7.20" -} - -repositories { - maven { - url=uri("http://""" + repo + """/releases") - isAllowInsecureProtocol=true - } -} - -val deps = listOf( - """ + ',\n '.join(f'"{dep}"' for dep in sorted(packages)) + """ -).flatMap { - listOf(it, it + ":sources", it + ":javadoc") -}.map { - configurations.create(it.replace(':', '_')) { - isCanBeResolved = true - isCanBeConsumed = false - } to it -} - -dependencies { - deps.forEach { (conf, dep) -> - conf(dep) - } -} - -tasks.register("downloadDependencies") { - val logger = getLogger() - - doLast { - deps.forEach { (conf, dep) -> - try { - conf.files - } catch (e: Exception) { - if (dep.endsWith(":sources")) { - logger.warn("Package '$dep' has no sources") - } else if (dep.endsWith(":javadoc")) { - logger.warn("Package '$dep' has no javadoc") - } else { - logger.warn("Error while fetching '$dep': $e") - } - } - } - } -} -""" - - -def create_gradle_settings(repo: str) -> str: - return """// Generated, do not edit -rootProject.name = "gradle sync job" - -pluginManagement { - repositories { - maven { - url=uri("http://""" + repo + """/releases") - isAllowInsecureProtocol=true - } - } -} -""" - -async def download(package: Package, queue: asyncio.Queue) -> None: - async with done_lock: - is_done = str(package) in done - - async with in_progress_lock: - is_in_progress = str(package) in in_progress - - if is_done: - logger.info(f'{package}: Already downloaded. Skipping.') - elif is_in_progress: - logger.info(f'{package}: Already in progress. Skipping.') - else: - async with in_progress_lock: - in_progress.add(str(package)) - - for _ in range(50): - try: - verified = await package.verify() - break - except TooManyRequestsException: - logger.info(f'{package}: Too many requests. Delaying next attempt') - await asyncio.sleep(3*random.random() + 0.2) - else: - logger.error(f'{package}: Verification failed after 50 tries') - exit(1) - - if verified: - for _ in range(50): - try: - pom = await package.pom - break - except TooManyRequestsException: - logger.info(f'{package}: Too many requests. Delaying next attempt') - await asyncio.sleep(3*random.random() + 0.2) - except WaitForPackage as e: - logger.info(f'{package}: Waiting for {e.package}') - - async with in_progress_lock: - if str(package) in in_progress: - in_progress.remove(str(package)) - - if str(e.package) not in in_progress: - await queue.put(e.package) - - await queue.put(package) - - return - else: - logger.error(f'{package}: POM parsing failed after 50 tries') - exit(1) - - if not pom: - logger.warn(f'{package}: No pom') - return - - async with gradle_packages_lock: - gradle_packages.update(pom.gradle_packages) - - if not pom.is_bom: - for dep in pom.dependency_management: - logger.info(f'{package}: Handling transitive dependency {dep}') - await queue.put(dep) - - async with done_lock: - logger.debug(f'{package}: Marking done') - p = copy.copy(package) - p.version = None - done.add(str(package)) - done.add(str(p)) - - async with in_progress_lock: - if str(package) in in_progress: - in_progress.remove(str(package)) - else: - p = copy.copy(package) - p.version = None - if str(p) in in_progress: - in_progress.remove(str(p)) - else: - logger.warning(f'{package}: Package is done, but not marked as in progress') - -async def worker(queue: asyncio.Queue) -> None: - while True: - package = await queue.get() - - while True: - try: - await download(package, queue) - break - except PackageError: - logger.exception(f'{package}: Error while processing package') - break - except Exception: - logger.exception(f'{package}: Unknown error while processing package') - break - - - queue.task_done() - - -async def main(package_list: Path, output_dir: Path, num_workers: int, gradle_repo: str) -> None: - queue: asyncio.Queue = asyncio.Queue() - tasks = [] - - load_package_list(package_list, queue) - - logger.debug(f'Starting {num_workers} workers') - for i in range(num_workers): - tasks.append( - asyncio.create_task( - worker(queue) - ) - ) - - await queue.join() - - logger.debug('Queue is empty. Cancelling workers') - for task in tasks: - task.cancel() - - await asyncio.gather(*tasks, return_exceptions=True) - - async with gradle_packages_lock: - logger.info('Generating build.gradle.kts') - (output_dir / 'build.gradle.kts').write_text(create_gradle_build(gradle_packages, gradle_repo)) - - logger.info('Generating settings.gradle.kts') - (output_dir / 'settings.gradle.kts').write_text(create_gradle_settings(gradle_repo)) - -logger = logging.getLogger(__name__) - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.add_argument('-w', '--workers', type=int, default=20) - parser.add_argument('-v', '--verbose', dest='verbosity', action='count', default=0) - parser.add_argument('--repo', type=str, help="The repository gradle should use", required=True) - parser.add_argument('--output_dir', type=Path, help="The directory to put the generated gradle files in", default=Path('.'), required=False) - parser.add_argument('package_list', type=Path, help="The list of packages to download") - args = parser.parse_args() - - if args.verbosity == 0: - log_level = 'WARNING' - elif args.verbosity == 1: - log_level = 'INFO' - else: - log_level = 'DEBUG' - - logging.basicConfig(level=log_level) - - asyncio.run( - main(args.package_list, args.output_dir, args.workers, args.repo) - ) diff --git a/sync/requirements.txt b/sync/requirements.txt index 9fe29a9..1554440 100644 --- a/sync/requirements.txt +++ b/sync/requirements.txt @@ -1 +1,2 @@ -aiohttp==3.8.4 +aiohttp~=3.8 +pyyaml~=6.0 diff --git a/sync/resolve-deps.sh b/sync/resolve-deps.sh index ca072cc..796e6f7 100755 --- a/sync/resolve-deps.sh +++ b/sync/resolve-deps.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash is_reposilite_up () { @@ -24,25 +24,34 @@ wait_for_reposilite () # Main scipt -if [[ ! -f /package-list.txt ]]; then - echo "No /package-list.txt file. Aborting" +if [[ ! -f /package-list.yaml ]]; then + echo "No /package-list.yaml file. Aborting" exit 255 fi +export PROJECTS_DIR=/gradle-projects +mkdir -p "$PROJECTS_DIR" + echo "Resolving packages and generating gradle config" -if ! python3 ./generate-gradle.py --repo="repo:80" /package-list.txt; then +mkdir -p /gradle +python3 --version +if ! python3 src/main.py --repo="repo:80" --output-dir "$PROJECTS_DIR" /package-list.yaml; then echo "Gradle generation failed" exit 255 fi -cat build.gradle.kts - echo "Check that reposilite is running" wait_for_reposilite if [ $? -lt 30 ]; then - echo "Running gradle" - gradle -Dorg.gradle.jvmargs=-Xms4096m downloadDependencies + for project in "$PROJECTS_DIR"/*; do + echo "-----------------------Config for $project-------------------------------" + echo "Running $project" + cat "$project/build.gradle.kts" + + echo "-----------------------Running gradle------------------------------------" + (cd "$project" && gradle -Dorg.gradle.jvmargs=-Xms4096m downloadDependencies --info) + done else echo "Can't connect to repository" exit 255 diff --git a/sync/src/config.py b/sync/src/config.py new file mode 100644 index 0000000..9f47ce8 --- /dev/null +++ b/sync/src/config.py @@ -0,0 +1,119 @@ +import yaml +import logging +from dataclasses import dataclass +from pathlib import Path + +logger = logging.getLogger(__name__) + + +@dataclass +class KotlinVersion: + version: str + name: str + plugins: dict[str, str] + packages: list[str] + + +@dataclass +class Config: + kotlin_versions: list[KotlinVersion] + mirrors: list[str] + + +def handle_packages(section) -> list[str]: + ignore = ["_kotlin-version", '_versions'] + result = [] + + for entry in section: + if entry not in ignore: + if ':' in entry: + versions = section[entry] + if isinstance(versions, str): + result.append(f'{entry}:{versions}') + elif isinstance(versions, list): + for version in versions: + result.append(f'{entry}:{version}') + else: + logger.warning(f'Invalid version "{versions}" for "{entry}". Should be a string or list.') + elif isinstance(section[entry], dict): + group_section = section[entry] + default_versions = group_section['_versions'] + + for artifact in group_section: + if artifact not in ignore: + versions = group_section[artifact] + + if not versions and default_versions: + versions = default_versions + + if isinstance(versions, str): + result.append(f'{entry}:{artifact}:{versions}') + elif isinstance(versions, list) and versions: + for version in versions: + result.append(f'{entry}:{artifact}:{version}') + else: + logger.warning(f'Invalid versions "{versions}" for "{entry}:{artifact}"') + else: + logger.warning(f'Invalid package spec "{entry}". Should be a full spec or a group ID') + + return result + + +def handle_plugins(section) -> dict[str, list[str]]: + result = {} + + for entry in section: + versions = section[entry] + + if isinstance(versions, str): + result[entry] = [versions] + elif isinstance(versions, list): + result[entry] = versions + else: + logger.warning(f'Invalid plugin version "{versions}" for {entry}') + + return result + + +def parse_config(path: Path) -> Config: + with path.open('r') as f: + data = yaml.safe_load(f) + + parsed = {} + + for version, section in data['kotlin'].items(): + plugins = handle_plugins(section.get('plugins', {})) + packages = handle_packages(section.get('packages', {})) + parsed[version] = { + 'plugins': plugins, + 'packages': packages, + } + + # Merge the general plugins and packages with that of the default version + if general := data.get('general'): + if default_version := general.get('_kotlin-version'): + parsed.setdefault(default_version, {}) + + parsed[default_version]['plugins'].update( + handle_plugins(general.get('plugins', {})) + ) + + parsed[default_version]['packages'].extend( + handle_packages(general.get('packages', {})) + ) + else: + logger.warning(f'"general" section exists, but "_kotlin-version" is not defined. Skipping.') + + kotlin_versions = [] + + for version, section in parsed.items(): + kotlin_versions.append(KotlinVersion(version, f'version-packages', {}, section['packages'])) + + for plugin, plugin_versions in section['plugins'].items(): + for plugin_version in plugin_versions: + kotlin_versions.append(KotlinVersion(version, f'version-plugins-{plugin}', {plugin: plugin_version}, [])) + + return Config( + kotlin_versions, + data.get('maven', {}).get('mirrors', []), + ) diff --git a/sync/src/gradle.py b/sync/src/gradle.py new file mode 100644 index 0000000..fb59bfb --- /dev/null +++ b/sync/src/gradle.py @@ -0,0 +1,68 @@ +def create_gradle_settings(repo: str) -> str: + return """// Generated, do not edit +rootProject.name = "gradle sync job" + +pluginManagement { + repositories { + maven { + url=uri("http://""" + repo + """/releases") + isAllowInsecureProtocol=true + } + } +} +""" + + +def create_gradle_build(kotlin_version: str, plugins: dict[str, str], packages: list[str], repo: str) -> str: + return """// Generated, do not edit +plugins { + """ + f'kotlin("jvm") version "{kotlin_version}"' + """ + + """ + '\n '.join(f'id("{name}") version "{version}"' for name, version in plugins.items()) + """ +} + +repositories { + maven { + url=uri("http://""" + repo + """/releases") + isAllowInsecureProtocol=true + } +} + +val deps = listOf( + """ + ',\n '.join(f'"{dep}"' for dep in sorted(packages)) + """ +).flatMap { + listOf(it, it + ":sources", it + ":javadoc") +}.map { + configurations.create(it.replace(':', '_')) { + isCanBeResolved = true + isCanBeConsumed = false + } to it +} + +dependencies { + deps.forEach { (conf, dep) -> + conf(dep) + } +} + +tasks.register("downloadDependencies") { + val logger = getLogger() + + doLast { + deps.forEach { (conf, dep) -> + logger.warn("$conf") + try { + conf.files + } catch (e: Exception) { + if (dep.endsWith(":sources")) { + logger.info("Package '$dep' has no sources") + } else if (dep.endsWith(":javadoc")) { + logger.info("Package '$dep' has no javadoc") + } else { + logger.info("Error while fetching '$dep': $e") + } + } + } + } +} +""" diff --git a/sync/src/main.py b/sync/src/main.py new file mode 100644 index 0000000..c1913dd --- /dev/null +++ b/sync/src/main.py @@ -0,0 +1,69 @@ +import argparse +import asyncio +import logging +from pathlib import Path + +from maven import get_effective_packages +from config import parse_config, KotlinVersion +from gradle import create_gradle_build, create_gradle_settings + +logger = logging.getLogger(__name__) + + +async def resolve_kotlin(ident: str, kotlin: KotlinVersion, output_dir: Path, mirrors: list[str], gradle_repo: str): + resolved_packages = [ + resolved + for package in kotlin.packages + for resolved in await get_effective_packages(package, mirrors) + ] + + path = output_dir / f"{ident}-kotlin-{kotlin.version}" + path.mkdir() + logger.debug(f'Creating {path}') + + logger.debug('Generating build.gradle.kts') + gradle_build = create_gradle_build(kotlin.version, kotlin.plugins, resolved_packages, gradle_repo) + (path / 'build.gradle.kts').write_text(gradle_build) + + logger.debug('Generating settings.gradle.kts') + gradle_settings = create_gradle_settings(gradle_repo) + (path / 'settings.gradle.kts').write_text(gradle_settings) + + +async def main(package_list: Path, output_dir: Path, gradle_repo: str) -> None: + config = parse_config(package_list) + + if not config.kotlin_versions: + print('No packages defined, nothing to do.') + elif not config.mirrors: + print('No mirrors defined. Add maven.mirrors in the config file.') + else: + for i, kotlin in enumerate(config.kotlin_versions): + logger.info(f'Resolving kotlin {kotlin.version}') + try: + await resolve_kotlin(str(i), kotlin, output_dir, config.mirrors, gradle_repo) + except: + logger.exception(f'Error resolving kotlin version {kotlin.version}') + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('-v', '--verbose', dest='verbosity', action='count', default=0) + parser.add_argument('--repo', type=str, help="The repository gradle should use", required=True) + parser.add_argument('--output-dir', type=Path, help="The directory to put the generated gradle files in", + default=Path('..'), required=False) + parser.add_argument('package_list', type=Path, help="The list of packages to download") + args = parser.parse_args() + + if args.verbosity == 0: + log_level = 'WARNING' + elif args.verbosity == 1: + log_level = 'INFO' + else: + log_level = 'DEBUG' + + logging.basicConfig(level=log_level) + + asyncio.run( + main(args.package_list, args.output_dir, args.repo) + ) diff --git a/sync/src/maven.py b/sync/src/maven.py new file mode 100644 index 0000000..e7752e6 --- /dev/null +++ b/sync/src/maven.py @@ -0,0 +1,111 @@ +import asyncio +import logging +from typing import Optional, Iterable +from aiohttp import ClientSession + +from pom import PropertyMissing, PackagePOM, Properties + +logger = logging.getLogger(__name__) + + +class TooManyRequestsException(Exception): + pass + + +async def get_effective_packages(package: str, mirrors: Iterable[str]) -> list[str]: + """ + Get a list of packages that is required for Gradle to fetch this package. + + For most packages, this just returns the package name. However, for BOMs, the full list of packages included + in the BOM is returned (including the BOM itself). This is because Gradle doesn't fetch all packages of a BOM. + + This requires querying Maven and parsing the POM to check if the package is a BOM. + """ + packages = [] + + if pom := await fetch_pom(package, mirrors): + packages.append(package) + + if pom.is_bom: + try: + packages.extend(pom.dependency_management) + except PropertyMissing as e1: + parent_props = await get_parent_props(pom.parent, mirrors) + if parent_props: + pom.set_properties(parent_props) + try: + packages.extend(pom.dependency_management) + except PropertyMissing as e2: + logger.warning( + f'{package}: Could not resolve property {e2.prop}, which is necessary for resolving dependencies' + ) + else: + logger.warning( + f'{package}: Could not resolve property {e1.prop}, and could not get properties from parent' + ) + + return packages + + +async def get_parent_props(parent: Optional[str], mirrors: Iterable[str]) -> Properties: + if parent: + if pom := await fetch_pom(parent, mirrors): + pom.set_properties(await get_parent_props(pom.parent, mirrors)) + return pom.properties + else: + logger.warning(f'{parent}: Error fetching pom') + + return {} + + +async def fetch_pom(package: str, mirrors: Iterable[str]) -> Optional[PackagePOM]: + pom = await fetch_maven_file(package, 'pom', mirrors) + return PackagePOM(package, pom) if pom else None + + +async def fetch_from_mirror(session: ClientSession, url: str) -> str | int: + async with session.get(url) as response: + if response.status == 200: + return await response.text() + elif response.status == 429: + raise TooManyRequestsException() + else: + return response.status + + +async def fetch_maven_file(package: str, extension: str, mirrors: Iterable[str]) -> Optional[str]: + group_id, artifact_id, version = package.split(':') + url = f'{group_id.replace(".", "/")}/{artifact_id}/{version}/{artifact_id}-{version}.{extension}' + logger.debug(f'{package}: Downloading {extension} from {url}') + + async with ClientSession() as session: + # Retry up to 5 times + for _ in range(5): + # Go through the mirrors, trying to find the package. + # If the package doesn't exist in a mirror, that mirror is removed from the list. + # If another error occurs the mirror is kept. + # After trying all mirrors, if the package isn't found, the remaining mirrors are retried. + + retry_mirrors = [] + for mirror in mirrors: + try: + result = await fetch_from_mirror(session, f'{mirror}/{url}') + + if isinstance(result, str): + logger.debug(f'{package}: {extension} downloaded') + return result + else: + logger.debug(f'{package}: HTTP error {result} from mirror {mirror}') + except TooManyRequestsException: + logger.info(f'{package}: Received Too Many Requests error. Trying other mirror.') + retry_mirrors.append(mirror) + + if retry_mirrors: + logger.info(f'{package}: Backing off, then trying again') + mirrors = retry_mirrors + await asyncio.sleep(0.1) + else: + break + + logger.warning(f'{package}: File download of {extension} failed for all mirrors') + return None diff --git a/sync/src/pom.py b/sync/src/pom.py new file mode 100644 index 0000000..b4ddca1 --- /dev/null +++ b/sync/src/pom.py @@ -0,0 +1,135 @@ +import logging +import re +from typing import Optional, TypeAlias +from xml.etree import ElementTree + +from xmlutils import pom_namespace as ns, find_tag_text + +logger = logging.getLogger(__name__) + +Properties: TypeAlias = dict[str, Optional[str]] + + +class PropertyMissing(Exception): + def __init__(self, prop: str): + self.prop = prop + + +class PackageError(Exception): + pass + + +class PackagePOM: + name: str + is_bom: bool + parent: Optional[str] + group_id: Optional[str] + artifact_id: Optional[str] + version: Optional[str] + properties: Properties + + _raw_root: ElementTree.Element + + def __init__(self, name: str, pom: str): + logger.debug(f'{name}: Parsing POM') + self.name = name + + self._raw_root = ElementTree.fromstring(pom) + + self.group_id, self.artifact_id, self.version = name.split(':') + + if self.group_id is None or self.artifact_id is None or self.version is None: + logger.warning( + f'{name}: One of groupId={self.group_id}, artifactId={self.artifact_id}, version={self.version} was ' + f'None. This can cause issues with dependency resolution' + ) + + self.parent = None + + if (parent_tag := self._raw_root.find('parent', ns)) is not None: + parent_group = find_tag_text(parent_tag, 'groupId') + parent_artifact = find_tag_text(parent_tag, 'artifactId') + parent_version = find_tag_text(parent_tag, 'version') + + if parent_group is None or parent_artifact is None or parent_version is None: + raise PackageError(f'Invalid parent {parent_group}:{parent_artifact}:{parent_version}') + else: + self.parent = f'{parent_group}:{parent_artifact}:{parent_version}' + + if (packaging := self._raw_root.find('packaging', ns)) is not None: + self.is_bom = packaging == 'pom' + else: + self.is_bom = False + + self.set_properties({}) + + @property + def dependency_management(self) -> list[str]: + return [ + self._package_from_xml_dep(dep) + for dep in self._raw_root.find('dependencyManagement/dependencies', ns) or [] + ] + + def set_properties(self, parent_properties: Optional[Properties]): + logger.debug(f'{self.name}: Parsing properties') + props: Properties = parent_properties or {} + + for prop_tag in self._raw_root.findall('.//properties/*', ns): + prop = prop_tag.tag.replace(f'{{{ns[""]}}}', '') + value = prop_tag.text if prop_tag.text is not None else '' + logger.debug(f'{self.name}: Setting prop {prop}={value}') + props[prop] = value + + changed = True + while changed: + changed = False + + for prop, value in props.items(): + new_value = self._prop_replace(value, props, True) + + if new_value != value: + changed = True + logger.debug(f'{self.name}: Setting prop {prop}={new_value}') + props[prop] = new_value + + self.properties = props + + def _prop_replace(self, text, props: Optional[Properties] = None, quiet: bool = False) -> str: + def lookup_prop(match) -> str: + prop = match.group(1) + + if prop == 'project.groupId': + value = str(self.group_id) + elif prop == 'project.artifactId': + value = str(self.artifact_id) + elif prop == 'project.version': + value = str(self.version) + elif prop.startswith('project.build') or prop.startswith('env.') or prop.startswith('maven.'): + value = None + elif prop in ['project.basedir', 'basedir', 'user.home', 'debug.port']: + value = None + else: + try: + value = props[prop] if props is not None else self.properties[prop] + except KeyError: + value = None + + if value is None and not quiet: + raise PropertyMissing(prop) + else: + logger.debug(f'{self.name}: Replacing property {prop} with {value}') + return value + + return re.sub( + r'\$\{([^}]*)}', + lookup_prop, + text, + ) + + def _package_from_xml_dep(self, dep: ElementTree.Element) -> str: + def prop_replace_tag(tag) -> str: + return self._prop_replace( + elem.text or '' if (elem := dep.find(tag, ns)) is not None else '', + ) + + return f"{prop_replace_tag('groupId')}:{prop_replace_tag('artifactId')}:{prop_replace_tag('version')}" diff --git a/sync/src/xmlutils.py b/sync/src/xmlutils.py new file mode 100644 index 0000000..1c2488c --- /dev/null +++ b/sync/src/xmlutils.py @@ -0,0 +1,6 @@ +pom_namespace = {'': 'http://maven.apache.org/POM/4.0.0'} + + +def find_tag_text(parent, tag) -> str | None: + elem = parent.find(tag, pom_namespace) + return elem.text if elem is not None else None