From da090e5b79948b6512e057a28db9148961567117 Mon Sep 17 00:00:00 2001 From: Sindre Stephansen Date: Fri, 17 Feb 2023 16:18:49 +0100 Subject: [PATCH] Change update-poms.py to generate gradle build files instead of poms --- update-poms.py | 128 ++++++++++++++++++++++--------------------------- 1 file changed, 56 insertions(+), 72 deletions(-) diff --git a/update-poms.py b/update-poms.py index cc70600..5b25cf3 100755 --- a/update-poms.py +++ b/update-poms.py @@ -31,6 +31,8 @@ done: set[str] = set() done_lock = asyncio.Lock() in_progress: set[str] = set() in_progress_lock = asyncio.Lock() +gradle_packages: set[str] = set() +gradle_packages_lock = asyncio.Lock() num_workers = 50 global_properties: dict[str, dict[str, str]] = {} @@ -94,78 +96,21 @@ class PackagePOM: self.is_bom = self.packaging == 'pom' + self.gradle_packages = [str(package)] + if self.packaging == 'pom': root_copy = copy.deepcopy(self.raw_root) dependencies = root_copy.find('dependencies', ns) or ET.SubElement(root_copy, 'dependencies') - depman = root_copy.find('dependencyManagement', ns) - if depman is not None: - tmp_deps = depman.findall('dependencies/*', ns) - dependencies.extend(tmp_deps) - root_copy.remove(depman) - - tmpGroupId = f'tmp.{package.groupId}' - tmpArtifactId = f'placeholder.{package.artifactId}' - tmpVersion = package.version - - if (groupId := root_copy.find('groupId', ns)) is not None: - groupId.text = tmpGroupId - else: - logger.info(f"{package}: Inserting new groupId tag in pom") - ET.SubElement(root_copy, 'groupId').text = tmpGroupId - - if (artifactId := root_copy.find('artifactId', ns)) is not None: - artifactId.text = tmpArtifactId - else: - logger.info(f"{package}: Inserting new artifactId tag in pom") - ET.SubElement(root_copy, 'artifactId').text = tmpArtifactId - - if (version := root_copy.find('version', ns)) is not None: - version.text = tmpVersion - else: - logger.info(f"{package}: Inserting new version tag in pom") - ET.SubElement(root_copy, 'version').text = tmpVersion - - # Add a dependency for the pom itself - self_dep = ET.SubElement(dependencies, 'dependency') - ET.SubElement(self_dep, 'groupId').text = package.groupId - ET.SubElement(self_dep, 'artifactId').text = package.artifactId - ET.SubElement(self_dep, 'version').text = package.version - - self.generated_root = root_copy - else: - self.generated_root = ET.fromstring( - f""" - - - 4.0.0 - tmp.{package.groupId} - placeholder-{package.artifactId} - {package.version} - Package {package.artifactId} - - - - {package.groupId} - {package.artifactId} - {package.version} - - - - """ + self.gradle_packages.extend( + [ + f'{dep.find("groupId").text}:{dep.find("artifactId").text}:{dep.find("version").text}' + for dep in dependencies.findall('dependency') + ] ) logger.debug(f'{package}: POM parsed') - def write(self, f): - tree = ET.ElementTree(self.generated_root) - ET.indent(tree) - tree.write(f) - - def resolve_props(self, initial: dict[str, str]): props = initial @@ -388,6 +333,7 @@ def load_package_list(list_path: Path, queue: asyncio.Queue) -> None: queue.put_nowait(package) + continue if not package.artifactId.endswith('-jvm'): queue.put_nowait( Package( @@ -398,6 +344,47 @@ def load_package_list(list_path: Path, queue: asyncio.Queue) -> None: ) ) +async def create_gradle_script() -> str: + async with gradle_packages_lock: + return """// Generated, do not edit +plugins { + kotlin("jvm") version "1.7.20" +} + +repositories { + maven { + url=uri("http://localhost:9001/releases") + isAllowInsecureProtocol=true + } +} + +val deps = listOf( + """ + ',\n '.join(f'"{dep}"' for dep in sorted(gradle_packages)) + """ +).map { + configurations.create(it.replace(':', '_')) to it +} + + +dependencies { + deps.forEach { (conf, dep) -> + conf(dep) + } +} + +tasks.register("downloadDependencies") { + doLast { + deps.forEach { (conf, dep) -> + conf.files.forEach { file -> + copy { + from(file) + into("data/") + } + } + } + } +} +""" + async def download(package: Package, queue: asyncio.Queue) -> None: async with done_lock: is_done = str(package) in done @@ -453,12 +440,8 @@ async def download(package: Package, queue: asyncio.Queue) -> None: logger.warn(f'{package}: No pom') return - pom_dir = base_pom_path / f'{package.groupId}-{package.artifactId}-{package.version}' - pom_path = pom_dir / 'pom.xml' - - pom_dir.mkdir(exist_ok=True) - pom.write(pom_path) - logger.info(f'{package}: Downloaded') + async with gradle_packages_lock: + gradle_packages.update(pom.gradle_packages) if not pom.is_bom: for dep in pom.dependency_management: @@ -524,9 +507,10 @@ async def main() -> None: await asyncio.gather(*tasks, return_exceptions=True) - logger.info('Generating master POM') - subprocess.call(['sh', 'generate_master_pom.sh']) + logger.info('Generating build.gradle.kts') + gradle_script = await create_gradle_script() + Path("test/build.gradle.kts").write_text(gradle_script) logger = logging.getLogger(__name__)