From 42e15a2d7ebebeafa2d1be6c2ff53c145c16cdbb Mon Sep 17 00:00:00 2001 From: d9pouces Date: Sat, 5 Sep 2026 23:16:49 +0200 Subject: [PATCH] feat: enhance build process and Dockerfiles for improved artifact handling and dependency resolution Signed-off-by: d9pouces --- tests/certs/test-ca.crt.pem | 22 ++++----- tests/configs/Java.yaml.j2 | 15 ++++++ tests/dockerfiles/Java.Dockerfile | 56 +++++++++++++++++++++-- tests/dockerfiles/Python.Dockerfile | 15 ++++++ tests/dockerfiles/repos/build_test_jar.py | 23 ++++++++-- tox.ini | 2 +- 6 files changed, 112 insertions(+), 21 deletions(-) diff --git a/tests/certs/test-ca.crt.pem b/tests/certs/test-ca.crt.pem index 6b5818d..1f1a761 100644 --- a/tests/certs/test-ca.crt.pem +++ b/tests/certs/test-ca.crt.pem @@ -1,21 +1,21 @@ -----BEGIN CERTIFICATE----- -MIIDZTCCAk2gAwIBAgIUY5npknVV83oHxLlZhV66VyobjvQwDQYJKoZIhvcNAQEL +MIIDdTCCAl2gAwIBAgIUKq4URHQwyg7IzJ3ViAGGlM9pGCwwDQYJKoZIhvcNAQEL BQAwQjEeMBwGA1UECgwVaWQtY29uZmlndXJhdG9yLXRlc3RzMSAwHgYDVQQDDBdp -ZC1jb25maWd1cmF0b3IgVGVzdCBDQTAeFw0yNjA5MDMyMTM5NDNaFw00NjA4Mjky -MTM5NDNaMEIxHjAcBgNVBAoMFWlkLWNvbmZpZ3VyYXRvci10ZXN0czEgMB4GA1UE +ZC1jb25maWd1cmF0b3IgVGVzdCBDQTAeFw0yNjA5MDUyMTA4MTNaFw00NjA4MzEy +MTA4MTNaMEIxHjAcBgNVBAoMFWlkLWNvbmZpZ3VyYXRvci10ZXN0czEgMB4GA1UE AwwXaWQtY29uZmlndXJhdG9yIFRlc3QgQ0EwggEiMA0GCSqGSIb3DQEBAQUAA4IB DwAwggEKAoIBAQD8bMteh3MDnbW6OoFMlhA4Z2QAabVoSFlSE/Si7ut+L/3v1GY4 lixenwLQc8a2koIH9CtmOodzfO7hpro/VUjYJ9J2eTsXLLiOg7OgggG1cKV9/kt7 RF0Q4hkcanGmiffc4xbz4ZedU6lP1vzLCsgU0A8OPt6svcNjGcWbjxss0KjgxGZ7 H4UKdL7Mv4W2Y6Fw1b4eV2M6jErkQK6IPXsHGFh7JkzliNcb0sX1VfbfauS84tBh f3EHwzLNhbjtEmQhQdqgsg4Cbk/gWur//6aEvlD/X12PAUbrp4DFD+CDuVagLX7I -7TQBXNFGKhozzQuDg7DFg1gIV8SFx2m+s60/AgMBAAGjUzBRMB0GA1UdDgQWBBSb +7TQBXNFGKhozzQuDg7DFg1gIV8SFx2m+s60/AgMBAAGjYzBhMB0GA1UdDgQWBBSb zJ7DsaxwCeqbfOXGJ3x+u7bjozAfBgNVHSMEGDAWgBSbzJ7DsaxwCeqbfOXGJ3x+ -u7bjozAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQA7RooUNpqn -XgpHKri5IcQw3kLnqz6yJ00rk2oIwSyji7uW2UHgURZac9kw4w7kXvcoe91cxI64 -0bOxdNmrNzbP6giqcyWt7pdt+XfIAzBuzbJj0ohXZHcX1d9HcvcpfiPabYWfPJmo -qUROsI5jfZ5uG6Q/r15hQpL8PANLmTbxMmg82BCJVUVX/WHnZwWwuCApbRBDb8W5 -jjzzNHlcLb8B3Iy5Ns2cd05xxkS9ATavvwU94oMhHx1dvxJMszMV3iMo28qbdbKi -o0h4KvQc8LrftuYpBuRPMYhLh4vw2A80RYbvuWp7sm7jDt1wKnTn0zioJbFozcJZ -otf1rfaEcRH0 +u7bjozAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBBjANBgkqhkiG9w0B +AQsFAAOCAQEAEV1+dG3agh0S/AR8jC8M33HXFyvneCCRtfb5q/loJ//VoWPbD3Gs +G+2SR0MVN+KKOMBvCBh3KrNSgFF9iFmBjNCJJZ8ycDIszUK3xLjUP2EzN1pNNl8S +D/yHzUCro3547Q+I+Nn5zRciG95EpMvtkIWem4H8luYSBt7w8y9uDUKWzaca/v3z +oGZPZP99w8WVqMKGRbsK94wsTVsWiE+oHzSufwrhvZJfQQ4jU6102IEnRS/eQFXA ++fozgEWuFatZe3Gz0HUJakrzSBwSTV6wmnEHCfu+uwdns1Ttq5FylZ4StJHsmweL +14w1xa3tjC5bQ3e1QWK3R5InHcSZ7tLpGA== -----END CERTIFICATE----- diff --git a/tests/configs/Java.yaml.j2 b/tests/configs/Java.yaml.j2 index d0e278d..b87cc4f 100644 --- a/tests/configs/Java.yaml.j2 +++ b/tests/configs/Java.yaml.j2 @@ -18,5 +18,20 @@ configurations: repositories: [] java: repositories: + # First entry: named "central" so Maven's mirrorOf=* (built on + # repositories[0]) redirects everything here without invalidating the + # maven-dependency-plugin already cached under that repository id + # during the Dockerfile's warm-up step. All other tools just see it as + # an ordinary repository, and find the test artifact here. - name: central url: https://host.docker.internal:{{ 'JAVA_REPO_PORT' | env }}/ + # Second entry: only SBT needs this. With + # -Dsbt.override.build.repos=true (the only way SBT actually consults + # this file for project dependencies, not just its own bootstrap), the + # repositories listed here become the *only* ones available - without a + # real Maven Central mirror among them, SBT can't resolve scala-library + # itself. Maven ignores it (mirrorOf=* already redirects everything to + # the first entry); Gradle/Ivy/Leiningen just see one more unused + # resolver, since the test artifact is already found via "central". + - name: maven-central-fallback + url: https://repo1.maven.org/maven2/ diff --git a/tests/dockerfiles/Java.Dockerfile b/tests/dockerfiles/Java.Dockerfile index 17333be..e301f90 100644 --- a/tests/dockerfiles/Java.Dockerfile +++ b/tests/dockerfiles/Java.Dockerfile @@ -4,11 +4,57 @@ COPY --from=target / /src/ COPY --from=generated-config Java.yaml /config.yaml RUN id-configurator apply --config /config.yaml --src /src --dst /dst FROM target AS runtime -# Warm up the local repository with the maven-dependency-plugin itself (from -# the real, network-reachable Maven Central) while the default mirror is -# still in place, so the actual test below never needs anything but the -# fixture repository configured by id-configurator. +# Every build tool the `java` module configures: Maven itself (already part +# of this image), Gradle, Ivy and Leiningen (all packaged for this Ubuntu +# base), and SBT (no package available, fetched as a plain release tarball). +RUN apt-get update \ + && apt-get install -y --no-install-recommends gradle ivy leiningen curl \ + && curl -sL https://github.com/sbt/sbt/releases/download/v1.10.1/sbt-1.10.1.tgz -o /tmp/sbt.tgz \ + && tar -xzf /tmp/sbt.tgz -C /opt \ + && rm /tmp/sbt.tgz +ENV PATH="/opt/sbt/bin:${PATH}" + +# Warm up every tool's own bootstrap/plugin dependencies from the real, +# network-reachable Maven Central while the default resolvers are still in +# place, so the actual tests below never need anything but the fixture +# repository configured by id-configurator (plus, for SBT only, the +# maven-central-fallback repository also configured below). RUN mvn -q org.apache.maven.plugins:maven-dependency-plugin:3.6.1:get -Dartifact=org.apache.commons:commons-lang3:3.12.0 +RUN mkdir -p /opt/sbt-warmup/project \ + && echo 'scalaVersion := "2.13.14"' > /opt/sbt-warmup/build.sbt \ + && echo 'sbt.version=1.10.1' > /opt/sbt-warmup/project/build.properties \ + && cd /opt/sbt-warmup && sbt update + COPY --from=configuration /dst/ / + RUN mvn org.apache.maven.plugins:maven-dependency-plugin:3.6.1:get -Dartifact=org.interdiode:idconfiguratortestpkg:0.0.1 \ - && find /root/.m2/repository/org/interdiode -type f + && test -f /root/.m2/repository/org/interdiode/idconfiguratortestpkg/0.0.1/idconfiguratortestpkg-0.0.1.jar + +RUN mkdir -p /tmp/gradletest && cd /tmp/gradletest \ + && cat > build.gradle <<'EOF' +configurations { deps } +dependencies { deps 'org.interdiode:idconfiguratortestpkg:0.0.1' } +task resolveDeps { doLast { configurations.deps.each { println it } } } +EOF +RUN cd /tmp/gradletest && gradle -q resolveDeps | grep -q idconfiguratortestpkg + +RUN java -jar /usr/share/java/ivy.jar -settings /root/.ivy2/ivysettings.xml \ + -dependency org.interdiode idconfiguratortestpkg 0.0.1 \ + -retrieve "/tmp/ivyretrieve/[artifact]-[revision].[ext]" \ + && test -f /tmp/ivyretrieve/idconfiguratortestpkg-0.0.1.jar + +RUN mkdir -p /tmp/leintest && cd /tmp/leintest \ + && cat > project.clj <<'EOF' +(defproject leintest "0.1.0" + :dependencies [[org.interdiode/idconfiguratortestpkg "0.0.1"]]) +EOF +RUN cd /tmp/leintest && lein deps :tree | grep -q idconfiguratortestpkg + +RUN mkdir -p /tmp/sbttest/project \ + && cat > /tmp/sbttest/build.sbt <<'EOF' +scalaVersion := "2.13.14" +libraryDependencies += "org.interdiode" % "idconfiguratortestpkg" % "0.0.1" +EOF +RUN echo 'sbt.version=1.10.1' > /tmp/sbttest/project/build.properties +RUN cd /tmp/sbttest \ + && sbt -Dsbt.override.build.repos=true "show Compile/dependencyClasspath" | grep -q idconfiguratortestpkg diff --git a/tests/dockerfiles/Python.Dockerfile b/tests/dockerfiles/Python.Dockerfile index a377572..b464ebe 100644 --- a/tests/dockerfiles/Python.Dockerfile +++ b/tests/dockerfiles/Python.Dockerfile @@ -4,6 +4,21 @@ COPY --from=target / /src/ COPY --from=generated-config Python.yaml /config.yaml RUN id-configurator apply --config /config.yaml --src /src --dst /dst FROM target AS runtime +ARG PYPI_REPO_PORT +# Warm up uv and poetry themselves from the real, network-reachable PyPI +# while the default index is still in place, so the actual tests below +# never need anything but the fixture repository configured by +# id-configurator. (easy_install/distutils, the fourth tool the `python` +# module configures, is not exercised here: both were removed from Python +# itself in 3.12, and setuptools no longer ships easy_install either, so +# there is no way to actually invoke it on a current interpreter.) +RUN pip install --no-cache-dir uv poetry COPY --from=configuration /dst/ / RUN pip install --no-cache-dir idconfiguratortestpkg \ && python3 -c "import idconfiguratortestpkg" +RUN uv pip install --system idconfiguratortestpkg \ + && python3 -c "import idconfiguratortestpkg" +RUN mkdir -p /tmp/poetrytest && cd /tmp/poetrytest \ + && poetry init -n --name poetrytest \ + && poetry source add --priority explicit demo "https://host.docker.internal:${PYPI_REPO_PORT}/simple/" \ + && poetry add idconfiguratortestpkg --source demo diff --git a/tests/dockerfiles/repos/build_test_jar.py b/tests/dockerfiles/repos/build_test_jar.py index 0a6e0ea..dbca5e6 100644 --- a/tests/dockerfiles/repos/build_test_jar.py +++ b/tests/dockerfiles/repos/build_test_jar.py @@ -9,6 +9,8 @@ can run in a minimal Python image with nothing installed. from __future__ import annotations +import hashlib +import io import pathlib import zipfile @@ -32,19 +34,32 @@ POM = f""" """ -def build_jar(path: pathlib.Path) -> None: - with zipfile.ZipFile(path, "w", zipfile.ZIP_DEFLATED) as archive: +def build_jar() -> bytes: + buffer = io.BytesIO() + with zipfile.ZipFile(buffer, "w", zipfile.ZIP_DEFLATED) as archive: archive.writestr("META-INF/MANIFEST.MF", MANIFEST) archive.writestr( "marker.txt", "test-only artifact served by the id-configurator integration tests\n", ) + return buffer.getvalue() + + +def _write_with_checksums(path: pathlib.Path, content: bytes) -> None: + """Write `content` to `path`, plus the .sha1/.md5 sidecar files most + build tools (Leiningen's aether resolver in particular) require to be + present, unlike Maven which merely warns when they are missing.""" + path.write_bytes(content) + sha1_digest = hashlib.sha1(content).hexdigest() # nosec B324 + (path.parent / f"{path.name}.sha1").write_text(sha1_digest, encoding="ascii") + md5_digest = hashlib.md5(content).hexdigest() # nosec B324 + (path.parent / f"{path.name}.md5").write_text(md5_digest, encoding="ascii") def main() -> None: OUT_DIR.mkdir(parents=True, exist_ok=True) - build_jar(OUT_DIR / f"{ARTIFACT}-{VERSION}.jar") - (OUT_DIR / f"{ARTIFACT}-{VERSION}.pom").write_text(POM, encoding="utf-8") + _write_with_checksums(OUT_DIR / f"{ARTIFACT}-{VERSION}.jar", build_jar()) + _write_with_checksums(OUT_DIR / f"{ARTIFACT}-{VERSION}.pom", POM.encode("utf-8")) if __name__ == "__main__": diff --git a/tox.ini b/tox.ini index a253628..3c24543 100644 --- a/tox.ini +++ b/tox.ini @@ -78,7 +78,7 @@ allowlist_externals = docker commands = python3 -m id_tools test render-config {tox_root}/tests/configs/Python.yaml.j2 {env_dir}/configs/Python.yaml - docker buildx build --platform linux/amd64 --add-host=host.docker.internal:host-gateway --build-context generated-config={env_dir}/configs -f {tox_root}/tests/dockerfiles/Python.Dockerfile -t id-configurator-python . + docker buildx build --platform linux/amd64 --add-host=host.docker.internal:host-gateway --build-arg PYPI_REPO_PORT={env:PYPI_REPO_PORT} --build-context generated-config={env_dir}/configs -f {tox_root}/tests/dockerfiles/Python.Dockerfile -t id-configurator-python . [docker:apt-repo] dockerfile = {tox_root}/tests/dockerfiles/repos/apt-repo.Dockerfile