feat: enhance build process and Dockerfiles for improved artifact handling and dependency resolution

Signed-off-by: d9pouces <github@19pouces.net>

tests/certs/test-ca.crt.pem

@@ -1,21 +1,21 @@
 -----BEGIN CERTIFICATE-----
-MIIDZTCCAk2gAwIBAgIUY5npknVV83oHxLlZhV66VyobjvQwDQYJKoZIhvcNAQEL
+MIIDdTCCAl2gAwIBAgIUKq4URHQwyg7IzJ3ViAGGlM9pGCwwDQYJKoZIhvcNAQEL
 BQAwQjEeMBwGA1UECgwVaWQtY29uZmlndXJhdG9yLXRlc3RzMSAwHgYDVQQDDBdp
-ZC1jb25maWd1cmF0b3IgVGVzdCBDQTAeFw0yNjA5MDMyMTM5NDNaFw00NjA4Mjky
+ZC1jb25maWd1cmF0b3IgVGVzdCBDQTAeFw0yNjA5MDUyMTA4MTNaFw00NjA4MzEy
-MTM5NDNaMEIxHjAcBgNVBAoMFWlkLWNvbmZpZ3VyYXRvci10ZXN0czEgMB4GA1UE
+MTA4MTNaMEIxHjAcBgNVBAoMFWlkLWNvbmZpZ3VyYXRvci10ZXN0czEgMB4GA1UE
 AwwXaWQtY29uZmlndXJhdG9yIFRlc3QgQ0EwggEiMA0GCSqGSIb3DQEBAQUAA4IB
 DwAwggEKAoIBAQD8bMteh3MDnbW6OoFMlhA4Z2QAabVoSFlSE/Si7ut+L/3v1GY4
 lixenwLQc8a2koIH9CtmOodzfO7hpro/VUjYJ9J2eTsXLLiOg7OgggG1cKV9/kt7
 RF0Q4hkcanGmiffc4xbz4ZedU6lP1vzLCsgU0A8OPt6svcNjGcWbjxss0KjgxGZ7
 H4UKdL7Mv4W2Y6Fw1b4eV2M6jErkQK6IPXsHGFh7JkzliNcb0sX1VfbfauS84tBh
 f3EHwzLNhbjtEmQhQdqgsg4Cbk/gWur//6aEvlD/X12PAUbrp4DFD+CDuVagLX7I
-7TQBXNFGKhozzQuDg7DFg1gIV8SFx2m+s60/AgMBAAGjUzBRMB0GA1UdDgQWBBSb
+7TQBXNFGKhozzQuDg7DFg1gIV8SFx2m+s60/AgMBAAGjYzBhMB0GA1UdDgQWBBSb
 zJ7DsaxwCeqbfOXGJ3x+u7bjozAfBgNVHSMEGDAWgBSbzJ7DsaxwCeqbfOXGJ3x+
-u7bjozAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQA7RooUNpqn
+u7bjozAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBBjANBgkqhkiG9w0B
-XgpHKri5IcQw3kLnqz6yJ00rk2oIwSyji7uW2UHgURZac9kw4w7kXvcoe91cxI64
+AQsFAAOCAQEAEV1+dG3agh0S/AR8jC8M33HXFyvneCCRtfb5q/loJ//VoWPbD3Gs
-0bOxdNmrNzbP6giqcyWt7pdt+XfIAzBuzbJj0ohXZHcX1d9HcvcpfiPabYWfPJmo
+G+2SR0MVN+KKOMBvCBh3KrNSgFF9iFmBjNCJJZ8ycDIszUK3xLjUP2EzN1pNNl8S
-qUROsI5jfZ5uG6Q/r15hQpL8PANLmTbxMmg82BCJVUVX/WHnZwWwuCApbRBDb8W5
+D/yHzUCro3547Q+I+Nn5zRciG95EpMvtkIWem4H8luYSBt7w8y9uDUKWzaca/v3z
-jjzzNHlcLb8B3Iy5Ns2cd05xxkS9ATavvwU94oMhHx1dvxJMszMV3iMo28qbdbKi
+oGZPZP99w8WVqMKGRbsK94wsTVsWiE+oHzSufwrhvZJfQQ4jU6102IEnRS/eQFXA
-o0h4KvQc8LrftuYpBuRPMYhLh4vw2A80RYbvuWp7sm7jDt1wKnTn0zioJbFozcJZ
++fozgEWuFatZe3Gz0HUJakrzSBwSTV6wmnEHCfu+uwdns1Ttq5FylZ4StJHsmweL
-otf1rfaEcRH0
+14w1xa3tjC5bQ3e1QWK3R5InHcSZ7tLpGA==
 -----END CERTIFICATE-----

tests/configs/Java.yaml.j2

@@ -18,5 +18,20 @@ configurations:
     repositories: []
   java:
     repositories:
+    # First entry: named "central" so Maven's mirrorOf=* (built on
+    # repositories[0]) redirects everything here without invalidating the
+    # maven-dependency-plugin already cached under that repository id
+    # during the Dockerfile's warm-up step. All other tools just see it as
+    # an ordinary repository, and find the test artifact here.
     - name: central
       url: https://host.docker.internal:{{ 'JAVA_REPO_PORT' | env }}/
+    # Second entry: only SBT needs this. With
+    # -Dsbt.override.build.repos=true (the only way SBT actually consults
+    # this file for project dependencies, not just its own bootstrap), the
+    # repositories listed here become the *only* ones available - without a
+    # real Maven Central mirror among them, SBT can't resolve scala-library
+    # itself. Maven ignores it (mirrorOf=* already redirects everything to
+    # the first entry); Gradle/Ivy/Leiningen just see one more unused
+    # resolver, since the test artifact is already found via "central".
+    - name: maven-central-fallback
+      url: https://repo1.maven.org/maven2/

tests/dockerfiles/Java.Dockerfile

@@ -4,11 +4,57 @@ COPY --from=target / /src/
 COPY --from=generated-config Java.yaml /config.yaml
 RUN id-configurator apply --config /config.yaml --src /src --dst /dst
 FROM target AS runtime
-# Warm up the local repository with the maven-dependency-plugin itself (from
+# Every build tool the `java` module configures: Maven itself (already part
-# the real, network-reachable Maven Central) while the default mirror is
+# of this image), Gradle, Ivy and Leiningen (all packaged for this Ubuntu
-# still in place, so the actual test below never needs anything but the
+# base), and SBT (no package available, fetched as a plain release tarball).
-# fixture repository configured by id-configurator.
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends gradle ivy leiningen curl \
+    && curl -sL https://github.com/sbt/sbt/releases/download/v1.10.1/sbt-1.10.1.tgz -o /tmp/sbt.tgz \
+    && tar -xzf /tmp/sbt.tgz -C /opt \
+    && rm /tmp/sbt.tgz
+ENV PATH="/opt/sbt/bin:${PATH}"
+
+# Warm up every tool's own bootstrap/plugin dependencies from the real,
+# network-reachable Maven Central while the default resolvers are still in
+# place, so the actual tests below never need anything but the fixture
+# repository configured by id-configurator (plus, for SBT only, the
+# maven-central-fallback repository also configured below).
 RUN mvn -q org.apache.maven.plugins:maven-dependency-plugin:3.6.1:get -Dartifact=org.apache.commons:commons-lang3:3.12.0
+RUN mkdir -p /opt/sbt-warmup/project \
+    && echo 'scalaVersion := "2.13.14"' > /opt/sbt-warmup/build.sbt \
+    && echo 'sbt.version=1.10.1' > /opt/sbt-warmup/project/build.properties \
+    && cd /opt/sbt-warmup && sbt update
+
 COPY --from=configuration /dst/ /
+
 RUN mvn org.apache.maven.plugins:maven-dependency-plugin:3.6.1:get -Dartifact=org.interdiode:idconfiguratortestpkg:0.0.1 \
-    && find /root/.m2/repository/org/interdiode -type f
+    && test -f /root/.m2/repository/org/interdiode/idconfiguratortestpkg/0.0.1/idconfiguratortestpkg-0.0.1.jar
+
+RUN mkdir -p /tmp/gradletest && cd /tmp/gradletest \
+    && cat > build.gradle <<'EOF'
+configurations { deps }
+dependencies { deps 'org.interdiode:idconfiguratortestpkg:0.0.1' }
+task resolveDeps { doLast { configurations.deps.each { println it } } }
+EOF
+RUN cd /tmp/gradletest && gradle -q resolveDeps | grep -q idconfiguratortestpkg
+
+RUN java -jar /usr/share/java/ivy.jar -settings /root/.ivy2/ivysettings.xml \
+       -dependency org.interdiode idconfiguratortestpkg 0.0.1 \
+       -retrieve "/tmp/ivyretrieve/[artifact]-[revision].[ext]" \
+    && test -f /tmp/ivyretrieve/idconfiguratortestpkg-0.0.1.jar
+
+RUN mkdir -p /tmp/leintest && cd /tmp/leintest \
+    && cat > project.clj <<'EOF'
+(defproject leintest "0.1.0"
+  :dependencies [[org.interdiode/idconfiguratortestpkg "0.0.1"]])
+EOF
+RUN cd /tmp/leintest && lein deps :tree | grep -q idconfiguratortestpkg
+
+RUN mkdir -p /tmp/sbttest/project \
+    && cat > /tmp/sbttest/build.sbt <<'EOF'
+scalaVersion := "2.13.14"
+libraryDependencies += "org.interdiode" % "idconfiguratortestpkg" % "0.0.1"
+EOF
+RUN echo 'sbt.version=1.10.1' > /tmp/sbttest/project/build.properties
+RUN cd /tmp/sbttest \
+    && sbt -Dsbt.override.build.repos=true "show Compile/dependencyClasspath" | grep -q idconfiguratortestpkg

tests/dockerfiles/Python.Dockerfile

@@ -4,6 +4,21 @@ COPY --from=target / /src/
 COPY --from=generated-config Python.yaml /config.yaml
 RUN id-configurator apply --config /config.yaml --src /src --dst /dst
 FROM target AS runtime
+ARG PYPI_REPO_PORT
+# Warm up uv and poetry themselves from the real, network-reachable PyPI
+# while the default index is still in place, so the actual tests below
+# never need anything but the fixture repository configured by
+# id-configurator. (easy_install/distutils, the fourth tool the `python`
+# module configures, is not exercised here: both were removed from Python
+# itself in 3.12, and setuptools no longer ships easy_install either, so
+# there is no way to actually invoke it on a current interpreter.)
+RUN pip install --no-cache-dir uv poetry
 COPY --from=configuration /dst/ /
 RUN pip install --no-cache-dir idconfiguratortestpkg \
     && python3 -c "import idconfiguratortestpkg"
+RUN uv pip install --system idconfiguratortestpkg \
+    && python3 -c "import idconfiguratortestpkg"
+RUN mkdir -p /tmp/poetrytest && cd /tmp/poetrytest \
+    && poetry init -n --name poetrytest \
+    && poetry source add --priority explicit demo "https://host.docker.internal:${PYPI_REPO_PORT}/simple/" \
+    && poetry add idconfiguratortestpkg --source demo

tests/dockerfiles/repos/build_test_jar.py

@@ -9,6 +9,8 @@ can run in a minimal Python image with nothing installed.
 
 from __future__ import annotations
 
+import hashlib
+import io
 import pathlib
 import zipfile
 
@@ -32,19 +34,32 @@ POM = f"""<?xml version="1.0" encoding="UTF-8"?>
 """
 
 
-def build_jar(path: pathlib.Path) -> None:
+def build_jar() -> bytes:
-    with zipfile.ZipFile(path, "w", zipfile.ZIP_DEFLATED) as archive:
+    buffer = io.BytesIO()
+    with zipfile.ZipFile(buffer, "w", zipfile.ZIP_DEFLATED) as archive:
         archive.writestr("META-INF/MANIFEST.MF", MANIFEST)
         archive.writestr(
             "marker.txt",
             "test-only artifact served by the id-configurator integration tests\n",
         )
+    return buffer.getvalue()
+
+
+def _write_with_checksums(path: pathlib.Path, content: bytes) -> None:
+    """Write `content` to `path`, plus the .sha1/.md5 sidecar files most
+    build tools (Leiningen's aether resolver in particular) require to be
+    present, unlike Maven which merely warns when they are missing."""
+    path.write_bytes(content)
+    sha1_digest = hashlib.sha1(content).hexdigest()  # nosec B324
+    (path.parent / f"{path.name}.sha1").write_text(sha1_digest, encoding="ascii")
+    md5_digest = hashlib.md5(content).hexdigest()  # nosec B324
+    (path.parent / f"{path.name}.md5").write_text(md5_digest, encoding="ascii")
 
 
 def main() -> None:
     OUT_DIR.mkdir(parents=True, exist_ok=True)
-    build_jar(OUT_DIR / f"{ARTIFACT}-{VERSION}.jar")
+    _write_with_checksums(OUT_DIR / f"{ARTIFACT}-{VERSION}.jar", build_jar())
-    (OUT_DIR / f"{ARTIFACT}-{VERSION}.pom").write_text(POM, encoding="utf-8")
+    _write_with_checksums(OUT_DIR / f"{ARTIFACT}-{VERSION}.pom", POM.encode("utf-8"))
 
 
 if __name__ == "__main__":

tox.ini

@@ -78,7 +78,7 @@ allowlist_externals =
     docker
 commands =
     python3 -m id_tools test render-config {tox_root}/tests/configs/Python.yaml.j2 {env_dir}/configs/Python.yaml
-    docker buildx build --platform linux/amd64 --add-host=host.docker.internal:host-gateway --build-context generated-config={env_dir}/configs -f {tox_root}/tests/dockerfiles/Python.Dockerfile -t id-configurator-python .
+    docker buildx build --platform linux/amd64 --add-host=host.docker.internal:host-gateway --build-arg PYPI_REPO_PORT={env:PYPI_REPO_PORT} --build-context generated-config={env_dir}/configs -f {tox_root}/tests/dockerfiles/Python.Dockerfile -t id-configurator-python .
 
 [docker:apt-repo]
 dockerfile = {tox_root}/tests/dockerfiles/repos/apt-repo.Dockerfile