From 665eb215bdba6eca68327987ca42f374eccb06d2 Mon Sep 17 00:00:00 2001 From: d9pouces Date: Tue, 6 Oct 2026 20:34:04 +0200 Subject: [PATCH 1/3] feat: add repository name sanitization to prevent invalid characters in poetry config Signed-off-by: d9pouces --- id_configurator/modules/python.py | 19 +++++++++++++++-- pyproject.toml | 2 +- tests/test_python.py | 35 +++++++++++++++++++++++++++++++ 3 files changed, 53 insertions(+), 3 deletions(-) diff --git a/id_configurator/modules/python.py b/id_configurator/modules/python.py index 4081805..083cb9b 100644 --- a/id_configurator/modules/python.py +++ b/id_configurator/modules/python.py @@ -25,6 +25,7 @@ from __future__ import annotations import json import logging +import re from urllib.parse import urlparse from id_configurator.config import PythonRepository @@ -38,12 +39,25 @@ _PIP_CONF = "etc/pip.conf" _UV_TOML = "etc/uv/uv.toml" _POETRY_CONFIG = "root/.config/pypoetry/config.toml" _DISTUTILS_GLOBS = ("usr/lib/python3.*", "usr/local/lib/python3.*") +_REPOSITORY_NAME_FORBIDDEN_CHARS = re.compile(r"[^A-Za-z0-9_-]") def _toml_string(value: str) -> str: return json.dumps(value) +def _sanitize_repository_name(name: str) -> str: + """Replace every character of `name` that isn't a plain identifier + character with `_`. + + `name` is written verbatim as a TOML table key (`[repositories.name]`, + `[certificates.name]`) in poetry's config: an unrestricted name could + break out of the table header, or (via a literal `.`) silently create + extra, nested tables instead of naming the one table it is meant to. + """ + return _REPOSITORY_NAME_FORBIDDEN_CHARS.sub("_", name) + + def _trusted_host(repository: PythonRepository) -> str | None: parsed = urlparse(repository.index_url) return None if parsed.scheme.lower() == "https" else parsed.hostname @@ -83,10 +97,11 @@ def _write_poetry_config( ) -> None: lines = [] for repository in repositories: - lines.append(f"[repositories.{repository.name}]") + name = _sanitize_repository_name(repository.name) + lines.append(f"[repositories.{name}]") lines.append(f"url = {_toml_string(repository.index_url)}") lines.append("") - lines.append(f"[certificates.{repository.name}]") + lines.append(f"[certificates.{name}]") lines.append(f"cert = {_toml_string(_CA_BUNDLE)}") lines.append("") content = ("\n".join(lines).rstrip() + "\n").encode("utf-8") diff --git a/pyproject.toml b/pyproject.toml index 983685f..47049f6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,7 +44,7 @@ maintainers = [ name = "id-configurator" readme = "README.md" requires-python = ">=3.13" -version = "2026.10.5" +version = "2026.10.6" [project.scripts] id-configurator = "id_configurator:main" diff --git a/tests/test_python.py b/tests/test_python.py index 0747633..c993c83 100644 --- a/tests/test_python.py +++ b/tests/test_python.py @@ -183,6 +183,41 @@ def test_apply_multiple_repositories(tmp_path) -> None: ) +def test_apply_sanitizes_poetry_repository_name_with_special_characters( + tmp_path, +) -> None: + data = { + "uuid": "12345678-1234-5678-1234-567812345678", + "organisation": "InterDiode", + "configurations": { + "python": { + "repositories": [ + { + "name": "weird.name]here[bad", + "index-url": "https://example.invalid/simple", + } + ] + } + }, + } + config = Configuration.model_validate(data) + dst = tmp_path / "dst" + context = Context(config=config, dst=dst, src=None) + + PythonModule().apply(context) + + poetry_config = (dst / "root/.config/pypoetry/config.toml").read_text( + encoding="utf-8" + ) + assert poetry_config == ( + "[repositories.weird_name_here_bad]\n" + 'url = "https://example.invalid/simple"\n' + "\n" + "[certificates.weird_name_here_bad]\n" + 'cert = "/etc/ssl/certs/ca-certificates.crt"\n' + ) + + def test_apply_writes_poetry_config_for_each_configured_user(tmp_path) -> None: config = Configuration.model_validate(ONE_REPO_CONFIG) dst = tmp_path / "dst" From 8598f323d9437ac2ec62fbf6173b3fc8d10cd2b2 Mon Sep 17 00:00:00 2001 From: d9pouces Date: Tue, 6 Oct 2026 22:08:02 +0200 Subject: [PATCH 2/3] feat: add plain-HTTP listeners for various repository configurations to support non-HTTPS access Signed-off-by: d9pouces --- id_configurator/modules/base.py | 23 ++ id_configurator/modules/java.py | 16 +- id_configurator/modules/python.py | 18 +- id_configurator/modules/rust.py | 21 +- id_configurator/modules/yum.py | 23 +- tests/configs/Apk.yaml.j2 | 3 +- tests/configs/Apt.yaml.j2 | 3 +- tests/configs/Go.yaml.j2 | 3 +- tests/configs/Java.yaml.j2 | 3 +- tests/configs/Npm.yaml.j2 | 3 +- tests/configs/NuGet.yaml.j2 | 3 +- tests/configs/Php.yaml.j2 | 3 +- tests/configs/Python.yaml.j2 | 3 +- tests/configs/Ruby.yaml.j2 | 3 +- tests/configs/Rust.yaml.j2 | 3 +- tests/configs/Yum.yaml.j2 | 3 +- tests/dockerfiles/Python-alpine.Dockerfile | 3 +- tests/dockerfiles/Python.Dockerfile | 3 +- tests/dockerfiles/repos/apk-repo.nginx.conf | 10 + tests/dockerfiles/repos/apt-repo.nginx.conf | 10 + tests/dockerfiles/repos/cargo-repo.nginx.conf | 22 +- tests/dockerfiles/repos/go-repo.nginx.conf | 10 + tests/dockerfiles/repos/java-repo.nginx.conf | 10 + tests/dockerfiles/repos/npm-repo.nginx.conf | 23 +- tests/dockerfiles/repos/nuget-repo.nginx.conf | 22 +- tests/dockerfiles/repos/php-repo.nginx.conf | 22 +- tests/dockerfiles/repos/pypi-repo.nginx.conf | 10 + tests/dockerfiles/repos/ruby-repo.nginx.conf | 10 + tests/dockerfiles/repos/yum-repo.nginx.conf | 10 + tests/test_java.py | 24 ++ tox.ini | 248 +++++++++++++++++- uv.lock | 2 +- 32 files changed, 495 insertions(+), 78 deletions(-) diff --git a/id_configurator/modules/base.py b/id_configurator/modules/base.py index 4a13bde..ccfb80a 100644 --- a/id_configurator/modules/base.py +++ b/id_configurator/modules/base.py @@ -7,6 +7,7 @@ the destination directory. from __future__ import annotations +import re from abc import ABC, abstractmethod from id_configurator.context import Context @@ -49,3 +50,25 @@ def sanitize_filename(name: str) -> str: create - instead of the single file the caller asked for. """ return name.replace("/", "_").replace("\\", "_") + + +_REPOSITORY_NAME_FORBIDDEN_CHARS = re.compile(r"[^A-Za-z0-9_-]") + + +def sanitize_repository_name(name: str) -> str: + """Replace every character of `name` that isn't a plain identifier + character (letters, digits, `_`, `-`) with `_`. + + Several modules write a repository's name verbatim as a bare, + unquoted identifier in a generated config file - an INI section + header, a TOML table key, ... - where an unrestricted name could break + out of that syntax (e.g. `]`, a newline, or - for TOML - a literal + `.`, which would silently open extra, nested tables instead of naming + the one table it's meant to). Excluding `.` from the allowed + characters also means the result can never come out as exactly `.` or + `..`, which matter if the name is ever reused as a filesystem path + component (`sanitize_filename` alone wouldn't catch those, since it + only strips `/` and `\\`). Spaces and every other non-identifier + character are replaced the same way, for consistency across callers. + """ + return _REPOSITORY_NAME_FORBIDDEN_CHARS.sub("_", name) diff --git a/id_configurator/modules/java.py b/id_configurator/modules/java.py index 4b30c26..711e1eb 100644 --- a/id_configurator/modules/java.py +++ b/id_configurator/modules/java.py @@ -30,7 +30,10 @@ tool targets Docker images, which are built and run as root: own bootstrapping and, when combined with the `sbt.override.build.repos` system property, for every project's resolvers as well. - Leiningen: `~/.lein/profiles.clj`, adding the repositories to the `:user` - profile. + profile. Leiningen, alone among these, refuses a plain-HTTP repository + outright ("Tried to use insecure HTTP repository without TLS"); a + repository whose URL isn't HTTPS gets `:insecure true` added to its entry, + Leiningen's own documented escape hatch for this. """ from __future__ import annotations @@ -39,6 +42,7 @@ import json import logging import xml.etree.ElementTree as ET from pathlib import Path +from urllib.parse import urlparse import jks from cryptography import x509 @@ -244,10 +248,16 @@ def _write_sbt_repositories( logger.info("configured sbt to use %d repositories", len(repositories)) +def _lein_repository_entry(repository: JavaRepository) -> str: + options = f":url {_string_literal(repository.url)}" + if urlparse(repository.url).scheme.lower() != "https": + options += " :insecure true" + return f"[{_string_literal(repository.name)} {{{options}}}]" + + def _write_lein_profiles(context: Context, repositories: list[JavaRepository]) -> None: entries = " ".join( - f"[{_string_literal(repository.name)} {{:url {_string_literal(repository.url)}}}]" - for repository in repositories + _lein_repository_entry(repository) for repository in repositories ) content = f"{{:user {{:repositories [{entries}]}}}}\n" context.write_file(_LEIN_PROFILES, content.encode("utf-8")) diff --git a/id_configurator/modules/python.py b/id_configurator/modules/python.py index 083cb9b..7982738 100644 --- a/id_configurator/modules/python.py +++ b/id_configurator/modules/python.py @@ -25,12 +25,11 @@ from __future__ import annotations import json import logging -import re from urllib.parse import urlparse from id_configurator.config import PythonRepository from id_configurator.context import Context -from id_configurator.modules.base import Module, register +from id_configurator.modules.base import Module, register, sanitize_repository_name logger = logging.getLogger(__name__) @@ -39,25 +38,12 @@ _PIP_CONF = "etc/pip.conf" _UV_TOML = "etc/uv/uv.toml" _POETRY_CONFIG = "root/.config/pypoetry/config.toml" _DISTUTILS_GLOBS = ("usr/lib/python3.*", "usr/local/lib/python3.*") -_REPOSITORY_NAME_FORBIDDEN_CHARS = re.compile(r"[^A-Za-z0-9_-]") def _toml_string(value: str) -> str: return json.dumps(value) -def _sanitize_repository_name(name: str) -> str: - """Replace every character of `name` that isn't a plain identifier - character with `_`. - - `name` is written verbatim as a TOML table key (`[repositories.name]`, - `[certificates.name]`) in poetry's config: an unrestricted name could - break out of the table header, or (via a literal `.`) silently create - extra, nested tables instead of naming the one table it is meant to. - """ - return _REPOSITORY_NAME_FORBIDDEN_CHARS.sub("_", name) - - def _trusted_host(repository: PythonRepository) -> str | None: parsed = urlparse(repository.index_url) return None if parsed.scheme.lower() == "https" else parsed.hostname @@ -97,7 +83,7 @@ def _write_poetry_config( ) -> None: lines = [] for repository in repositories: - name = _sanitize_repository_name(repository.name) + name = sanitize_repository_name(repository.name) lines.append(f"[repositories.{name}]") lines.append(f"url = {_toml_string(repository.index_url)}") lines.append("") diff --git a/id_configurator/modules/rust.py b/id_configurator/modules/rust.py index 02843c9..741056b 100644 --- a/id_configurator/modules/rust.py +++ b/id_configurator/modules/rust.py @@ -29,29 +29,14 @@ which this module otherwise has no visibility into. from __future__ import annotations import logging -import re from id_configurator.config import RustRepository from id_configurator.context import Context -from id_configurator.modules.base import Module, register +from id_configurator.modules.base import Module, register, sanitize_repository_name logger = logging.getLogger(__name__) _CARGO_CONFIG = ".cargo/config.toml" -_REPOSITORY_NAME_FORBIDDEN_CHARS = re.compile(r"[^A-Za-z0-9_-]") - - -def _sanitize_repository_name(name: str) -> str: - """Replace every character of `name` that isn't a plain identifier - character with `_`. - - `name` is written verbatim both as a TOML table key (`[source.name]`, - `[registries.name]`) and inside a quoted string (`replace-with = - "name"`): an unrestricted name could break out of the table header or - string, or (via a literal `.`) silently create extra, nested tables - instead of naming the one table it is meant to. - """ - return _REPOSITORY_NAME_FORBIDDEN_CHARS.sub("_", name) def _index_url(repository: RustRepository) -> str: @@ -60,7 +45,7 @@ def _index_url(repository: RustRepository) -> str: def _write_cargo_config(context: Context, repositories: list[RustRepository]) -> None: primary = repositories[0] - primary_name = _sanitize_repository_name(primary.name) + primary_name = sanitize_repository_name(primary.name) lines = [ "[source.crates-io]", f'replace-with = "{primary_name}"', @@ -69,7 +54,7 @@ def _write_cargo_config(context: Context, repositories: list[RustRepository]) -> f'registry = "{_index_url(primary)}"', ] for repository in repositories[1:]: - name = _sanitize_repository_name(repository.name) + name = sanitize_repository_name(repository.name) lines += [ "", f"[registries.{name}]", diff --git a/id_configurator/modules/yum.py b/id_configurator/modules/yum.py index f8c1b3f..df8871c 100644 --- a/id_configurator/modules/yum.py +++ b/id_configurator/modules/yum.py @@ -32,7 +32,6 @@ from __future__ import annotations import configparser import fnmatch import logging -import re import shutil import subprocess import tempfile @@ -41,25 +40,17 @@ from io import StringIO from id_configurator.config import NamedContent, YumRepository from id_configurator.context import Context -from id_configurator.modules.base import Module, register, sanitize_filename +from id_configurator.modules.base import ( + Module, + register, + sanitize_filename, + sanitize_repository_name, +) logger = logging.getLogger(__name__) _REPOS_DIR = "etc/yum.repos.d" _RPM_GPG_DIR = "etc/pki/rpm-gpg" -_REPOSITORY_NAME_FORBIDDEN_CHARS = re.compile(r"[^A-Za-z0-9._-]") - - -def _sanitize_repository_name(name: str) -> str: - """Replace every character of `name` that isn't a plain identifier - character with `_`. - - `name` is written verbatim as the `.repo` file's INI section header - (`[name]`): an unrestricted name (containing `]`, a newline, ...) could - otherwise inject extra, attacker-controlled sections into the file - instead of just naming the one section it is meant to. - """ - return _REPOSITORY_NAME_FORBIDDEN_CHARS.sub("_", name) def _disable_repo_content(content: str, patterns: Sequence[str]) -> str: @@ -105,7 +96,7 @@ def _disable_existing_repos(context: Context, patterns: Sequence[str]) -> None: def _write_repo_file(context: Context, repository: YumRepository) -> None: - name = _sanitize_repository_name(repository.name) + name = sanitize_repository_name(repository.name) lines = [ f"[{name}]", f"name={name}", diff --git a/tests/configs/Apk.yaml.j2 b/tests/configs/Apk.yaml.j2 index 86787d6..2113056 100644 --- a/tests/configs/Apk.yaml.j2 +++ b/tests/configs/Apk.yaml.j2 @@ -11,7 +11,8 @@ configurations: apk: repositories: - name: demo - url: https://host.docker.internal:{{ 'APK_REPO_PORT' | env }}/ +{% set scheme = 'APK_REPO_SCHEME' | env %} + url: {{ scheme }}://host.docker.internal:{{ ('APK_REPO_PORT' if scheme == 'https' else 'APK_REPO_HTTP_PORT') | env }}/ url_to_replace: https://dl-cdn.alpinelinux.org/* key: | {{ 'dockerfiles/repos/040d0af5f97a0a44.rsa.pub' | file | indent(8, first=true) }} diff --git a/tests/configs/Apt.yaml.j2 b/tests/configs/Apt.yaml.j2 index 504c7ac..5f729e4 100644 --- a/tests/configs/Apt.yaml.j2 +++ b/tests/configs/Apt.yaml.j2 @@ -11,7 +11,8 @@ configurations: - name: demo types: [deb] uris: - - https://host.docker.internal:{{ 'APT_REPO_PORT' | env }}/ +{% set scheme = 'APT_REPO_SCHEME' | env %} + - {{ scheme }}://host.docker.internal:{{ ('APT_REPO_PORT' if scheme == 'https' else 'APT_REPO_HTTP_PORT') | env }}/ suites: ["./"] uri-to-replace: http://deb.debian.org/* signed-by: | diff --git a/tests/configs/Go.yaml.j2 b/tests/configs/Go.yaml.j2 index 2f7ce8b..75089c6 100644 --- a/tests/configs/Go.yaml.j2 +++ b/tests/configs/Go.yaml.j2 @@ -26,7 +26,8 @@ configurations: go: repositories: - name: demo - url: https://host.docker.internal:{{ 'GO_REPO_PORT' | env }}/ +{% set scheme = 'GO_REPO_SCHEME' | env %} + url: {{ scheme }}://host.docker.internal:{{ ('GO_REPO_PORT' if scheme == 'https' else 'GO_REPO_HTTP_PORT') | env }}/ ruby: repositories: [] nuget: diff --git a/tests/configs/Java.yaml.j2 b/tests/configs/Java.yaml.j2 index 532ffc4..80b63d6 100644 --- a/tests/configs/Java.yaml.j2 +++ b/tests/configs/Java.yaml.j2 @@ -23,7 +23,8 @@ configurations: # during the Dockerfile's warm-up step. All other tools just see it as # an ordinary repository, and find the test artifact here. - name: central - url: https://host.docker.internal:{{ 'JAVA_REPO_PORT' | env }}/ +{% set scheme = 'JAVA_REPO_SCHEME' | env %} + url: {{ scheme }}://host.docker.internal:{{ ('JAVA_REPO_PORT' if scheme == 'https' else 'JAVA_REPO_HTTP_PORT') | env }}/ # Second entry: only SBT needs this. With # -Dsbt.override.build.repos=true (the only way SBT actually consults # this file for project dependencies, not just its own bootstrap), the diff --git a/tests/configs/Npm.yaml.j2 b/tests/configs/Npm.yaml.j2 index 349ec0e..a1f0954 100644 --- a/tests/configs/Npm.yaml.j2 +++ b/tests/configs/Npm.yaml.j2 @@ -22,7 +22,8 @@ configurations: npm: repositories: - name: demo - url: https://host.docker.internal:{{ 'NPM_REPO_PORT' | env }}/ +{% set scheme = 'NPM_REPO_SCHEME' | env %} + url: {{ scheme }}://host.docker.internal:{{ ('NPM_REPO_PORT' if scheme == 'https' else 'NPM_REPO_HTTP_PORT') | env }}/ rust: repositories: [] go: diff --git a/tests/configs/NuGet.yaml.j2 b/tests/configs/NuGet.yaml.j2 index 5fd3287..2842a99 100644 --- a/tests/configs/NuGet.yaml.j2 +++ b/tests/configs/NuGet.yaml.j2 @@ -30,4 +30,5 @@ configurations: nuget: repositories: - name: demo - url: https://host.docker.internal:{{ 'NUGET_REPO_PORT' | env }}/index.json +{% set scheme = 'NUGET_REPO_SCHEME' | env %} + url: {{ scheme }}://host.docker.internal:{{ ('NUGET_REPO_PORT' if scheme == 'https' else 'NUGET_REPO_HTTP_PORT') | env }}/index.json diff --git a/tests/configs/Php.yaml.j2 b/tests/configs/Php.yaml.j2 index 11c1a0c..1b11361 100644 --- a/tests/configs/Php.yaml.j2 +++ b/tests/configs/Php.yaml.j2 @@ -20,7 +20,8 @@ configurations: php: repositories: - name: demo - url: https://host.docker.internal:{{ 'PHP_REPO_PORT' | env }}/ +{% set scheme = 'PHP_REPO_SCHEME' | env %} + url: {{ scheme }}://host.docker.internal:{{ ('PHP_REPO_PORT' if scheme == 'https' else 'PHP_REPO_HTTP_PORT') | env }}/ npm: repositories: [] rust: diff --git a/tests/configs/Python.yaml.j2 b/tests/configs/Python.yaml.j2 index 749151e..45bd9ed 100644 --- a/tests/configs/Python.yaml.j2 +++ b/tests/configs/Python.yaml.j2 @@ -16,7 +16,8 @@ configurations: python: repositories: - name: demo - index-url: https://host.docker.internal:{{ 'PYPI_REPO_PORT' | env }}/simple/ +{% set scheme = 'PYPI_REPO_SCHEME' | env %} + index-url: {{ scheme }}://host.docker.internal:{{ ('PYPI_REPO_PORT' if scheme == 'https' else 'PYPI_REPO_HTTP_PORT') | env }}/simple/ java: repositories: [] php: diff --git a/tests/configs/Ruby.yaml.j2 b/tests/configs/Ruby.yaml.j2 index ccb6447..12be460 100644 --- a/tests/configs/Ruby.yaml.j2 +++ b/tests/configs/Ruby.yaml.j2 @@ -28,6 +28,7 @@ configurations: ruby: repositories: - name: demo - url: https://host.docker.internal:{{ 'RUBY_REPO_PORT' | env }}/ +{% set scheme = 'RUBY_REPO_SCHEME' | env %} + url: {{ scheme }}://host.docker.internal:{{ ('RUBY_REPO_PORT' if scheme == 'https' else 'RUBY_REPO_HTTP_PORT') | env }}/ nuget: repositories: [] diff --git a/tests/configs/Rust.yaml.j2 b/tests/configs/Rust.yaml.j2 index 2a6d91a..d1aa9a2 100644 --- a/tests/configs/Rust.yaml.j2 +++ b/tests/configs/Rust.yaml.j2 @@ -24,7 +24,8 @@ configurations: rust: repositories: - name: demo - url: https://host.docker.internal:{{ 'CARGO_REPO_PORT' | env }}/ +{% set scheme = 'CARGO_REPO_SCHEME' | env %} + url: {{ scheme }}://host.docker.internal:{{ ('CARGO_REPO_PORT' if scheme == 'https' else 'CARGO_REPO_HTTP_PORT') | env }}/ is_sparse: true go: repositories: [] diff --git a/tests/configs/Yum.yaml.j2 b/tests/configs/Yum.yaml.j2 index 5696450..a453237 100644 --- a/tests/configs/Yum.yaml.j2 +++ b/tests/configs/Yum.yaml.j2 @@ -13,7 +13,8 @@ configurations: yum: repositories: - name: demo - index-url: https://host.docker.internal:{{ 'YUM_REPO_PORT' | env }}/ +{% set scheme = 'YUM_REPO_SCHEME' | env %} + index-url: {{ scheme }}://host.docker.internal:{{ ('YUM_REPO_PORT' if scheme == 'https' else 'YUM_REPO_HTTP_PORT') | env }}/ index-url-to-replace: https://mirrors.rockylinux.org/* keys: [] python: diff --git a/tests/dockerfiles/Python-alpine.Dockerfile b/tests/dockerfiles/Python-alpine.Dockerfile index 3c8317b..b72f880 100644 --- a/tests/dockerfiles/Python-alpine.Dockerfile +++ b/tests/dockerfiles/Python-alpine.Dockerfile @@ -5,6 +5,7 @@ COPY --from=generated-config Python.yaml /config.yaml RUN id-configurator apply --config /config.yaml --src /src --dst /dst FROM target AS runtime ARG PYPI_REPO_PORT +ARG PYPI_REPO_SCHEME # Warm up uv and poetry themselves from the real, network-reachable PyPI # while the default index is still in place, so the actual tests below # never need anything but the fixture repository configured by @@ -20,5 +21,5 @@ RUN uv pip install --system idconfiguratortestpkg \ && python3 -c "import idconfiguratortestpkg" RUN mkdir -p /tmp/poetrytest && cd /tmp/poetrytest \ && poetry init -n --name poetrytest \ - && poetry source add --priority explicit demo "https://host.docker.internal:${PYPI_REPO_PORT}/simple/" \ + && poetry source add --priority explicit demo "${PYPI_REPO_SCHEME}://host.docker.internal:${PYPI_REPO_PORT}/simple/" \ && poetry add idconfiguratortestpkg --source demo diff --git a/tests/dockerfiles/Python.Dockerfile b/tests/dockerfiles/Python.Dockerfile index b464ebe..3107b35 100644 --- a/tests/dockerfiles/Python.Dockerfile +++ b/tests/dockerfiles/Python.Dockerfile @@ -5,6 +5,7 @@ COPY --from=generated-config Python.yaml /config.yaml RUN id-configurator apply --config /config.yaml --src /src --dst /dst FROM target AS runtime ARG PYPI_REPO_PORT +ARG PYPI_REPO_SCHEME # Warm up uv and poetry themselves from the real, network-reachable PyPI # while the default index is still in place, so the actual tests below # never need anything but the fixture repository configured by @@ -20,5 +21,5 @@ RUN uv pip install --system idconfiguratortestpkg \ && python3 -c "import idconfiguratortestpkg" RUN mkdir -p /tmp/poetrytest && cd /tmp/poetrytest \ && poetry init -n --name poetrytest \ - && poetry source add --priority explicit demo "https://host.docker.internal:${PYPI_REPO_PORT}/simple/" \ + && poetry source add --priority explicit demo "${PYPI_REPO_SCHEME}://host.docker.internal:${PYPI_REPO_PORT}/simple/" \ && poetry add idconfiguratortestpkg --source demo diff --git a/tests/dockerfiles/repos/apk-repo.nginx.conf b/tests/dockerfiles/repos/apk-repo.nginx.conf index aafed09..935425e 100644 --- a/tests/dockerfiles/repos/apk-repo.nginx.conf +++ b/tests/dockerfiles/repos/apk-repo.nginx.conf @@ -8,3 +8,13 @@ server { root /srv; autoindex off; } + +# Plain-HTTP listener, for the module-*-http tox envs that verify id-configurator +# correctly configures a non-HTTPS repository. +server { + listen 8080 default_server; + server_name _; + + root /srv; + autoindex off; +} diff --git a/tests/dockerfiles/repos/apt-repo.nginx.conf b/tests/dockerfiles/repos/apt-repo.nginx.conf index aafed09..935425e 100644 --- a/tests/dockerfiles/repos/apt-repo.nginx.conf +++ b/tests/dockerfiles/repos/apt-repo.nginx.conf @@ -8,3 +8,13 @@ server { root /srv; autoindex off; } + +# Plain-HTTP listener, for the module-*-http tox envs that verify id-configurator +# correctly configures a non-HTTPS repository. +server { + listen 8080 default_server; + server_name _; + + root /srv; + autoindex off; +} diff --git a/tests/dockerfiles/repos/cargo-repo.nginx.conf b/tests/dockerfiles/repos/cargo-repo.nginx.conf index 98fe171..5cb468d 100644 --- a/tests/dockerfiles/repos/cargo-repo.nginx.conf +++ b/tests/dockerfiles/repos/cargo-repo.nginx.conf @@ -12,10 +12,28 @@ server { # templates, since the host:port this fixture is actually reached on is # only known once tox-docker starts the container, not at image-build # time. $http_host reflects exactly what the client (cargo) used to - # connect, e.g. host.docker.internal:55123. + # connect, e.g. host.docker.internal:55123; $scheme likewise reflects + # whichever of this file's two listeners actually served the request. location = /config.json { default_type application/json; - sub_filter '__BASE_URL__' 'https://$http_host'; + sub_filter '__BASE_URL__' '$scheme://$http_host'; + sub_filter_once off; + sub_filter_types application/json; + } +} + +# Plain-HTTP listener, for the module-rust-http tox env that verifies +# id-configurator correctly configures a non-HTTPS repository. +server { + listen 8080 default_server; + server_name _; + + root /srv; + autoindex off; + + location = /config.json { + default_type application/json; + sub_filter '__BASE_URL__' '$scheme://$http_host'; sub_filter_once off; sub_filter_types application/json; } diff --git a/tests/dockerfiles/repos/go-repo.nginx.conf b/tests/dockerfiles/repos/go-repo.nginx.conf index aafed09..935425e 100644 --- a/tests/dockerfiles/repos/go-repo.nginx.conf +++ b/tests/dockerfiles/repos/go-repo.nginx.conf @@ -8,3 +8,13 @@ server { root /srv; autoindex off; } + +# Plain-HTTP listener, for the module-*-http tox envs that verify id-configurator +# correctly configures a non-HTTPS repository. +server { + listen 8080 default_server; + server_name _; + + root /srv; + autoindex off; +} diff --git a/tests/dockerfiles/repos/java-repo.nginx.conf b/tests/dockerfiles/repos/java-repo.nginx.conf index aafed09..935425e 100644 --- a/tests/dockerfiles/repos/java-repo.nginx.conf +++ b/tests/dockerfiles/repos/java-repo.nginx.conf @@ -8,3 +8,13 @@ server { root /srv; autoindex off; } + +# Plain-HTTP listener, for the module-*-http tox envs that verify id-configurator +# correctly configures a non-HTTPS repository. +server { + listen 8080 default_server; + server_name _; + + root /srv; + autoindex off; +} diff --git a/tests/dockerfiles/repos/npm-repo.nginx.conf b/tests/dockerfiles/repos/npm-repo.nginx.conf index 0e95535..0c97e3d 100644 --- a/tests/dockerfiles/repos/npm-repo.nginx.conf +++ b/tests/dockerfiles/repos/npm-repo.nginx.conf @@ -12,10 +12,29 @@ server { # its tarball URL, since the host:port this fixture is actually reached # on is only known once tox-docker starts the container, not at # image-build time. $http_host reflects exactly what the client (npm) - # used to connect, e.g. host.docker.internal:55123. + # used to connect, e.g. host.docker.internal:55123; $scheme likewise + # reflects whichever of this file's two listeners actually served the + # request. location = /idconfiguratortestpkg { default_type application/json; - sub_filter '__BASE_URL__' 'https://$http_host'; + sub_filter '__BASE_URL__' '$scheme://$http_host'; + sub_filter_once off; + sub_filter_types application/json; + } +} + +# Plain-HTTP listener, for the module-npm-http tox env that verifies +# id-configurator correctly configures a non-HTTPS repository. +server { + listen 8080 default_server; + server_name _; + + root /srv; + autoindex off; + + location = /idconfiguratortestpkg { + default_type application/json; + sub_filter '__BASE_URL__' '$scheme://$http_host'; sub_filter_once off; sub_filter_types application/json; } diff --git a/tests/dockerfiles/repos/nuget-repo.nginx.conf b/tests/dockerfiles/repos/nuget-repo.nginx.conf index 22a6cc7..efe69c2 100644 --- a/tests/dockerfiles/repos/nuget-repo.nginx.conf +++ b/tests/dockerfiles/repos/nuget-repo.nginx.conf @@ -12,10 +12,28 @@ server { # fields, since the host:port this fixture is actually reached on is # only known once tox-docker starts the container, not at image-build # time. $http_host reflects exactly what the client (nuget) used to - # connect, e.g. host.docker.internal:55123. + # connect, e.g. host.docker.internal:55123; $scheme likewise reflects + # whichever of this file's two listeners actually served the request. location = /index.json { default_type application/json; - sub_filter '__BASE_URL__' 'https://$http_host'; + sub_filter '__BASE_URL__' '$scheme://$http_host'; + sub_filter_once off; + sub_filter_types application/json; + } +} + +# Plain-HTTP listener, for the module-nuget-http tox env that verifies +# id-configurator correctly configures a non-HTTPS repository. +server { + listen 8080 default_server; + server_name _; + + root /srv; + autoindex off; + + location = /index.json { + default_type application/json; + sub_filter '__BASE_URL__' '$scheme://$http_host'; sub_filter_once off; sub_filter_types application/json; } diff --git a/tests/dockerfiles/repos/php-repo.nginx.conf b/tests/dockerfiles/repos/php-repo.nginx.conf index b59093c..550a6b7 100644 --- a/tests/dockerfiles/repos/php-repo.nginx.conf +++ b/tests/dockerfiles/repos/php-repo.nginx.conf @@ -12,10 +12,28 @@ server { # since the host:port this fixture is actually reached on is only known # once tox-docker starts the container, not at image-build time. # $http_host reflects exactly what the client (Composer) used to - # connect, e.g. host.docker.internal:55123. + # connect, e.g. host.docker.internal:55123; $scheme likewise reflects + # whichever of this file's two listeners actually served the request. location = /packages.json { default_type application/json; - sub_filter '__BASE_URL__' 'https://$http_host'; + sub_filter '__BASE_URL__' '$scheme://$http_host'; + sub_filter_once off; + sub_filter_types application/json; + } +} + +# Plain-HTTP listener, for the module-php-http tox env that verifies +# id-configurator correctly configures a non-HTTPS repository. +server { + listen 8080 default_server; + server_name _; + + root /srv; + autoindex off; + + location = /packages.json { + default_type application/json; + sub_filter '__BASE_URL__' '$scheme://$http_host'; sub_filter_once off; sub_filter_types application/json; } diff --git a/tests/dockerfiles/repos/pypi-repo.nginx.conf b/tests/dockerfiles/repos/pypi-repo.nginx.conf index aafed09..935425e 100644 --- a/tests/dockerfiles/repos/pypi-repo.nginx.conf +++ b/tests/dockerfiles/repos/pypi-repo.nginx.conf @@ -8,3 +8,13 @@ server { root /srv; autoindex off; } + +# Plain-HTTP listener, for the module-*-http tox envs that verify id-configurator +# correctly configures a non-HTTPS repository. +server { + listen 8080 default_server; + server_name _; + + root /srv; + autoindex off; +} diff --git a/tests/dockerfiles/repos/ruby-repo.nginx.conf b/tests/dockerfiles/repos/ruby-repo.nginx.conf index aafed09..935425e 100644 --- a/tests/dockerfiles/repos/ruby-repo.nginx.conf +++ b/tests/dockerfiles/repos/ruby-repo.nginx.conf @@ -8,3 +8,13 @@ server { root /srv; autoindex off; } + +# Plain-HTTP listener, for the module-*-http tox envs that verify id-configurator +# correctly configures a non-HTTPS repository. +server { + listen 8080 default_server; + server_name _; + + root /srv; + autoindex off; +} diff --git a/tests/dockerfiles/repos/yum-repo.nginx.conf b/tests/dockerfiles/repos/yum-repo.nginx.conf index aafed09..935425e 100644 --- a/tests/dockerfiles/repos/yum-repo.nginx.conf +++ b/tests/dockerfiles/repos/yum-repo.nginx.conf @@ -8,3 +8,13 @@ server { root /srv; autoindex off; } + +# Plain-HTTP listener, for the module-*-http tox envs that verify id-configurator +# correctly configures a non-HTTPS repository. +server { + listen 8080 default_server; + server_name _; + + root /srv; + autoindex off; +} diff --git a/tests/test_java.py b/tests/test_java.py index 931534f..de5d416 100644 --- a/tests/test_java.py +++ b/tests/test_java.py @@ -353,6 +353,30 @@ def test_apply_writes_lein_profiles(tmp_path) -> None: ) +def test_apply_marks_non_https_lein_repository_as_insecure(tmp_path) -> None: + data = { + "uuid": "12345678-1234-5678-1234-567812345678", + "organisation": "InterDiode", + "configurations": { + "java": { + "repositories": [ + {"name": "interdiode", "url": "http://example.invalid/maven"} + ] + } + }, + } + dst = tmp_path / "dst" + context = Context(config=make_config(data), dst=dst, src=None) + + JavaModule().apply(context) + + profiles = (dst / "root/.lein/profiles.clj").read_text(encoding="utf-8") + assert profiles == ( + "{:user {:repositories " + '[["interdiode" {:url "http://example.invalid/maven" :insecure true}]]}}\n' + ) + + def test_apply_with_certificates_and_repositories_writes_both(tmp_path) -> None: src = tmp_path / "src" (src / "usr/lib/jvm/java-21/lib/security").mkdir(parents=True) diff --git a/tox.ini b/tox.ini index c233c97..2f50d16 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -env_list = py313,py314,coverage,packaging,packaging-amd64,packaging-arm64,publish,module-python,module-apt,module-yum,module-apk,module-java,module-certificates,module-php,module-npm,module-rust,module-go,module-ruby,module-nuget,config-url +env_list = py313,py314,coverage,packaging,packaging-amd64,packaging-arm64,publish,module-python,module-python-http,module-apt,module-apt-http,module-yum,module-yum-http,module-apk,module-apk-http,module-java,module-java-http,module-certificates,module-php,module-php-http,module-npm,module-npm-http,module-rust,module-rust-http,module-go,module-go-http,module-ruby,module-ruby-http,module-nuget,module-nuget-http,config-url isolated_build = true requires = tox>=4 @@ -91,6 +91,7 @@ passenv = dockerfile = {tox_root}/tests/dockerfiles/repos/pypi-repo.Dockerfile expose = PYPI_REPO_PORT=8443/tcp + PYPI_REPO_HTTP_PORT=8080/tcp healthcheck_cmd = wget --no-check-certificate -qO- https://127.0.0.1:8443/simple/idconfiguratortestpkg/ || exit 1 healthcheck_interval = 1 healthcheck_retries = 30 @@ -106,17 +107,41 @@ docker = pypi-repo allowlist_externals = docker +set_env = + PYPI_REPO_SCHEME = https commands = python3 -m id_tools test render-config {tox_root}/tests/configs/Python.yaml.j2 {env_dir}/configs/Python.yaml - docker buildx build --platform linux/amd64 --add-host=host.docker.internal:host-gateway --build-arg PYPI_REPO_PORT={env:PYPI_REPO_PORT} --build-context generated-config={env_dir}/configs -f {tox_root}/tests/dockerfiles/Python-alpine.Dockerfile -t id-configurator-python . + docker buildx build --platform linux/amd64 --add-host=host.docker.internal:host-gateway --build-arg PYPI_REPO_SCHEME={env:PYPI_REPO_SCHEME} --build-arg PYPI_REPO_PORT={env:PYPI_REPO_PORT} --build-context generated-config={env_dir}/configs -f {tox_root}/tests/dockerfiles/Python-alpine.Dockerfile -t id-configurator-python . docker rmi id-configurator-python - docker buildx build --platform linux/amd64 --add-host=host.docker.internal:host-gateway --build-arg PYPI_REPO_PORT={env:PYPI_REPO_PORT} --build-context generated-config={env_dir}/configs -f {tox_root}/tests/dockerfiles/Python.Dockerfile -t id-configurator-python . + docker buildx build --platform linux/amd64 --add-host=host.docker.internal:host-gateway --build-arg PYPI_REPO_SCHEME={env:PYPI_REPO_SCHEME} --build-arg PYPI_REPO_PORT={env:PYPI_REPO_PORT} --build-context generated-config={env_dir}/configs -f {tox_root}/tests/dockerfiles/Python.Dockerfile -t id-configurator-python . docker rmi id-configurator-python +[testenv:module-python-http] +package = skip +labels = end-to-end +change_dir = {tox_root} +dependency_groups = dev +docker = + pypi-repo +allowlist_externals = + docker +description = + Same as module-python, but against a plain-HTTP index, to verify pip's + trusted-host is configured correctly for a non-HTTPS repository. +set_env = + PYPI_REPO_SCHEME = http +commands = + python3 -m id_tools test render-config {tox_root}/tests/configs/Python.yaml.j2 {env_dir}/configs/Python.yaml + docker buildx build --platform linux/amd64 --add-host=host.docker.internal:host-gateway --build-arg PYPI_REPO_SCHEME={env:PYPI_REPO_SCHEME} --build-arg PYPI_REPO_PORT={env:PYPI_REPO_HTTP_PORT} --build-context generated-config={env_dir}/configs -f {tox_root}/tests/dockerfiles/Python-alpine.Dockerfile -t id-configurator-python-http . + docker rmi id-configurator-python-http + docker buildx build --platform linux/amd64 --add-host=host.docker.internal:host-gateway --build-arg PYPI_REPO_SCHEME={env:PYPI_REPO_SCHEME} --build-arg PYPI_REPO_PORT={env:PYPI_REPO_HTTP_PORT} --build-context generated-config={env_dir}/configs -f {tox_root}/tests/dockerfiles/Python.Dockerfile -t id-configurator-python-http . + docker rmi id-configurator-python-http + [docker:apt-repo] dockerfile = {tox_root}/tests/dockerfiles/repos/apt-repo.Dockerfile expose = APT_REPO_PORT=8443/tcp + APT_REPO_HTTP_PORT=8080/tcp healthcheck_cmd = wget --no-check-certificate -qO- https://127.0.0.1:8443/Packages || exit 1 healthcheck_interval = 1 healthcheck_retries = 30 @@ -132,15 +157,37 @@ docker = apt-repo allowlist_externals = docker +set_env = + APT_REPO_SCHEME = https commands = python3 -m id_tools test render-config {tox_root}/tests/configs/Apt.yaml.j2 {env_dir}/configs/Apt.yaml docker buildx build --platform linux/amd64 --add-host=host.docker.internal:host-gateway --build-context generated-config={env_dir}/configs -f {tox_root}/tests/dockerfiles/Apt.Dockerfile -t id-configurator-apt . docker rmi id-configurator-apt +[testenv:module-apt-http] +package = skip +labels = end-to-end +change_dir = {tox_root} +dependency_groups = dev +docker = + apt-repo +allowlist_externals = + docker +description = + Same as module-apt, but against a plain-HTTP source, since apt's trust + model is the signed-by keyring, independent of transport scheme. +set_env = + APT_REPO_SCHEME = http +commands = + python3 -m id_tools test render-config {tox_root}/tests/configs/Apt.yaml.j2 {env_dir}/configs/Apt.yaml + docker buildx build --platform linux/amd64 --add-host=host.docker.internal:host-gateway --build-context generated-config={env_dir}/configs -f {tox_root}/tests/dockerfiles/Apt.Dockerfile -t id-configurator-apt-http . + docker rmi id-configurator-apt-http + [docker:yum-repo] dockerfile = {tox_root}/tests/dockerfiles/repos/yum-repo.Dockerfile expose = YUM_REPO_PORT=8443/tcp + YUM_REPO_HTTP_PORT=8080/tcp healthcheck_cmd = wget --no-check-certificate -qO- https://127.0.0.1:8443/repodata/repomd.xml || exit 1 healthcheck_interval = 1 healthcheck_retries = 30 @@ -156,15 +203,37 @@ docker = yum-repo allowlist_externals = docker +set_env = + YUM_REPO_SCHEME = https commands = python3 -m id_tools test render-config {tox_root}/tests/configs/Yum.yaml.j2 {env_dir}/configs/Yum.yaml docker buildx build --platform linux/amd64 --add-host=host.docker.internal:host-gateway --build-context generated-config={env_dir}/configs -f {tox_root}/tests/dockerfiles/Yum.Dockerfile -t id-configurator-yum . docker rmi id-configurator-yum +[testenv:module-yum-http] +package = skip +labels = end-to-end +change_dir = {tox_root} +dependency_groups = dev +docker = + yum-repo +allowlist_externals = + docker +description = + Same as module-yum, but against a plain-HTTP repository, since yum/dnf's + trust model is the imported GPG key, independent of transport scheme. +set_env = + YUM_REPO_SCHEME = http +commands = + python3 -m id_tools test render-config {tox_root}/tests/configs/Yum.yaml.j2 {env_dir}/configs/Yum.yaml + docker buildx build --platform linux/amd64 --add-host=host.docker.internal:host-gateway --build-context generated-config={env_dir}/configs -f {tox_root}/tests/dockerfiles/Yum.Dockerfile -t id-configurator-yum-http . + docker rmi id-configurator-yum-http + [docker:apk-repo] dockerfile = {tox_root}/tests/dockerfiles/repos/apk-repo.Dockerfile expose = APK_REPO_PORT=8443/tcp + APK_REPO_HTTP_PORT=8080/tcp healthcheck_cmd = wget --no-check-certificate -qO- https://127.0.0.1:8443/APKINDEX.tar.gz || exit 1 healthcheck_interval = 1 healthcheck_retries = 30 @@ -189,15 +258,39 @@ description = native platform keeps them in agreement everywhere (including under QEMU emulation on non-amd64 hosts); CI runners are amd64, so this still matches the other envs there. +set_env = + APK_REPO_SCHEME = https commands = python3 -m id_tools test render-config {tox_root}/tests/configs/Apk.yaml.j2 {env_dir}/configs/Apk.yaml docker buildx build --add-host=host.docker.internal:host-gateway --build-context generated-config={env_dir}/configs -f {tox_root}/tests/dockerfiles/Apk.Dockerfile -t id-configurator-apk . docker rmi id-configurator-apk +[testenv:module-apk-http] +package = skip +labels = end-to-end +change_dir = {tox_root} +dependency_groups = dev +docker = + apk-repo +allowlist_externals = + docker +description = + Same as module-apk, but against a plain-HTTP repository, since apk's + trust model is the signed index, independent of transport scheme. See + module-apk's own description for why this isn't pinned to --platform + linux/amd64 either. +set_env = + APK_REPO_SCHEME = http +commands = + python3 -m id_tools test render-config {tox_root}/tests/configs/Apk.yaml.j2 {env_dir}/configs/Apk.yaml + docker buildx build --add-host=host.docker.internal:host-gateway --build-context generated-config={env_dir}/configs -f {tox_root}/tests/dockerfiles/Apk.Dockerfile -t id-configurator-apk-http . + docker rmi id-configurator-apk-http + [docker:java-repo] dockerfile = {tox_root}/tests/dockerfiles/repos/java-repo.Dockerfile expose = JAVA_REPO_PORT=8443/tcp + JAVA_REPO_HTTP_PORT=8080/tcp healthcheck_cmd = wget --no-check-certificate -qO- https://127.0.0.1:8443/org/interdiode/idconfiguratortestpkg/0.0.1/idconfiguratortestpkg-0.0.1.pom || exit 1 healthcheck_interval = 1 healthcheck_retries = 30 @@ -213,11 +306,31 @@ docker = java-repo allowlist_externals = docker +set_env = + JAVA_REPO_SCHEME = https commands = python3 -m id_tools test render-config {tox_root}/tests/configs/Java.yaml.j2 {env_dir}/configs/Java.yaml docker buildx build --platform linux/amd64 --add-host=host.docker.internal:host-gateway --build-context generated-config={env_dir}/configs -f {tox_root}/tests/dockerfiles/Java.Dockerfile -t id-configurator-java . docker rmi id-configurator-java +[testenv:module-java-http] +package = skip +labels = end-to-end +change_dir = {tox_root} +dependency_groups = dev +docker = + java-repo +allowlist_externals = + docker +description = + Same as module-java, but against a plain-HTTP repository. +set_env = + JAVA_REPO_SCHEME = http +commands = + python3 -m id_tools test render-config {tox_root}/tests/configs/Java.yaml.j2 {env_dir}/configs/Java.yaml + docker buildx build --platform linux/amd64 --add-host=host.docker.internal:host-gateway --build-context generated-config={env_dir}/configs -f {tox_root}/tests/dockerfiles/Java.Dockerfile -t id-configurator-java-http . + docker rmi id-configurator-java-http + [docker:certificate-repo] dockerfile = {tox_root}/tests/dockerfiles/repos/certificate-repo.Dockerfile expose = @@ -251,6 +364,7 @@ commands = dockerfile = {tox_root}/tests/dockerfiles/repos/php-repo.Dockerfile expose = PHP_REPO_PORT=8443/tcp + PHP_REPO_HTTP_PORT=8080/tcp healthcheck_cmd = wget --no-check-certificate -qO- https://127.0.0.1:8443/packages.json || exit 1 healthcheck_interval = 1 healthcheck_retries = 30 @@ -266,15 +380,37 @@ docker = php-repo allowlist_externals = docker +set_env = + PHP_REPO_SCHEME = https commands = python3 -m id_tools test render-config {tox_root}/tests/configs/Php.yaml.j2 {env_dir}/configs/Php.yaml docker buildx build --platform linux/amd64 --add-host=host.docker.internal:host-gateway --build-context generated-config={env_dir}/configs -f {tox_root}/tests/dockerfiles/Php.Dockerfile -t id-configurator-php . docker rmi id-configurator-php +[testenv:module-php-http] +package = skip +labels = end-to-end +change_dir = {tox_root} +dependency_groups = dev +docker = + php-repo +allowlist_externals = + docker +description = + Same as module-php, but against a plain-HTTP repository, to verify + secure-http is disabled (and packagist.org too) for a non-HTTPS source. +set_env = + PHP_REPO_SCHEME = http +commands = + python3 -m id_tools test render-config {tox_root}/tests/configs/Php.yaml.j2 {env_dir}/configs/Php.yaml + docker buildx build --platform linux/amd64 --add-host=host.docker.internal:host-gateway --build-context generated-config={env_dir}/configs -f {tox_root}/tests/dockerfiles/Php.Dockerfile -t id-configurator-php-http . + docker rmi id-configurator-php-http + [docker:npm-repo] dockerfile = {tox_root}/tests/dockerfiles/repos/npm-repo.Dockerfile expose = NPM_REPO_PORT=8443/tcp + NPM_REPO_HTTP_PORT=8080/tcp healthcheck_cmd = wget --no-check-certificate -qO- https://127.0.0.1:8443/idconfiguratortestpkg || exit 1 healthcheck_interval = 1 healthcheck_retries = 30 @@ -290,15 +426,36 @@ docker = npm-repo allowlist_externals = docker +set_env = + NPM_REPO_SCHEME = https commands = python3 -m id_tools test render-config {tox_root}/tests/configs/Npm.yaml.j2 {env_dir}/configs/Npm.yaml docker buildx build --platform linux/amd64 --add-host=host.docker.internal:host-gateway --build-context generated-config={env_dir}/configs -f {tox_root}/tests/dockerfiles/Npm.Dockerfile -t id-configurator-npm . docker rmi id-configurator-npm +[testenv:module-npm-http] +package = skip +labels = end-to-end +change_dir = {tox_root} +dependency_groups = dev +docker = + npm-repo +allowlist_externals = + docker +description = + Same as module-npm, but against a plain-HTTP registry. +set_env = + NPM_REPO_SCHEME = http +commands = + python3 -m id_tools test render-config {tox_root}/tests/configs/Npm.yaml.j2 {env_dir}/configs/Npm.yaml + docker buildx build --platform linux/amd64 --add-host=host.docker.internal:host-gateway --build-context generated-config={env_dir}/configs -f {tox_root}/tests/dockerfiles/Npm.Dockerfile -t id-configurator-npm-http . + docker rmi id-configurator-npm-http + [docker:cargo-repo] dockerfile = {tox_root}/tests/dockerfiles/repos/cargo-repo.Dockerfile expose = CARGO_REPO_PORT=8443/tcp + CARGO_REPO_HTTP_PORT=8080/tcp healthcheck_cmd = wget --no-check-certificate -qO- https://127.0.0.1:8443/config.json || exit 1 healthcheck_interval = 1 healthcheck_retries = 30 @@ -314,15 +471,36 @@ docker = cargo-repo allowlist_externals = docker +set_env = + CARGO_REPO_SCHEME = https commands = python3 -m id_tools test render-config {tox_root}/tests/configs/Rust.yaml.j2 {env_dir}/configs/Rust.yaml docker buildx build --platform linux/amd64 --add-host=host.docker.internal:host-gateway --build-context generated-config={env_dir}/configs -f {tox_root}/tests/dockerfiles/Rust.Dockerfile -t id-configurator-rust . docker rmi id-configurator-rust +[testenv:module-rust-http] +package = skip +labels = end-to-end +change_dir = {tox_root} +dependency_groups = dev +docker = + cargo-repo +allowlist_externals = + docker +description = + Same as module-rust, but against a plain-HTTP sparse registry. +set_env = + CARGO_REPO_SCHEME = http +commands = + python3 -m id_tools test render-config {tox_root}/tests/configs/Rust.yaml.j2 {env_dir}/configs/Rust.yaml + docker buildx build --platform linux/amd64 --add-host=host.docker.internal:host-gateway --build-context generated-config={env_dir}/configs -f {tox_root}/tests/dockerfiles/Rust.Dockerfile -t id-configurator-rust-http . + docker rmi id-configurator-rust-http + [docker:go-repo] dockerfile = {tox_root}/tests/dockerfiles/repos/go-repo.Dockerfile expose = GO_REPO_PORT=8443/tcp + GO_REPO_HTTP_PORT=8080/tcp healthcheck_cmd = wget --no-check-certificate -qO- https://127.0.0.1:8443/example.invalid/idconfiguratortestpkg/@v/list || exit 1 healthcheck_interval = 1 healthcheck_retries = 30 @@ -338,15 +516,37 @@ docker = go-repo allowlist_externals = docker +set_env = + GO_REPO_SCHEME = https commands = python3 -m id_tools test render-config {tox_root}/tests/configs/Go.yaml.j2 {env_dir}/configs/Go.yaml docker buildx build --platform linux/amd64 --add-host=host.docker.internal:host-gateway --build-context generated-config={env_dir}/configs -f {tox_root}/tests/dockerfiles/Go.Dockerfile -t id-configurator-go . docker rmi id-configurator-go +[testenv:module-go-http] +package = skip +labels = end-to-end +change_dir = {tox_root} +dependency_groups = dev +docker = + go-repo +allowlist_externals = + docker +description = + Same as module-go, but against a plain-HTTP module proxy, to verify + GOINSECURE is configured correctly for a non-HTTPS GOPROXY. +set_env = + GO_REPO_SCHEME = http +commands = + python3 -m id_tools test render-config {tox_root}/tests/configs/Go.yaml.j2 {env_dir}/configs/Go.yaml + docker buildx build --platform linux/amd64 --add-host=host.docker.internal:host-gateway --build-context generated-config={env_dir}/configs -f {tox_root}/tests/dockerfiles/Go.Dockerfile -t id-configurator-go-http . + docker rmi id-configurator-go-http + [docker:ruby-repo] dockerfile = {tox_root}/tests/dockerfiles/repos/ruby-repo.Dockerfile expose = RUBY_REPO_PORT=8443/tcp + RUBY_REPO_HTTP_PORT=8080/tcp healthcheck_cmd = wget --no-check-certificate -qO- https://127.0.0.1:8443/versions || exit 1 healthcheck_interval = 1 healthcheck_retries = 30 @@ -362,15 +562,36 @@ docker = ruby-repo allowlist_externals = docker +set_env = + RUBY_REPO_SCHEME = https commands = python3 -m id_tools test render-config {tox_root}/tests/configs/Ruby.yaml.j2 {env_dir}/configs/Ruby.yaml docker buildx build --platform linux/amd64 --add-host=host.docker.internal:host-gateway --build-context generated-config={env_dir}/configs -f {tox_root}/tests/dockerfiles/Ruby.Dockerfile -t id-configurator-ruby . docker rmi id-configurator-ruby +[testenv:module-ruby-http] +package = skip +labels = end-to-end +change_dir = {tox_root} +dependency_groups = dev +docker = + ruby-repo +allowlist_externals = + docker +description = + Same as module-ruby, but against a plain-HTTP gem source. +set_env = + RUBY_REPO_SCHEME = http +commands = + python3 -m id_tools test render-config {tox_root}/tests/configs/Ruby.yaml.j2 {env_dir}/configs/Ruby.yaml + docker buildx build --platform linux/amd64 --add-host=host.docker.internal:host-gateway --build-context generated-config={env_dir}/configs -f {tox_root}/tests/dockerfiles/Ruby.Dockerfile -t id-configurator-ruby-http . + docker rmi id-configurator-ruby-http + [docker:nuget-repo] dockerfile = {tox_root}/tests/dockerfiles/repos/nuget-repo.Dockerfile expose = NUGET_REPO_PORT=8443/tcp + NUGET_REPO_HTTP_PORT=8080/tcp healthcheck_cmd = wget --no-check-certificate -qO- https://127.0.0.1:8443/index.json || exit 1 healthcheck_interval = 1 healthcheck_retries = 30 @@ -386,11 +607,32 @@ docker = nuget-repo allowlist_externals = docker +set_env = + NUGET_REPO_SCHEME = https commands = python3 -m id_tools test render-config {tox_root}/tests/configs/NuGet.yaml.j2 {env_dir}/configs/NuGet.yaml docker buildx build --platform linux/amd64 --add-host=host.docker.internal:host-gateway --build-context generated-config={env_dir}/configs -f {tox_root}/tests/dockerfiles/NuGet.Dockerfile -t id-configurator-nuget . docker rmi id-configurator-nuget +[testenv:module-nuget-http] +package = skip +labels = end-to-end +change_dir = {tox_root} +dependency_groups = dev +docker = + nuget-repo +allowlist_externals = + docker +description = + Same as module-nuget, but against a plain-HTTP feed, to verify + allowInsecureConnections is configured correctly for a non-HTTPS source. +set_env = + NUGET_REPO_SCHEME = http +commands = + python3 -m id_tools test render-config {tox_root}/tests/configs/NuGet.yaml.j2 {env_dir}/configs/NuGet.yaml + docker buildx build --platform linux/amd64 --add-host=host.docker.internal:host-gateway --build-context generated-config={env_dir}/configs -f {tox_root}/tests/dockerfiles/NuGet.Dockerfile -t id-configurator-nuget-http . + docker rmi id-configurator-nuget-http + [docker:config-url-repo] dockerfile = {tox_root}/tests/dockerfiles/repos/config-url-repo.Dockerfile expose = diff --git a/uv.lock b/uv.lock index 12d6339..2dbdc9b 100644 --- a/uv.lock +++ b/uv.lock @@ -424,7 +424,7 @@ dependencies = [ ] name = "id-configurator" source = {editable = "."} -version = "2026.10.5" +version = "2026.10.6" [package.dev-dependencies] dev = [ From ebea3071bcf36c45bab0a0961a78064fa5b04a83 Mon Sep 17 00:00:00 2001 From: d9pouces Date: Tue, 6 Oct 2026 23:16:42 +0200 Subject: [PATCH 3/3] feat: add plain-HTTP listeners for various repository configurations to support non-HTTPS access Signed-off-by: d9pouces --- id_configurator/modules/java.py | 37 ++++++++++++++++++------------- tests/dockerfiles/Java.Dockerfile | 14 +++++++++++- tests/test_java.py | 33 ++++++++++++++------------- tox.ini | 8 ++++--- 4 files changed, 57 insertions(+), 35 deletions(-) diff --git a/id_configurator/modules/java.py b/id_configurator/modules/java.py index 711e1eb..4299eb8 100644 --- a/id_configurator/modules/java.py +++ b/id_configurator/modules/java.py @@ -28,12 +28,20 @@ tool targets Docker images, which are built and run as root: - Ivy: `~/.ivy2/ivysettings.xml`, as a chain of `ibiblio` resolvers. - SBT: `~/.sbt/repositories`, the resolver override file SBT reads for its own bootstrapping and, when combined with the `sbt.override.build.repos` - system property, for every project's resolvers as well. + system property, for every project's resolvers as well. A repository + whose URL isn't HTTPS gets `, allowInsecureProtocol` appended to its + line - SBT's Coursier-backed resolver otherwise refuses it outright + ("insecure protocol is unsupported"), unlike every other entry in + `~/.sbt/repositories` which takes no such suffix. - Leiningen: `~/.lein/profiles.clj`, adding the repositories to the `:user` profile. Leiningen, alone among these, refuses a plain-HTTP repository - outright ("Tried to use insecure HTTP repository without TLS"); a - repository whose URL isn't HTTPS gets `:insecure true` added to its entry, - Leiningen's own documented escape hatch for this. + outright ("Tried to use insecure HTTP repository without TLS") - its + standalone `lein` launcher (`leiningen.core.main/-main`) unconditionally + registers an aborting wagon for the "http" scheme before even reading the + project/profile config, so there is no project.clj/profiles.clj setting + (there is no `:insecure` key anywhere in Leiningen's own source) that + can work around this: a non-HTTPS repository configured here will not be + usable from Leiningen specifically, regardless of what this module does. """ from __future__ import annotations @@ -237,27 +245,26 @@ def _write_ivy_settings(context: Context, repositories: list[JavaRepository]) -> logger.info("configured ivy to use %d repositories", len(repositories)) +def _sbt_repository_line(repository: JavaRepository) -> str: + line = f" {repository.name}: {repository.url}" + if urlparse(repository.url).scheme.lower() != "https": + line += ", allowInsecureProtocol" + return line + + def _write_sbt_repositories( context: Context, repositories: list[JavaRepository] ) -> None: lines = ["[repositories]", " local"] - lines.extend( - f" {repository.name}: {repository.url}" for repository in repositories - ) + lines.extend(_sbt_repository_line(repository) for repository in repositories) context.write_file(_SBT_REPOSITORIES, ("\n".join(lines) + "\n").encode("utf-8")) logger.info("configured sbt to use %d repositories", len(repositories)) -def _lein_repository_entry(repository: JavaRepository) -> str: - options = f":url {_string_literal(repository.url)}" - if urlparse(repository.url).scheme.lower() != "https": - options += " :insecure true" - return f"[{_string_literal(repository.name)} {{{options}}}]" - - def _write_lein_profiles(context: Context, repositories: list[JavaRepository]) -> None: entries = " ".join( - _lein_repository_entry(repository) for repository in repositories + f"[{_string_literal(repository.name)} {{:url {_string_literal(repository.url)}}}]" + for repository in repositories ) content = f"{{:user {{:repositories [{entries}]}}}}\n" context.write_file(_LEIN_PROFILES, content.encode("utf-8")) diff --git a/tests/dockerfiles/Java.Dockerfile b/tests/dockerfiles/Java.Dockerfile index e301f90..3d29e48 100644 --- a/tests/dockerfiles/Java.Dockerfile +++ b/tests/dockerfiles/Java.Dockerfile @@ -4,6 +4,7 @@ COPY --from=target / /src/ COPY --from=generated-config Java.yaml /config.yaml RUN id-configurator apply --config /config.yaml --src /src --dst /dst FROM target AS runtime +ARG JAVA_REPO_SCHEME # Every build tool the `java` module configures: Maven itself (already part # of this image), Gradle, Ivy and Leiningen (all packaged for this Ubuntu # base), and SBT (no package available, fetched as a plain release tarball). @@ -43,12 +44,23 @@ RUN java -jar /usr/share/java/ivy.jar -settings /root/.ivy2/ivysettings.xml \ -retrieve "/tmp/ivyretrieve/[artifact]-[revision].[ext]" \ && test -f /tmp/ivyretrieve/idconfiguratortestpkg-0.0.1.jar +# Leiningen's standalone launcher unconditionally refuses any plain-HTTP +# repository ("Tried to use insecure HTTP repository without TLS") before +# even reading project.clj/profiles.clj - there is no `:insecure` key or +# other configuration escape hatch for this (confirmed from Leiningen's own +# source: leiningen.core.main/-main registers an aborting wagon for the +# "http" scheme unconditionally). So this check is skipped for the +# module-java-http variant: no id-configurator config could make it pass. RUN mkdir -p /tmp/leintest && cd /tmp/leintest \ && cat > project.clj <<'EOF' (defproject leintest "0.1.0" :dependencies [[org.interdiode/idconfiguratortestpkg "0.0.1"]]) EOF -RUN cd /tmp/leintest && lein deps :tree | grep -q idconfiguratortestpkg +RUN if [ "$JAVA_REPO_SCHEME" = "https" ]; then \ + cd /tmp/leintest && lein deps :tree | grep -q idconfiguratortestpkg; \ + else \ + echo "skipping leiningen: it cannot use a non-HTTPS repository"; \ + fi RUN mkdir -p /tmp/sbttest/project \ && cat > /tmp/sbttest/build.sbt <<'EOF' diff --git a/tests/test_java.py b/tests/test_java.py index de5d416..c4569ae 100644 --- a/tests/test_java.py +++ b/tests/test_java.py @@ -339,21 +339,7 @@ def test_apply_writes_sbt_repositories(tmp_path) -> None: ) -def test_apply_writes_lein_profiles(tmp_path) -> None: - dst = tmp_path / "dst" - context = Context(config=make_config(TWO_REPOS_CONFIG), dst=dst, src=None) - - JavaModule().apply(context) - - profiles = (dst / "root/.lein/profiles.clj").read_text(encoding="utf-8") - assert profiles == ( - "{:user {:repositories " - '[["interdiode" {:url "https://example.invalid/maven"}] ' - '["interdiode-extra" {:url "https://example2.invalid/maven"}]]}}\n' - ) - - -def test_apply_marks_non_https_lein_repository_as_insecure(tmp_path) -> None: +def test_apply_marks_non_https_sbt_repository_as_insecure(tmp_path) -> None: data = { "uuid": "12345678-1234-5678-1234-567812345678", "organisation": "InterDiode", @@ -370,10 +356,25 @@ def test_apply_marks_non_https_lein_repository_as_insecure(tmp_path) -> None: JavaModule().apply(context) + repositories = (dst / "root/.sbt/repositories").read_text(encoding="utf-8") + assert repositories == ( + "[repositories]\n" + " local\n" + " interdiode: http://example.invalid/maven, allowInsecureProtocol\n" + ) + + +def test_apply_writes_lein_profiles(tmp_path) -> None: + dst = tmp_path / "dst" + context = Context(config=make_config(TWO_REPOS_CONFIG), dst=dst, src=None) + + JavaModule().apply(context) + profiles = (dst / "root/.lein/profiles.clj").read_text(encoding="utf-8") assert profiles == ( "{:user {:repositories " - '[["interdiode" {:url "http://example.invalid/maven" :insecure true}]]}}\n' + '[["interdiode" {:url "https://example.invalid/maven"}] ' + '["interdiode-extra" {:url "https://example2.invalid/maven"}]]}}\n' ) diff --git a/tox.ini b/tox.ini index 2f50d16..6fb7a32 100644 --- a/tox.ini +++ b/tox.ini @@ -310,7 +310,7 @@ set_env = JAVA_REPO_SCHEME = https commands = python3 -m id_tools test render-config {tox_root}/tests/configs/Java.yaml.j2 {env_dir}/configs/Java.yaml - docker buildx build --platform linux/amd64 --add-host=host.docker.internal:host-gateway --build-context generated-config={env_dir}/configs -f {tox_root}/tests/dockerfiles/Java.Dockerfile -t id-configurator-java . + docker buildx build --platform linux/amd64 --add-host=host.docker.internal:host-gateway --build-arg JAVA_REPO_SCHEME={env:JAVA_REPO_SCHEME} --build-context generated-config={env_dir}/configs -f {tox_root}/tests/dockerfiles/Java.Dockerfile -t id-configurator-java . docker rmi id-configurator-java [testenv:module-java-http] @@ -323,12 +323,14 @@ docker = allowlist_externals = docker description = - Same as module-java, but against a plain-HTTP repository. + Same as module-java, but against a plain-HTTP repository. Leiningen is + skipped here (see Java.Dockerfile): it refuses any non-HTTPS repository + unconditionally, with no configuration able to change that. set_env = JAVA_REPO_SCHEME = http commands = python3 -m id_tools test render-config {tox_root}/tests/configs/Java.yaml.j2 {env_dir}/configs/Java.yaml - docker buildx build --platform linux/amd64 --add-host=host.docker.internal:host-gateway --build-context generated-config={env_dir}/configs -f {tox_root}/tests/dockerfiles/Java.Dockerfile -t id-configurator-java-http . + docker buildx build --platform linux/amd64 --add-host=host.docker.internal:host-gateway --build-arg JAVA_REPO_SCHEME={env:JAVA_REPO_SCHEME} --build-context generated-config={env_dir}/configs -f {tox_root}/tests/dockerfiles/Java.Dockerfile -t id-configurator-java-http . docker rmi id-configurator-java-http [docker:certificate-repo]