From a8143b61b11e25ab8de7cb732c1e0a5137bc8fc2 Mon Sep 17 00:00:00 2001 From: d9pouces Date: Sun, 6 Sep 2026 21:14:18 +0200 Subject: [PATCH] feat: add support for Go, Ruby, and NuGet modules with integration testing configurations Signed-off-by: d9pouces --- .github/workflows/minimal_testing.yml | 2 +- id_configurator/config.py | 37 +++++++++ id_configurator/modules/__init__.py | 6 ++ id_configurator/modules/go.py | 50 +++++++++++ id_configurator/modules/nuget.py | 66 +++++++++++++++ id_configurator/modules/ruby.py | 52 ++++++++++++ pyproject.toml | 2 +- tests/configs/Apk.yaml.j2 | 6 ++ tests/configs/Apt.yaml.j2 | 6 ++ tests/configs/Certificate.yaml.j2 | 6 ++ tests/configs/Go.yaml.j2 | 34 ++++++++ tests/configs/Java.yaml.j2 | 6 ++ tests/configs/Npm.yaml.j2 | 6 ++ tests/configs/NuGet.yaml.j2 | 34 ++++++++ tests/configs/Php.yaml.j2 | 6 ++ tests/configs/Python.yaml.j2 | 6 ++ tests/configs/Ruby.yaml.j2 | 34 ++++++++ tests/configs/Rust.yaml.j2 | 6 ++ tests/configs/Yum.yaml.j2 | 6 ++ tests/dockerfiles/Go.Dockerfile | 11 +++ tests/dockerfiles/NuGet.Dockerfile | 11 +++ tests/dockerfiles/Ruby.Dockerfile | 10 +++ .../repos/IdConfiguratorTestPkg.csproj | 12 +++ tests/dockerfiles/repos/build_test_gem.sh | 32 +++++++ .../dockerfiles/repos/build_test_go_module.py | 47 +++++++++++ .../repos/build_test_nuget_feed.py | 61 ++++++++++++++ tests/dockerfiles/repos/go-repo.Dockerfile | 17 ++++ tests/dockerfiles/repos/go-repo.nginx.conf | 10 +++ tests/dockerfiles/repos/nuget-repo.Dockerfile | 23 +++++ tests/dockerfiles/repos/nuget-repo.nginx.conf | 22 +++++ tests/dockerfiles/repos/ruby-repo.Dockerfile | 17 ++++ tests/dockerfiles/repos/ruby-repo.nginx.conf | 10 +++ tests/test_cli.py | 3 + tests/test_go.py | 67 +++++++++++++++ tests/test_nuget.py | 82 ++++++++++++++++++ tests/test_ruby.py | 67 +++++++++++++++ tox.ini | 83 ++++++++++++++++++- 37 files changed, 953 insertions(+), 3 deletions(-) create mode 100644 id_configurator/modules/go.py create mode 100644 id_configurator/modules/nuget.py create mode 100644 id_configurator/modules/ruby.py create mode 100644 tests/configs/Go.yaml.j2 create mode 100644 tests/configs/NuGet.yaml.j2 create mode 100644 tests/configs/Ruby.yaml.j2 create mode 100644 tests/dockerfiles/Go.Dockerfile create mode 100644 tests/dockerfiles/NuGet.Dockerfile create mode 100644 tests/dockerfiles/Ruby.Dockerfile create mode 100644 tests/dockerfiles/repos/IdConfiguratorTestPkg.csproj create mode 100644 tests/dockerfiles/repos/build_test_gem.sh create mode 100644 tests/dockerfiles/repos/build_test_go_module.py create mode 100644 tests/dockerfiles/repos/build_test_nuget_feed.py create mode 100644 tests/dockerfiles/repos/go-repo.Dockerfile create mode 100644 tests/dockerfiles/repos/go-repo.nginx.conf create mode 100644 tests/dockerfiles/repos/nuget-repo.Dockerfile create mode 100644 tests/dockerfiles/repos/nuget-repo.nginx.conf create mode 100644 tests/dockerfiles/repos/ruby-repo.Dockerfile create mode 100644 tests/dockerfiles/repos/ruby-repo.nginx.conf create mode 100644 tests/test_go.py create mode 100644 tests/test_nuget.py create mode 100644 tests/test_ruby.py diff --git a/.github/workflows/minimal_testing.yml b/.github/workflows/minimal_testing.yml index 1f027c9..d5cb64c 100644 --- a/.github/workflows/minimal_testing.yml +++ b/.github/workflows/minimal_testing.yml @@ -20,7 +20,7 @@ jobs: python-version: 3.${{ matrix.python-version }} - run: python3 -m pip install --upgrade tox tox-docker-id tox-uv - run: tox -e packaging-amd64,py313,py314,coverage - - run: tox -e module-apk,module-apt,module-certificates,module-java,module-npm,module-php,module-python,module-rust,module-yum + - run: tox -m end-to-end strategy: matrix: python-version: ['14'] diff --git a/id_configurator/config.py b/id_configurator/config.py index 092be60..3681240 100644 --- a/id_configurator/config.py +++ b/id_configurator/config.py @@ -188,6 +188,40 @@ class RustConfig(BaseModel): repositories: list[RustRepository] = Field(default_factory=list) +class GoRepository(BaseModel): + """A Go module proxy (see GOPROXY), served over the standard Go module + proxy protocol.""" + + name: str + url: str + + +class GoConfig(BaseModel): + repositories: list[GoRepository] = Field(default_factory=list) + + +class RubyRepository(BaseModel): + """A RubyGems source (see the ":sources:" key in gemrc).""" + + name: str + url: str + + +class RubyConfig(BaseModel): + repositories: list[RubyRepository] = Field(default_factory=list) + + +class NuGetRepository(BaseModel): + """A NuGet package source (see the "packageSources" key in NuGet.Config).""" + + name: str + url: str + + +class NuGetConfig(BaseModel): + repositories: list[NuGetRepository] = Field(default_factory=list) + + class ConfigurationsBlock(BaseModel): certificates: list[NamedContent] = Field(default_factory=list) apt: AptConfig = Field(default_factory=AptConfig) @@ -198,6 +232,9 @@ class ConfigurationsBlock(BaseModel): php: PhpConfig = Field(default_factory=PhpConfig) npm: NpmConfig = Field(default_factory=NpmConfig) rust: RustConfig = Field(default_factory=RustConfig) + go: GoConfig = Field(default_factory=GoConfig) + ruby: RubyConfig = Field(default_factory=RubyConfig) + nuget: NuGetConfig = Field(default_factory=NuGetConfig) class Configuration(BaseModel): diff --git a/id_configurator/modules/__init__.py b/id_configurator/modules/__init__.py index e5061ae..db0560a 100644 --- a/id_configurator/modules/__init__.py +++ b/id_configurator/modules/__init__.py @@ -7,10 +7,13 @@ from id_configurator.modules import ( apk, apt, certificates, + go, java, npm, + nuget, php, python, + ruby, rust, yum, ) @@ -19,10 +22,13 @@ __all__ = [ "apk", "apt", "certificates", + "go", "java", "npm", + "nuget", "php", "python", + "ruby", "rust", "yum", ] diff --git a/id_configurator/modules/go.py b/id_configurator/modules/go.py new file mode 100644 index 0000000..6239dbc --- /dev/null +++ b/id_configurator/modules/go.py @@ -0,0 +1,50 @@ +"""Configure the Go toolchain to use the configured GoRepository entries as +its module proxy (GOPROXY), replacing the default (proxy.golang.org). Since +an air-gapped mirror won't be listed in the public checksum database and +sum.golang.org is unreachable anyway, GOSUMDB is also turned off - each +module's hash is then simply trusted on first use, exactly as it would be +for any other GOPROXY. + +Go's own HTTPS client already trusts whatever the `certificates` module +installs into the system CA store, so no separate certificate configuration +is needed here, unlike npm's. + +Written to `$GOROOT/go.env` - not the per-user `go env -w` file (found at +`os.UserConfigDir()/go/env`, e.g. `$HOME/.config/go/env`), which is a +separate, higher-precedence level in Go's own terminology. `$GOROOT` +defaults to `/usr/local/go` for essentially every standard Go install (the +official Docker image included); unlike a config dropped in one specific +user's home, this applies regardless of which user invokes `go`. The go +command reads `$GOROOT/go.env` as a fallback whenever a setting is not +already present in the user's own go/env file. +""" + +from __future__ import annotations + +import logging + +from id_configurator.config import GoRepository +from id_configurator.context import Context +from id_configurator.modules.base import Module, register + +logger = logging.getLogger(__name__) + +_GO_ENV = "usr/local/go/go.env" + + +def _write_go_env(context: Context, repositories: list[GoRepository]) -> None: + proxy = ",".join(repository.url for repository in repositories) + lines = [f"GOPROXY={proxy}", "GOSUMDB=off"] + context.write_file(_GO_ENV, ("\n".join(lines) + "\n").encode("utf-8")) + logger.info("configured go to use %d module prox(y/ies)", len(repositories)) + + +@register +class GoModule(Module): + name = "go" + + def is_relevant(self, context: Context) -> bool: + return bool(context.config.configurations.go.repositories) + + def apply(self, context: Context) -> None: + _write_go_env(context, context.config.configurations.go.repositories) diff --git a/id_configurator/modules/nuget.py b/id_configurator/modules/nuget.py new file mode 100644 index 0000000..45a7e4a --- /dev/null +++ b/id_configurator/modules/nuget.py @@ -0,0 +1,66 @@ +"""Configure NuGet to use the configured NuGetRepository entries as its +package sources, replacing the default (nuget.org). + +NuGet's own HTTPS client (.NET's HttpClient, backed by OpenSSL on Linux) +already trusts whatever the `certificates` module installs into the system +CA store, so no separate certificate configuration is needed here, unlike +npm's. + +Microsoft documents a "Computer"-wide config file for this purpose, +`/etc/opt/NuGet/Config` on Linux (or `$NUGET_COMMON_APPLICATION_DATA/NuGet/Config` +should that variable be set) - but the .NET SDK does not actually read it in +practice (confirmed empirically; a known, still-open upstream gap, see +dotnet/sdk#12674), so it would silently have no effect. This instead writes +the "user" level config, `~/.nuget/NuGet/NuGet.Config` (the location the +.NET CLI itself uses; Mono/NuGet.exe use `~/.config/NuGet/NuGet.Config` +instead). This tool targets Docker images, which are built and run as root, +so `~` is assumed to be `/root`, exactly like a fresh dotnet install does +when not overridden - the same tradeoff the `php` module's Composer config +already accepts. A `` element is included so the configured +sources fully replace nuget.org rather than being merged alongside it, +matching every other package-manager module's behaviour. +""" + +from __future__ import annotations + +import logging +import xml.etree.ElementTree as ET + +from id_configurator.config import NuGetRepository +from id_configurator.context import Context +from id_configurator.modules.base import Module, register + +logger = logging.getLogger(__name__) + +_NUGET_CONFIG = "root/.nuget/NuGet/NuGet.Config" + + +def _render_xml(root: ET.Element) -> bytes: + ET.indent(root) + content = '\n' + ET.tostring( + root, encoding="unicode" + ) + return (content + "\n").encode("utf-8") + + +def _write_nuget_config(context: Context, repositories: list[NuGetRepository]) -> None: + configuration = ET.Element("configuration") + package_sources = ET.SubElement(configuration, "packageSources") + ET.SubElement(package_sources, "clear") + for repository in repositories: + ET.SubElement( + package_sources, "add", {"key": repository.name, "value": repository.url} + ) + context.write_file(_NUGET_CONFIG, _render_xml(configuration)) + logger.info("configured nuget to use %d package source(s)", len(repositories)) + + +@register +class NuGetModule(Module): + name = "nuget" + + def is_relevant(self, context: Context) -> bool: + return bool(context.config.configurations.nuget.repositories) + + def apply(self, context: Context) -> None: + _write_nuget_config(context, context.config.configurations.nuget.repositories) diff --git a/id_configurator/modules/ruby.py b/id_configurator/modules/ruby.py new file mode 100644 index 0000000..69fc9df --- /dev/null +++ b/id_configurator/modules/ruby.py @@ -0,0 +1,52 @@ +"""Configure RubyGems to use the configured RubyRepository entries as its +gem sources, replacing the default (rubygems.org). + +RubyGems' own HTTPS client (Net::HTTP, backed by OpenSSL) already trusts +whatever the `certificates` module installs into the system CA store, so no +separate certificate configuration is needed here, unlike npm's. + +Written to RubyGems' system-wide config file, `Gem::ConfigFile::SYSTEM_WIDE_CONFIG_FILE` +- `$sysconfdir/gemrc`, where `$sysconfdir` is Ruby's own configured + sysconfdir, not necessarily `/etc`. The official Ruby Docker image builds +Ruby with `--prefix=/usr/local`, giving `/usr/local/etc/gemrc` (confirmed +empirically, since Debian's own `sysconfdir` convention of `/etc` does not +apply to a from-source build). Also written to the more conventional +`/etc/gemrc`, for any Ruby built with the Debian/`sysconfdir=/etc` +convention instead (e.g. a distro-packaged `ruby` rather than the official +image) - harmless to write in both places, since only whichever one a given +Ruby build actually treats as its sysconfdir is ever read. Unlike a config +dropped in one specific user's home (`~/.gemrc`, RubyGems' separate "per +user" level), this applies regardless of which user invokes `gem`. +""" + +from __future__ import annotations + +import logging + +from id_configurator.config import RubyRepository +from id_configurator.context import Context +from id_configurator.modules.base import Module, register + +logger = logging.getLogger(__name__) + +_GEMRC_PATHS = ("usr/local/etc/gemrc", "etc/gemrc") + + +def _write_gemrc(context: Context, repositories: list[RubyRepository]) -> None: + lines = [":sources:"] + lines.extend(f"- {repository.url}" for repository in repositories) + content = ("\n".join(lines) + "\n").encode("utf-8") + for path in _GEMRC_PATHS: + context.write_file(path, content) + logger.info("configured gem to use %d source(s)", len(repositories)) + + +@register +class RubyModule(Module): + name = "ruby" + + def is_relevant(self, context: Context) -> bool: + return bool(context.config.configurations.ruby.repositories) + + def apply(self, context: Context) -> None: + _write_gemrc(context, context.config.configurations.ruby.repositories) diff --git a/pyproject.toml b/pyproject.toml index b47ae5e..b751be0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,7 +45,7 @@ maintainers = [ name = "id-configurator" readme = "README.md" requires-python = ">=3.13" -version = "0.1.0" +version = "0.2.0" [project.scripts] id-configurator = "id_configurator:main" diff --git a/tests/configs/Apk.yaml.j2 b/tests/configs/Apk.yaml.j2 index 3b48d9e..9cb96ef 100644 --- a/tests/configs/Apk.yaml.j2 +++ b/tests/configs/Apk.yaml.j2 @@ -27,3 +27,9 @@ configurations: repositories: [] rust: repositories: [] + go: + repositories: [] + ruby: + repositories: [] + nuget: + repositories: [] diff --git a/tests/configs/Apt.yaml.j2 b/tests/configs/Apt.yaml.j2 index 129ba8e..7642da6 100644 --- a/tests/configs/Apt.yaml.j2 +++ b/tests/configs/Apt.yaml.j2 @@ -30,3 +30,9 @@ configurations: repositories: [] rust: repositories: [] + go: + repositories: [] + ruby: + repositories: [] + nuget: + repositories: [] diff --git a/tests/configs/Certificate.yaml.j2 b/tests/configs/Certificate.yaml.j2 index 5905148..cac01ea 100644 --- a/tests/configs/Certificate.yaml.j2 +++ b/tests/configs/Certificate.yaml.j2 @@ -24,3 +24,9 @@ configurations: repositories: [] rust: repositories: [] + go: + repositories: [] + ruby: + repositories: [] + nuget: + repositories: [] diff --git a/tests/configs/Go.yaml.j2 b/tests/configs/Go.yaml.j2 new file mode 100644 index 0000000..8fff407 --- /dev/null +++ b/tests/configs/Go.yaml.j2 @@ -0,0 +1,34 @@ +uuid: 3f1a2b4c-5d6e-4f70-8a91-2b3c4d5e6f70 +organisation: InterDiode +description: Go test configuration +configurations: + certificates: + - name: Test CA + content: | +{{ 'certs/test-ca.crt.pem' | file | indent(6, first=true) }} + apt: + sources: [] + keys: [] + apk: + repositories: [] + yum: + repositories: [] + keys: [] + python: + repositories: [] + java: + repositories: [] + php: + repositories: [] + npm: + repositories: [] + rust: + repositories: [] + go: + repositories: + - name: demo + url: https://host.docker.internal:{{ 'GO_REPO_PORT' | env }}/ + ruby: + repositories: [] + nuget: + repositories: [] diff --git a/tests/configs/Java.yaml.j2 b/tests/configs/Java.yaml.j2 index 4193ffa..435fff5 100644 --- a/tests/configs/Java.yaml.j2 +++ b/tests/configs/Java.yaml.j2 @@ -39,3 +39,9 @@ configurations: repositories: [] rust: repositories: [] + go: + repositories: [] + ruby: + repositories: [] + nuget: + repositories: [] diff --git a/tests/configs/Npm.yaml.j2 b/tests/configs/Npm.yaml.j2 index 758cd7e..5c34770 100644 --- a/tests/configs/Npm.yaml.j2 +++ b/tests/configs/Npm.yaml.j2 @@ -26,3 +26,9 @@ configurations: url: https://host.docker.internal:{{ 'NPM_REPO_PORT' | env }}/ rust: repositories: [] + go: + repositories: [] + ruby: + repositories: [] + nuget: + repositories: [] diff --git a/tests/configs/NuGet.yaml.j2 b/tests/configs/NuGet.yaml.j2 new file mode 100644 index 0000000..5c6cb17 --- /dev/null +++ b/tests/configs/NuGet.yaml.j2 @@ -0,0 +1,34 @@ +uuid: 5b3c4d6e-7f80-4192-8b13-4d5e6f708192 +organisation: InterDiode +description: NuGet test configuration +configurations: + certificates: + - name: Test CA + content: | +{{ 'certs/test-ca.crt.pem' | file | indent(6, first=true) }} + apt: + sources: [] + keys: [] + apk: + repositories: [] + yum: + repositories: [] + keys: [] + python: + repositories: [] + java: + repositories: [] + php: + repositories: [] + npm: + repositories: [] + rust: + repositories: [] + go: + repositories: [] + ruby: + repositories: [] + nuget: + repositories: + - name: demo + url: https://host.docker.internal:{{ 'NUGET_REPO_PORT' | env }}/index.json diff --git a/tests/configs/Php.yaml.j2 b/tests/configs/Php.yaml.j2 index a355c9d..5439afd 100644 --- a/tests/configs/Php.yaml.j2 +++ b/tests/configs/Php.yaml.j2 @@ -26,3 +26,9 @@ configurations: repositories: [] rust: repositories: [] + go: + repositories: [] + ruby: + repositories: [] + nuget: + repositories: [] diff --git a/tests/configs/Python.yaml.j2 b/tests/configs/Python.yaml.j2 index 084f482..037f742 100644 --- a/tests/configs/Python.yaml.j2 +++ b/tests/configs/Python.yaml.j2 @@ -26,3 +26,9 @@ configurations: repositories: [] rust: repositories: [] + go: + repositories: [] + ruby: + repositories: [] + nuget: + repositories: [] diff --git a/tests/configs/Ruby.yaml.j2 b/tests/configs/Ruby.yaml.j2 new file mode 100644 index 0000000..a9b5a8c --- /dev/null +++ b/tests/configs/Ruby.yaml.j2 @@ -0,0 +1,34 @@ +uuid: 4a2b3c5d-6e7f-4081-9a02-3c4d5e6f7081 +organisation: InterDiode +description: Ruby test configuration +configurations: + certificates: + - name: Test CA + content: | +{{ 'certs/test-ca.crt.pem' | file | indent(6, first=true) }} + apt: + sources: [] + keys: [] + apk: + repositories: [] + yum: + repositories: [] + keys: [] + python: + repositories: [] + java: + repositories: [] + php: + repositories: [] + npm: + repositories: [] + rust: + repositories: [] + go: + repositories: [] + ruby: + repositories: + - name: demo + url: https://host.docker.internal:{{ 'RUBY_REPO_PORT' | env }}/ + nuget: + repositories: [] diff --git a/tests/configs/Rust.yaml.j2 b/tests/configs/Rust.yaml.j2 index 47aa80a..4910f88 100644 --- a/tests/configs/Rust.yaml.j2 +++ b/tests/configs/Rust.yaml.j2 @@ -26,3 +26,9 @@ configurations: repositories: - name: demo url: https://host.docker.internal:{{ 'CARGO_REPO_PORT' | env }}/ + go: + repositories: [] + ruby: + repositories: [] + nuget: + repositories: [] diff --git a/tests/configs/Yum.yaml.j2 b/tests/configs/Yum.yaml.j2 index 3f68807..c108b93 100644 --- a/tests/configs/Yum.yaml.j2 +++ b/tests/configs/Yum.yaml.j2 @@ -26,3 +26,9 @@ configurations: repositories: [] rust: repositories: [] + go: + repositories: [] + ruby: + repositories: [] + nuget: + repositories: [] diff --git a/tests/dockerfiles/Go.Dockerfile b/tests/dockerfiles/Go.Dockerfile new file mode 100644 index 0000000..f314d08 --- /dev/null +++ b/tests/dockerfiles/Go.Dockerfile @@ -0,0 +1,11 @@ +FROM golang:1-bookworm AS target +FROM interdiode/id-configurator:latest AS configuration +COPY --from=target / /src/ +COPY --from=generated-config Go.yaml /config.yaml +RUN id-configurator apply --config /config.yaml --src /src --dst /dst +FROM target AS runtime +COPY --from=configuration /dst/ / +RUN mkdir -p /tmp/gotest && cd /tmp/gotest && go mod init gotest \ + && go get example.invalid/idconfiguratortestpkg@v0.0.1 \ + && grep -qx 'test-only module served by the id-configurator integration tests' \ + "$(find "$(go env GOMODCACHE)" -name marker.txt)" diff --git a/tests/dockerfiles/NuGet.Dockerfile b/tests/dockerfiles/NuGet.Dockerfile new file mode 100644 index 0000000..37d4538 --- /dev/null +++ b/tests/dockerfiles/NuGet.Dockerfile @@ -0,0 +1,11 @@ +FROM mcr.microsoft.com/dotnet/sdk:9.0 AS target +FROM interdiode/id-configurator:latest AS configuration +COPY --from=target / /src/ +COPY --from=generated-config NuGet.yaml /config.yaml +RUN id-configurator apply --config /config.yaml --src /src --dst /dst +FROM target AS runtime +COPY --from=configuration /dst/ / +RUN mkdir -p /tmp/nugettest && cd /tmp/nugettest && dotnet new console -o . --no-restore >/dev/null \ + && dotnet add package idconfiguratortestpkg --version 0.0.1 \ + && grep -qx 'test-only package served by the id-configurator integration tests' \ + "$(find /root/.nuget/packages/idconfiguratortestpkg -name marker.txt)" diff --git a/tests/dockerfiles/Ruby.Dockerfile b/tests/dockerfiles/Ruby.Dockerfile new file mode 100644 index 0000000..4c4be78 --- /dev/null +++ b/tests/dockerfiles/Ruby.Dockerfile @@ -0,0 +1,10 @@ +FROM ruby:3-slim AS target +FROM interdiode/id-configurator:latest AS configuration +COPY --from=target / /src/ +COPY --from=generated-config Ruby.yaml /config.yaml +RUN id-configurator apply --config /config.yaml --src /src --dst /dst +FROM target AS runtime +COPY --from=configuration /dst/ / +RUN gem install idconfiguratortestpkg --no-document \ + && grep -qx 'test-only gem served by the id-configurator integration tests' \ + "$(find "$(gem environment gemdir)" -name marker.txt)" diff --git a/tests/dockerfiles/repos/IdConfiguratorTestPkg.csproj b/tests/dockerfiles/repos/IdConfiguratorTestPkg.csproj new file mode 100644 index 0000000..0959553 --- /dev/null +++ b/tests/dockerfiles/repos/IdConfiguratorTestPkg.csproj @@ -0,0 +1,12 @@ + + + net9.0 + idconfiguratortestpkg + 0.0.1 + InterDiode + test-only package served by the id-configurator integration tests + + + + + diff --git a/tests/dockerfiles/repos/build_test_gem.sh b/tests/dockerfiles/repos/build_test_gem.sh new file mode 100644 index 0000000..aa26cdd --- /dev/null +++ b/tests/dockerfiles/repos/build_test_gem.sh @@ -0,0 +1,32 @@ +#!/bin/sh +# Build a minimal, hand-rolled RubyGems package and a full gem repository +# index for it (legacy + compact index formats), for integration-testing +# the `ruby` module of id-configurator. Uses RubyGems' own `gem build` and +# `gem generate_index` (the same tools a real gem server host would use) +# rather than hand-rolling the compact index's binary/marshal formats. +set -eu + +PACKAGE=idconfiguratortestpkg +VERSION=0.0.1 +MARKER="test-only gem served by the id-configurator integration tests" + +gem install rubygems-generate_index --no-document + +mkdir -p "/build/${PACKAGE}-${VERSION}" +cd "/build/${PACKAGE}-${VERSION}" +cat > "${PACKAGE}.gemspec" < marker.txt +gem build "${PACKAGE}.gemspec" + +mkdir -p /srv/gems +cp ./*.gem /srv/gems/ +gem generate_index -d /srv diff --git a/tests/dockerfiles/repos/build_test_go_module.py b/tests/dockerfiles/repos/build_test_go_module.py new file mode 100644 index 0000000..26027fb --- /dev/null +++ b/tests/dockerfiles/repos/build_test_go_module.py @@ -0,0 +1,47 @@ +"""Build a minimal, offline, hand-rolled Go module proxy tree, for +integration-testing the `go` module of id-configurator. + +No network access and no Go toolchain is used: the module proxy protocol +(https://go.dev/ref/mod#module-proxy) only requires a handful of static +files, none of which embed the server's own base URL (unlike Composer/ +npm/cargo), so this can be served directly by nginx with no rewriting. +""" + +from __future__ import annotations + +import io +import json +import pathlib +import zipfile + +MODULE = "example.invalid/idconfiguratortestpkg" +VERSION = "v0.0.1" +OUT_DIR = pathlib.Path("/srv") +MARKER = "test-only module served by the id-configurator integration tests\n" + +GO_MOD = f"module {MODULE}\n\ngo 1.21\n" + + +def build_zip() -> bytes: + buffer = io.BytesIO() + prefix = f"{MODULE}@{VERSION}" + with zipfile.ZipFile(buffer, "w") as archive: + archive.writestr(f"{prefix}/go.mod", GO_MOD) + archive.writestr(f"{prefix}/marker.txt", MARKER) + return buffer.getvalue() + + +def main() -> None: + module_dir = OUT_DIR / MODULE / "@v" + module_dir.mkdir(parents=True, exist_ok=True) + (module_dir / "list").write_text(f"{VERSION}\n", encoding="utf-8") + (module_dir / f"{VERSION}.info").write_text( + json.dumps({"Version": VERSION, "Time": "2024-01-01T00:00:00Z"}), + encoding="utf-8", + ) + (module_dir / f"{VERSION}.mod").write_text(GO_MOD, encoding="utf-8") + (module_dir / f"{VERSION}.zip").write_bytes(build_zip()) + + +if __name__ == "__main__": + main() diff --git a/tests/dockerfiles/repos/build_test_nuget_feed.py b/tests/dockerfiles/repos/build_test_nuget_feed.py new file mode 100644 index 0000000..f1ed843 --- /dev/null +++ b/tests/dockerfiles/repos/build_test_nuget_feed.py @@ -0,0 +1,61 @@ +"""Build a minimal NuGet v3 static feed around one real, `dotnet pack`-built +.nupkg, for integration-testing the `nuget` module of id-configurator. + +Only two resources are populated: the flat container (`PackageBaseAddress`, +used to enumerate versions and download the .nupkg directly) and the +top-level service index pointing at it. `dotnet add package`/`dotnet +restore` with an explicit, dependency-free package version only ever calls +these two - never the (considerably more complex) registration resource - +so it is intentionally not implemented here. + +The service index bakes in a __BASE_URL__ placeholder for its "@id" fields, +since the host:port this fixture is actually reached on is only known once +tox-docker starts the container, not at image-build time (see +nuget-repo.nginx.conf). The flat container's own files never embed a URL, +so they need no such rewriting. +""" + +from __future__ import annotations + +import json +import pathlib +import shutil + +PACKAGE_ID = "idconfiguratortestpkg" +VERSION = "0.0.1" +NUPKG_NAME = f"{PACKAGE_ID}.{VERSION}.nupkg" +NUPKG_SRC = pathlib.Path("/nupkg") / NUPKG_NAME +OUT_DIR = pathlib.Path("/srv") + + +def write_flat_container() -> None: + package_dir = OUT_DIR / "flatcontainer" / PACKAGE_ID + version_dir = package_dir / VERSION + version_dir.mkdir(parents=True, exist_ok=True) + (package_dir / "index.json").write_text( + json.dumps({"versions": [VERSION]}), encoding="utf-8" + ) + shutil.copy(NUPKG_SRC, version_dir / NUPKG_NAME) + + +def write_service_index() -> None: + index = { + "version": "3.0.0", + "resources": [ + { + "@id": "__BASE_URL__/flatcontainer/", + "@type": "PackageBaseAddress/3.0.0", + } + ], + } + (OUT_DIR / "index.json").write_text(json.dumps(index), encoding="utf-8") + + +def main() -> None: + OUT_DIR.mkdir(parents=True, exist_ok=True) + write_flat_container() + write_service_index() + + +if __name__ == "__main__": + main() diff --git a/tests/dockerfiles/repos/go-repo.Dockerfile b/tests/dockerfiles/repos/go-repo.Dockerfile new file mode 100644 index 0000000..4f6f504 --- /dev/null +++ b/tests/dockerfiles/repos/go-repo.Dockerfile @@ -0,0 +1,17 @@ +# Minimal Go module proxy serving one hand-built module over TLS, using the +# same Test CA as the other *-repo fixtures, used by tox-docker to exercise +# the `go` module integration test (tests/configs/Go.yaml.j2, +# tests/dockerfiles/Go.Dockerfile). +# +# Built by tox-docker, whose build context is this directory (not the repo +# root) - every COPY source below must live under tests/dockerfiles/repos/. +FROM python:3.13-slim AS packagebuild +COPY build_test_go_module.py /build_test_go_module.py +RUN python3 /build_test_go_module.py + +FROM nginx:alpine +COPY --from=packagebuild /srv /srv +COPY repo-server.crt.pem /etc/nginx/certs/repo-server.crt.pem +COPY repo-server.key.pem /etc/nginx/certs/repo-server.key.pem +COPY go-repo.nginx.conf /etc/nginx/conf.d/default.conf +EXPOSE 8443 diff --git a/tests/dockerfiles/repos/go-repo.nginx.conf b/tests/dockerfiles/repos/go-repo.nginx.conf new file mode 100644 index 0000000..aafed09 --- /dev/null +++ b/tests/dockerfiles/repos/go-repo.nginx.conf @@ -0,0 +1,10 @@ +server { + listen 8443 ssl default_server; + server_name _; + + ssl_certificate /etc/nginx/certs/repo-server.crt.pem; + ssl_certificate_key /etc/nginx/certs/repo-server.key.pem; + + root /srv; + autoindex off; +} diff --git a/tests/dockerfiles/repos/nuget-repo.Dockerfile b/tests/dockerfiles/repos/nuget-repo.Dockerfile new file mode 100644 index 0000000..1b43561 --- /dev/null +++ b/tests/dockerfiles/repos/nuget-repo.Dockerfile @@ -0,0 +1,23 @@ +# Minimal NuGet v3 static feed serving one real, dotnet-packed .nupkg over +# TLS, using the same Test CA as the other *-repo fixtures, used by +# tox-docker to exercise the `nuget` module integration test +# (tests/configs/NuGet.yaml.j2, tests/dockerfiles/NuGet.Dockerfile). +# +# Built by tox-docker, whose build context is this directory (not the repo +# root) - every COPY source below must live under tests/dockerfiles/repos/. +FROM mcr.microsoft.com/dotnet/sdk:9.0 AS packagebuild +COPY IdConfiguratorTestPkg.csproj /pkgsrc/IdConfiguratorTestPkg.csproj +RUN echo 'test-only package served by the id-configurator integration tests' > /pkgsrc/marker.txt \ + && dotnet pack /pkgsrc -c Release -o /nupkg + +FROM python:3.13-slim AS feedbuild +COPY --from=packagebuild /nupkg /nupkg +COPY build_test_nuget_feed.py /build_test_nuget_feed.py +RUN python3 /build_test_nuget_feed.py + +FROM nginx:alpine +COPY --from=feedbuild /srv /srv +COPY repo-server.crt.pem /etc/nginx/certs/repo-server.crt.pem +COPY repo-server.key.pem /etc/nginx/certs/repo-server.key.pem +COPY nuget-repo.nginx.conf /etc/nginx/conf.d/default.conf +EXPOSE 8443 diff --git a/tests/dockerfiles/repos/nuget-repo.nginx.conf b/tests/dockerfiles/repos/nuget-repo.nginx.conf new file mode 100644 index 0000000..22a6cc7 --- /dev/null +++ b/tests/dockerfiles/repos/nuget-repo.nginx.conf @@ -0,0 +1,22 @@ +server { + listen 8443 ssl default_server; + server_name _; + + ssl_certificate /etc/nginx/certs/repo-server.crt.pem; + ssl_certificate_key /etc/nginx/certs/repo-server.key.pem; + + root /srv; + autoindex off; + + # The service index bakes in a __BASE_URL__ placeholder for its "@id" + # fields, since the host:port this fixture is actually reached on is + # only known once tox-docker starts the container, not at image-build + # time. $http_host reflects exactly what the client (nuget) used to + # connect, e.g. host.docker.internal:55123. + location = /index.json { + default_type application/json; + sub_filter '__BASE_URL__' 'https://$http_host'; + sub_filter_once off; + sub_filter_types application/json; + } +} diff --git a/tests/dockerfiles/repos/ruby-repo.Dockerfile b/tests/dockerfiles/repos/ruby-repo.Dockerfile new file mode 100644 index 0000000..7c9c653 --- /dev/null +++ b/tests/dockerfiles/repos/ruby-repo.Dockerfile @@ -0,0 +1,17 @@ +# Minimal RubyGems repository serving one hand-built gem over TLS, using the +# same Test CA as the other *-repo fixtures, used by tox-docker to exercise +# the `ruby` module integration test (tests/configs/Ruby.yaml.j2, +# tests/dockerfiles/Ruby.Dockerfile). +# +# Built by tox-docker, whose build context is this directory (not the repo +# root) - every COPY source below must live under tests/dockerfiles/repos/. +FROM ruby:3-slim AS packagebuild +COPY build_test_gem.sh /build_test_gem.sh +RUN sh /build_test_gem.sh + +FROM nginx:alpine +COPY --from=packagebuild /srv /srv +COPY repo-server.crt.pem /etc/nginx/certs/repo-server.crt.pem +COPY repo-server.key.pem /etc/nginx/certs/repo-server.key.pem +COPY ruby-repo.nginx.conf /etc/nginx/conf.d/default.conf +EXPOSE 8443 diff --git a/tests/dockerfiles/repos/ruby-repo.nginx.conf b/tests/dockerfiles/repos/ruby-repo.nginx.conf new file mode 100644 index 0000000..aafed09 --- /dev/null +++ b/tests/dockerfiles/repos/ruby-repo.nginx.conf @@ -0,0 +1,10 @@ +server { + listen 8443 ssl default_server; + server_name _; + + ssl_certificate /etc/nginx/certs/repo-server.crt.pem; + ssl_certificate_key /etc/nginx/certs/repo-server.key.pem; + + root /srv; + autoindex off; +} diff --git a/tests/test_cli.py b/tests/test_cli.py index 6062db0..954d663 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -43,10 +43,13 @@ def test_list_modules() -> None: "apk", "apt", "certificates", + "go", "java", "npm", + "nuget", "php", "python", + "ruby", "rust", "yum", } diff --git a/tests/test_go.py b/tests/test_go.py new file mode 100644 index 0000000..045f81c --- /dev/null +++ b/tests/test_go.py @@ -0,0 +1,67 @@ +from id_configurator.config import Configuration +from id_configurator.context import Context +from id_configurator.modules.go import GoModule + +CONFIG_WITH_REPO = { + "uuid": "12345678-1234-5678-1234-567812345678", + "organisation": "InterDiode", + "configurations": { + "go": {"repositories": [{"name": "demo", "url": "https://example.invalid/"}]} + }, +} + +CONFIG_WITH_TWO_REPOS = { + "uuid": "12345678-1234-5678-1234-567812345678", + "organisation": "InterDiode", + "configurations": { + "go": { + "repositories": [ + {"name": "demo", "url": "https://example.invalid/"}, + {"name": "other", "url": "https://other.invalid/"}, + ] + } + }, +} + +EMPTY_CONFIG = { + "uuid": "12345678-1234-5678-1234-567812345678", + "organisation": "InterDiode", +} + + +def make_config(data: dict) -> Configuration: + return Configuration.model_validate(data) + + +def test_is_relevant_false_when_no_repositories(tmp_path) -> None: + context = Context(config=make_config(EMPTY_CONFIG), dst=tmp_path / "dst", src=None) + assert GoModule().is_relevant(context) is False + + +def test_is_relevant_true_with_repository(tmp_path) -> None: + context = Context( + config=make_config(CONFIG_WITH_REPO), dst=tmp_path / "dst", src=None + ) + assert GoModule().is_relevant(context) is True + + +def test_apply_writes_goproxy(tmp_path) -> None: + dst = tmp_path / "dst" + context = Context(config=make_config(CONFIG_WITH_REPO), dst=dst, src=None) + + GoModule().apply(context) + + go_env = (dst / "usr/local/go/go.env").read_text(encoding="utf-8") + assert go_env == "GOPROXY=https://example.invalid/\nGOSUMDB=off\n" + + +def test_apply_joins_multiple_repositories_with_comma(tmp_path) -> None: + dst = tmp_path / "dst" + context = Context(config=make_config(CONFIG_WITH_TWO_REPOS), dst=dst, src=None) + + GoModule().apply(context) + + go_env = (dst / "usr/local/go/go.env").read_text(encoding="utf-8") + assert go_env == ( + "GOPROXY=https://example.invalid/,https://other.invalid/\nGOSUMDB=off\n" + ) diff --git a/tests/test_nuget.py b/tests/test_nuget.py new file mode 100644 index 0000000..b74a621 --- /dev/null +++ b/tests/test_nuget.py @@ -0,0 +1,82 @@ +from id_configurator.config import Configuration +from id_configurator.context import Context +from id_configurator.modules.nuget import NuGetModule + +CONFIG_WITH_REPO = { + "uuid": "12345678-1234-5678-1234-567812345678", + "organisation": "InterDiode", + "configurations": { + "nuget": {"repositories": [{"name": "demo", "url": "https://example.invalid/"}]} + }, +} + +CONFIG_WITH_TWO_REPOS = { + "uuid": "12345678-1234-5678-1234-567812345678", + "organisation": "InterDiode", + "configurations": { + "nuget": { + "repositories": [ + {"name": "demo", "url": "https://example.invalid/"}, + {"name": "other", "url": "https://other.invalid/"}, + ] + } + }, +} + +EMPTY_CONFIG = { + "uuid": "12345678-1234-5678-1234-567812345678", + "organisation": "InterDiode", +} + + +def make_config(data: dict) -> Configuration: + return Configuration.model_validate(data) + + +def test_is_relevant_false_when_no_repositories(tmp_path) -> None: + context = Context(config=make_config(EMPTY_CONFIG), dst=tmp_path / "dst", src=None) + assert NuGetModule().is_relevant(context) is False + + +def test_is_relevant_true_with_repository(tmp_path) -> None: + context = Context( + config=make_config(CONFIG_WITH_REPO), dst=tmp_path / "dst", src=None + ) + assert NuGetModule().is_relevant(context) is True + + +def test_apply_writes_nuget_config(tmp_path) -> None: + dst = tmp_path / "dst" + context = Context(config=make_config(CONFIG_WITH_REPO), dst=dst, src=None) + + NuGetModule().apply(context) + + content = (dst / "root/.nuget/NuGet/NuGet.Config").read_text(encoding="utf-8") + assert content == ( + '\n' + "\n" + " \n" + " \n" + ' \n' + " \n" + "\n" + ) + + +def test_apply_writes_multiple_sources_in_order(tmp_path) -> None: + dst = tmp_path / "dst" + context = Context(config=make_config(CONFIG_WITH_TWO_REPOS), dst=dst, src=None) + + NuGetModule().apply(context) + + content = (dst / "root/.nuget/NuGet/NuGet.Config").read_text(encoding="utf-8") + assert content == ( + '\n' + "\n" + " \n" + " \n" + ' \n' + ' \n' + " \n" + "\n" + ) diff --git a/tests/test_ruby.py b/tests/test_ruby.py new file mode 100644 index 0000000..d2e64aa --- /dev/null +++ b/tests/test_ruby.py @@ -0,0 +1,67 @@ +from id_configurator.config import Configuration +from id_configurator.context import Context +from id_configurator.modules.ruby import RubyModule + +CONFIG_WITH_REPO = { + "uuid": "12345678-1234-5678-1234-567812345678", + "organisation": "InterDiode", + "configurations": { + "ruby": {"repositories": [{"name": "demo", "url": "https://example.invalid/"}]} + }, +} + +CONFIG_WITH_TWO_REPOS = { + "uuid": "12345678-1234-5678-1234-567812345678", + "organisation": "InterDiode", + "configurations": { + "ruby": { + "repositories": [ + {"name": "demo", "url": "https://example.invalid/"}, + {"name": "other", "url": "https://other.invalid/"}, + ] + } + }, +} + +EMPTY_CONFIG = { + "uuid": "12345678-1234-5678-1234-567812345678", + "organisation": "InterDiode", +} + + +def make_config(data: dict) -> Configuration: + return Configuration.model_validate(data) + + +def test_is_relevant_false_when_no_repositories(tmp_path) -> None: + context = Context(config=make_config(EMPTY_CONFIG), dst=tmp_path / "dst", src=None) + assert RubyModule().is_relevant(context) is False + + +def test_is_relevant_true_with_repository(tmp_path) -> None: + context = Context( + config=make_config(CONFIG_WITH_REPO), dst=tmp_path / "dst", src=None + ) + assert RubyModule().is_relevant(context) is True + + +def test_apply_writes_gemrc(tmp_path) -> None: + dst = tmp_path / "dst" + context = Context(config=make_config(CONFIG_WITH_REPO), dst=dst, src=None) + + RubyModule().apply(context) + + expected = ":sources:\n- https://example.invalid/\n" + assert (dst / "usr/local/etc/gemrc").read_text(encoding="utf-8") == expected + assert (dst / "etc/gemrc").read_text(encoding="utf-8") == expected + + +def test_apply_writes_multiple_sources_in_order(tmp_path) -> None: + dst = tmp_path / "dst" + context = Context(config=make_config(CONFIG_WITH_TWO_REPOS), dst=dst, src=None) + + RubyModule().apply(context) + + expected = ":sources:\n- https://example.invalid/\n- https://other.invalid/\n" + assert (dst / "usr/local/etc/gemrc").read_text(encoding="utf-8") == expected + assert (dst / "etc/gemrc").read_text(encoding="utf-8") == expected diff --git a/tox.ini b/tox.ini index 4872b62..5187797 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -env_list = py313,py314,coverage,packaging,packaging-amd64,packaging-arm64,publish,module-python,module-apt,module-yum,module-apk,module-java,module-certificates,module-php,module-npm,module-rust +env_list = py313,py314,coverage,packaging,packaging-amd64,packaging-arm64,publish,module-python,module-apt,module-yum,module-apk,module-java,module-certificates,module-php,module-npm,module-rust,module-go,module-ruby,module-nuget isolated_build = true requires = tox>=4 @@ -96,6 +96,7 @@ healthcheck_start_period = 1 [testenv:module-python] package = skip +labels = end-to-end change_dir = {tox_root} dependency_groups = dev docker = @@ -121,6 +122,7 @@ healthcheck_start_period = 1 [testenv:module-apt] package = skip +labels = end-to-end change_dir = {tox_root} dependency_groups = dev docker = @@ -144,6 +146,7 @@ healthcheck_start_period = 1 [testenv:module-yum] package = skip +labels = end-to-end change_dir = {tox_root} dependency_groups = dev docker = @@ -167,6 +170,7 @@ healthcheck_start_period = 1 [testenv:module-apk] package = skip +labels = end-to-end change_dir = {tox_root} dependency_groups = dev docker = @@ -199,6 +203,7 @@ healthcheck_start_period = 1 [testenv:module-java] package = skip +labels = end-to-end change_dir = {tox_root} dependency_groups = dev docker = @@ -222,6 +227,7 @@ healthcheck_start_period = 1 [testenv:module-certificates] package = skip +labels = end-to-end change_dir = {tox_root} dependency_groups = dev docker = @@ -250,6 +256,7 @@ healthcheck_start_period = 1 [testenv:module-php] package = skip +labels = end-to-end change_dir = {tox_root} dependency_groups = dev docker = @@ -273,6 +280,7 @@ healthcheck_start_period = 1 [testenv:module-npm] package = skip +labels = end-to-end change_dir = {tox_root} dependency_groups = dev docker = @@ -296,6 +304,7 @@ healthcheck_start_period = 1 [testenv:module-rust] package = skip +labels = end-to-end change_dir = {tox_root} dependency_groups = dev docker = @@ -306,3 +315,75 @@ commands = python3 -m id_tools test render-config {tox_root}/tests/configs/Rust.yaml.j2 {env_dir}/configs/Rust.yaml docker buildx build --platform linux/amd64 --add-host=host.docker.internal:host-gateway --build-context generated-config={env_dir}/configs -f {tox_root}/tests/dockerfiles/Rust.Dockerfile -t id-configurator-rust . docker rmi id-configurator-rust + +[docker:go-repo] +dockerfile = {tox_root}/tests/dockerfiles/repos/go-repo.Dockerfile +expose = + GO_REPO_PORT=8443/tcp +healthcheck_cmd = wget --no-check-certificate -qO- https://127.0.0.1:8443/example.invalid/idconfiguratortestpkg/@v/list || exit 1 +healthcheck_interval = 1 +healthcheck_retries = 30 +healthcheck_timeout = 2 +healthcheck_start_period = 1 + +[testenv:module-go] +package = skip +labels = end-to-end +change_dir = {tox_root} +dependency_groups = dev +docker = + go-repo +allowlist_externals = + docker +commands = + python3 -m id_tools test render-config {tox_root}/tests/configs/Go.yaml.j2 {env_dir}/configs/Go.yaml + docker buildx build --platform linux/amd64 --add-host=host.docker.internal:host-gateway --build-context generated-config={env_dir}/configs -f {tox_root}/tests/dockerfiles/Go.Dockerfile -t id-configurator-go . + docker rmi id-configurator-go + +[docker:ruby-repo] +dockerfile = {tox_root}/tests/dockerfiles/repos/ruby-repo.Dockerfile +expose = + RUBY_REPO_PORT=8443/tcp +healthcheck_cmd = wget --no-check-certificate -qO- https://127.0.0.1:8443/versions || exit 1 +healthcheck_interval = 1 +healthcheck_retries = 30 +healthcheck_timeout = 2 +healthcheck_start_period = 1 + +[testenv:module-ruby] +package = skip +labels = end-to-end +change_dir = {tox_root} +dependency_groups = dev +docker = + ruby-repo +allowlist_externals = + docker +commands = + python3 -m id_tools test render-config {tox_root}/tests/configs/Ruby.yaml.j2 {env_dir}/configs/Ruby.yaml + docker buildx build --platform linux/amd64 --add-host=host.docker.internal:host-gateway --build-context generated-config={env_dir}/configs -f {tox_root}/tests/dockerfiles/Ruby.Dockerfile -t id-configurator-ruby . + docker rmi id-configurator-ruby + +[docker:nuget-repo] +dockerfile = {tox_root}/tests/dockerfiles/repos/nuget-repo.Dockerfile +expose = + NUGET_REPO_PORT=8443/tcp +healthcheck_cmd = wget --no-check-certificate -qO- https://127.0.0.1:8443/index.json || exit 1 +healthcheck_interval = 1 +healthcheck_retries = 30 +healthcheck_timeout = 2 +healthcheck_start_period = 1 + +[testenv:module-nuget] +package = skip +labels = end-to-end +change_dir = {tox_root} +dependency_groups = dev +docker = + nuget-repo +allowlist_externals = + docker +commands = + python3 -m id_tools test render-config {tox_root}/tests/configs/NuGet.yaml.j2 {env_dir}/configs/NuGet.yaml + docker buildx build --platform linux/amd64 --add-host=host.docker.internal:host-gateway --build-context generated-config={env_dir}/configs -f {tox_root}/tests/dockerfiles/NuGet.Dockerfile -t id-configurator-nuget . + docker rmi id-configurator-nuget