From 2c2448fcf42f5479d9c004def0e72001a316c6f4 Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Sat, 15 Aug 2026 22:41:51 -0500 Subject: [PATCH 01/42] feat: add isolated tester host installer --- docs/OPERATOR-WORKFLOWS.md | 2 +- docs/README.md | 1 + docs/TESTER-HOST.md | 110 ++++++++ host/systemd/ci-fleet-tester-cleanup.service | 20 ++ host/systemd/ci-fleet-tester-cleanup.timer | 12 + host/systemd/ci-fleet-tester-health.service | 20 ++ host/systemd/ci-fleet-tester-health.timer | 12 + scripts/fixtures/fake-tester-docker.sh | 41 +++ scripts/install-tester.sh | 174 +++++++++++++ scripts/test-install-tester.sh | 104 ++++++++ scripts/tester-runtime.sh | 258 +++++++++++++++++++ scripts/validate.sh | 1 + 12 files changed, 754 insertions(+), 1 deletion(-) create mode 100644 docs/TESTER-HOST.md create mode 100644 host/systemd/ci-fleet-tester-cleanup.service create mode 100644 host/systemd/ci-fleet-tester-cleanup.timer create mode 100644 host/systemd/ci-fleet-tester-health.service create mode 100644 host/systemd/ci-fleet-tester-health.timer create mode 100755 scripts/fixtures/fake-tester-docker.sh create mode 100755 scripts/install-tester.sh create mode 100755 scripts/test-install-tester.sh create mode 100755 scripts/tester-runtime.sh diff --git a/docs/OPERATOR-WORKFLOWS.md b/docs/OPERATOR-WORKFLOWS.md index f37f144a..4477df79 100644 --- a/docs/OPERATOR-WORKFLOWS.md +++ b/docs/OPERATOR-WORKFLOWS.md @@ -19,7 +19,7 @@ Use this page to choose an outcome, confirm the responsible role, and either fol | Authorize and onboard an application project | Current, experimental | Application repository, private configuration, and GitHub runner-group policy | Yes; no host mutation | Follow [adding a project](ADDING-A-PROJECT.md), then the [migration procedure](MIGRATING-EXISTING-CI.md). | | Create controller GitHub credentials and runner-group prerequisites | Current manual procedure | GitHub web UI and the target host | Yes | Follow [GitHub App setup](GITHUB-APP-SETUP.md) and the [runner-group procedure](LIVE-PILOT.md#2-create-the-organization-runner-group). Automated target-host bootstrap remains [planned](https://github.com/RandomDevelopment/ci-fleet/issues/27). | | Run an isolated first-job proof | Prototype only | GitHub Actions and one isolated worker host | Yes, transiently | Follow the [live pilot](LIVE-PILOT.md). Stop if the matching job queue is not proven empty. | -| Install a persistent test/staging environment host | Planned | Separate test host | N/A | No supported command exists. Stop at [issue #23](https://github.com/RandomDevelopment/ci-fleet/issues/23); do not adapt the worker installer. | +| Install a persistent test/staging environment host | In review | Separate test host | Yes; isolated-host proof remains | Follow [test-environment host](TESTER-HOST.md) and draft issue [#23](https://github.com/RandomDevelopment/ci-fleet/issues/23). Do not install until its prepared-host acceptance evidence is complete. | | Install a deployment host | In review | Separate deployment host | N/A on the default branch | [Issue #22](https://github.com/RandomDevelopment/ci-fleet/issues/22) and [PR #69](https://github.com/RandomDevelopment/ci-fleet/pull/69) track the installer and its required real-host evidence. Do not install it from the default branch yet. | | Deploy to production | Not production-ready | Separate production boundary | N/A | Stop. The project status and production evidence do not authorize this workflow. | | Publish or update from the standalone configuration template | Planned release path | Management workstation and a private configuration repository | Yes | Use the template vendored in the exact reviewed engine commit today. A standalone immutable release/compatibility signal remains [open](https://github.com/RandomDevelopment/ci-fleet-config-template/issues/12). | diff --git a/docs/README.md b/docs/README.md index 90b04996..2d54991c 100644 --- a/docs/README.md +++ b/docs/README.md @@ -15,6 +15,7 @@ New operator? Use [Operator workflows](OPERATOR-WORKFLOWS.md) to select a suppor | Decide whether it fits my infrastructure | [Architecture](ARCHITECTURE.md) and the root [README](../README.md) | | Try the experimental implementation safely | [Live pilot runbook](LIVE-PILOT.md) | | Add another Docker host, VM, computer, or VPS | [Adding a host](ADDING-A-HOST.md) | +| Install an isolated deployed-application test host | [Test-environment host](TESTER-HOST.md) | | Manage controller configuration from Git | [Git-authored controller desired state](DESIRED-STATE.md) | | Create the controller GitHub App identity | [GitHub App setup](GITHUB-APP-SETUP.md) | | Add a private project to the shared runner pool | [Adding a project](ADDING-A-PROJECT.md) | diff --git a/docs/TESTER-HOST.md b/docs/TESTER-HOST.md new file mode 100644 index 00000000..1ef26641 --- /dev/null +++ b/docs/TESTER-HOST.md @@ -0,0 +1,110 @@ +# Test-environment host + +Status: repository implementation complete; prepared isolated-host acceptance remains required. + +This role runs persistent or expiring deployed application test environments. It is deliberately separate from ephemeral CI workers and production deployers. It cannot register ordinary CI runners or promote production releases. + +## Boundary + +A tester host accepts only: + +- a reviewed `ci-fleet` source commit for the tester service; +- root-owned host configuration and environment declarations; +- application images addressed by an immutable `sha256` digest; +- environment secrets stored below that environment's fixed host-local secret directory. + +It rejects mutable images, public port binds, host bind mounts, external/unscoped Docker resources, privileged containers, added capabilities, host namespaces, global container names, and credentials outside the environment secret boundary. Every service must be read-only, drop all capabilities, and set `no-new-privileges`. Test identity, networks, storage, routes, domains, data, and credentials must have no production authority. Host/network isolation is an external acceptance gate, not something a repository script can prove. + +## Prepare host-local configuration + +The installer supports only `/etc/ci-fleet-tester/tester.env`. Create it and all protected directories as root; never commit them: + +```bash +sudo install -d -m 0700 \ + /etc/ci-fleet-tester \ + /etc/ci-fleet-tester/environments \ + /etc/ci-fleet-tester/definitions \ + /etc/ci-fleet-tester/secrets +sudo install -m 0600 /dev/null /etc/ci-fleet-tester/tester.env +``` + +Example non-secret settings: + +```text +CI_FLEET_TESTER_DEFAULT_TTL_SECONDS=86400 +CI_FLEET_TESTER_MAX_ENVIRONMENTS=20 +CI_FLEET_TESTER_DISK_WARN_PERCENT=80 +CI_FLEET_TESTER_NETWORK_PROBE_HOST=tester-probe.invalid +CI_FLEET_TESTER_HTTPS_PROBE_URL=https://tester-probe.invalid/health +CI_FLEET_TESTER_ISOLATION_ACK=test-only-no-production-authority +``` + +Set both probe values to a test-only host whose DNS resolution and HTTPS HEAD response exercise the intended local proxy path without carrying credentials. The acknowledgement is required but is not proof: an authorized operator must still verify that the prepared host has no production identity or network authority. + +For environment `example-preview`, create root-owned mode-`0600` `/etc/ci-fleet-tester/environments/example-preview.env`: + +```text +CI_FLEET_TESTER_PROJECT=example-project +CI_FLEET_TESTER_OWNER=example-owner +CI_FLEET_TESTER_COMPOSE_FILE=/etc/ci-fleet-tester/definitions/example-preview.yaml +CI_FLEET_TESTER_EXPIRES_AT=REVIEWED_FUTURE_UNIX_TIME +CI_FLEET_TESTER_ROUTE_SERVICE=web +CI_FLEET_TESTER_ROUTE_PORT=18080 +``` + +The Compose file is root-owned mode `0644` and may contain no credential value. Each image must use `registry/path@sha256:REVIEWED_64_HEX_DIGEST`. Exactly one route is published, on loopback only, at the declared port. Compose-generated network and volume names must remain below `ci-fleet-test-_...`; explicit external names are rejected. + +If credentials are required, create `/etc/ci-fleet-tester/secrets/example-preview` as root-owned mode `0700`, put only test-scope regular files there as root-owned mode `0600`, and reference them through Compose `secrets.file`. Symlinks, external secrets, production credentials, environment-variable secret transport, and files outside that exact directory are unsupported. + +## Fresh install or repair + +Use a clean reviewed checkout at the exact commit: + +```bash +ref=$(git rev-parse 'HEAD^{commit}') +sudo ./scripts/install-tester.sh --install \ + --config /etc/ci-fleet-tester/tester.env \ + --ref "$ref" +``` + +The command fails before mutation unless it sees Debian 12 or newer, root, the local default Docker context/socket/root, Compose v2, required generic tools, protected paths, and Docker storage below 80%. It stages an immutable source release, validates it, switches the `current` symlink, installs health and expiration timers, and verifies both configuration and active environments. Repeating the same command is idempotent. + +## Environment lifecycle + +The runtime command is the one interface for create/update, inspect, reset, and removal: + +```bash +sudo /opt/ci-fleet-tester/current/scripts/tester-runtime.sh --converge --environment example-preview +sudo /opt/ci-fleet-tester/current/scripts/tester-runtime.sh --inspect --environment example-preview +sudo /opt/ci-fleet-tester/current/scripts/tester-runtime.sh --reset --environment example-preview +sudo /opt/ci-fleet-tester/current/scripts/tester-runtime.sh --remove --environment example-preview +``` + +`--converge` validates the full resolved Compose model before `up --wait`. `--reset` removes only that exact Compose project and its volumes, then recreates it from the approved definition and digest. `--remove` uses the same scoped `compose down --volumes`; no global Docker prune is used. State reports only environment/project/owner, loopback route, expiry, source revision, timestamps, and health—not Compose environment values or secret content. + +`ci-fleet-tester-health.timer` checks every five minutes. `ci-fleet-tester-cleanup.timer` checks expiration every fifteen minutes. Expired environments are removed through the same scoped path. Disposable environment data is intentionally removed on reset/expiry. Reviewed definitions and fixture sources belong outside runtime state and need ordinary configuration backups; credentials and disposable volumes are never backed up by this service. + +## Upgrade, validation, and rollback + +```bash +ref=$(git rev-parse 'HEAD^{commit}') +sudo ./scripts/install-tester.sh --upgrade --config /etc/ci-fleet-tester/tester.env --ref "$ref" +sudo ./scripts/install-tester.sh --check --config /etc/ci-fleet-tester/tester.env +sudo ./scripts/install-tester.sh --rollback --config /etc/ci-fleet-tester/tester.env +``` + +Upgrade validates the candidate before activation and restores the complete previous release if post-switch checks fail. A successful switch records only the previous complete source revision as last known good. Rollback changes the tester service release; it does not rewrite an application's immutable image digest or reset environment data. + +## Removal + +Remove every environment explicitly, verify no state remains, then uninstall: + +```bash +sudo ./scripts/install-tester.sh --uninstall --config /etc/ci-fleet-tester/tester.env +``` + +Uninstall fails while any managed environment exists. It removes only tester units and immutable service releases. Host configuration, definitions, and secrets remain for explicit operator disposition; the script never guesses which credential may be deleted. + +## External acceptance gate + +Before this draft can merge, an authorized operator must use a prepared isolated Debian Docker host with test-only identity/network/storage/domain boundaries and provide evidence for: fresh install, unchanged second install, immutable application converge, health, scoped reset, expiration cleanup, upgrade, failed-upgrade restoration, rollback, and removal. No production credentials, database, network authority, ordinary CI runner, or live production system may participate. diff --git a/host/systemd/ci-fleet-tester-cleanup.service b/host/systemd/ci-fleet-tester-cleanup.service new file mode 100644 index 00000000..ccd60125 --- /dev/null +++ b/host/systemd/ci-fleet-tester-cleanup.service @@ -0,0 +1,20 @@ +[Unit] +Description=Remove expired ci-fleet test environments +After=docker.service +Requires=docker.service +ConditionPathExists=/etc/ci-fleet-tester/tester.env + +[Service] +Type=oneshot +ExecStart=/opt/ci-fleet-tester/current/scripts/tester-runtime.sh --cleanup +User=root +Group=root +UMask=0077 +NoNewPrivileges=yes +PrivateTmp=yes +ProtectHome=yes +ProtectSystem=strict +ReadWritePaths=/var/lib/ci-fleet-tester /run/lock +RestrictAddressFamilies=AF_UNIX +LockPersonality=yes +MemoryDenyWriteExecute=yes diff --git a/host/systemd/ci-fleet-tester-cleanup.timer b/host/systemd/ci-fleet-tester-cleanup.timer new file mode 100644 index 00000000..d2053d86 --- /dev/null +++ b/host/systemd/ci-fleet-tester-cleanup.timer @@ -0,0 +1,12 @@ +[Unit] +Description=Expire abandoned ci-fleet test environments + +[Timer] +OnBootSec=5m +OnUnitActiveSec=15m +Persistent=true +RandomizedDelaySec=1m +Unit=ci-fleet-tester-cleanup.service + +[Install] +WantedBy=timers.target diff --git a/host/systemd/ci-fleet-tester-health.service b/host/systemd/ci-fleet-tester-health.service new file mode 100644 index 00000000..92585f1f --- /dev/null +++ b/host/systemd/ci-fleet-tester-health.service @@ -0,0 +1,20 @@ +[Unit] +Description=Validate ci-fleet test environments +After=docker.service +Requires=docker.service +ConditionPathExists=/etc/ci-fleet-tester/tester.env + +[Service] +Type=oneshot +ExecStart=/opt/ci-fleet-tester/current/scripts/tester-runtime.sh --health +User=root +Group=root +UMask=0077 +NoNewPrivileges=yes +PrivateTmp=yes +ProtectHome=yes +ProtectSystem=strict +ReadWritePaths=/var/lib/ci-fleet-tester /run/lock +RestrictAddressFamilies=AF_UNIX +LockPersonality=yes +MemoryDenyWriteExecute=yes diff --git a/host/systemd/ci-fleet-tester-health.timer b/host/systemd/ci-fleet-tester-health.timer new file mode 100644 index 00000000..e9ec1ebc --- /dev/null +++ b/host/systemd/ci-fleet-tester-health.timer @@ -0,0 +1,12 @@ +[Unit] +Description=Check ci-fleet test environments every five minutes + +[Timer] +OnBootSec=2m +OnUnitActiveSec=5m +Persistent=true +RandomizedDelaySec=30s +Unit=ci-fleet-tester-health.service + +[Install] +WantedBy=timers.target diff --git a/scripts/fixtures/fake-tester-docker.sh b/scripts/fixtures/fake-tester-docker.sh new file mode 100755 index 00000000..e3d47df4 --- /dev/null +++ b/scripts/fixtures/fake-tester-docker.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +set -Eeuo pipefail +printf '%s\n' "$*" >>"${FAKE_TESTER_DOCKER_LOG:?}" +if [[ $1 == context && $2 == show ]]; then printf 'default\n'; exit 0; fi +if [[ $1 == info ]]; then printf '%s\n' "${FAKE_TESTER_DOCKER_ROOT:?}"; exit 0; fi +if [[ $1 == ps ]]; then printf 'fixture-container-id\n'; exit 0; fi +if [[ $1 == inspect ]]; then printf '1024\n'; exit 0; fi +if [[ $1 == volume && $2 == ls ]]; then printf 'fixture-volume\n'; exit 0; fi +if [[ $1 == volume && $2 == inspect ]]; then printf '%s\n' "${FAKE_TESTER_VOLUME_ROOT:?}"; exit 0; fi +if [[ $1 == compose && $2 == version ]]; then printf 'Docker Compose version v2.fixture\n'; exit 0; fi +if [[ $1 != compose ]]; then exit 2; fi +shift +project= +while (($#)); do + case $1 in + -p) project=$2; shift 2 ;; + -f) shift 2 ;; + config|up|down|ps) operation=$1; shift; break ;; + *) shift ;; + esac +done +case ${operation:-} in + config) + digest=$(printf 'a%.0s' {1..64}) + privileged=false; read_only=true; host_ip=127.0.0.1; image="registry.example/example/app@sha256:$digest"; network_name="${project}_default"; secrets='{}' + case ${FAKE_TESTER_POLICY:-valid} in + mutable) image=registry.example/example/app:latest ;; + privileged) privileged=true ;; + bind) volume='{"type":"bind","source":"/","target":"/host"}' ;; + broad-port) host_ip=0.0.0.0 ;; + external-network) network_name=shared ;; + valid-secret|outside-secret) secrets=$(printf '{"credential":{"file":"%s"}}' "${FAKE_TESTER_SECRET_FILE:?}") ;; + esac + volume=${volume:-'{"type":"volume","source":"data","target":"/data"}'} + printf '{"services":{"web":{"image":"%s","privileged":%s,"read_only":%s,"cap_drop":["ALL"],"security_opt":["no-new-privileges:true"],"volumes":[%s],"ports":[{"host_ip":"%s","published":%s,"target":8080,"protocol":"tcp"}]}},"networks":{"default":{"name":"%s"}},"volumes":{"data":{"name":"%s_data"}},"secrets":%s}\n' \ + "$image" "$privileged" "$read_only" "$volume" "$host_ip" "${FAKE_TESTER_ROUTE_PORT:-18080}" "$network_name" "$project" "$secrets" + ;; + up|down) ;; + ps) [[ ${FAKE_TESTER_UNHEALTHY:-0} == 1 ]] || printf 'fixture-container-id\n' ;; + *) exit 2 ;; +esac diff --git a/scripts/install-tester.sh b/scripts/install-tester.sh new file mode 100755 index 00000000..f7a2276c --- /dev/null +++ b/scripts/install-tester.sh @@ -0,0 +1,174 @@ +#!/usr/bin/env bash +set -Eeuo pipefail +umask 077 + +die() { printf 'ERROR: %s\n' "$*" >&2; exit 1; } +report() { printf '%s\n' "$*"; } +usage() { printf 'Usage: install-tester.sh {--check|--install|--upgrade|--rollback|--uninstall|--reset} [--ref COMMIT] [--config /etc/ci-fleet-tester/tester.env] [--environment ID]\n'; } + +action=; ref=; config=/etc/ci-fleet-tester/tester.env; environment= +while (($#)); do + case $1 in + --check|--install|--upgrade|--rollback|--uninstall|--reset) [[ -z $action ]] || die 'choose one action'; action=$1; shift ;; + --ref) (($# >= 2)) || die '--ref requires a value'; ref=$2; shift 2 ;; + --config) (($# >= 2)) || die '--config requires a value'; config=$2; shift 2 ;; + --environment) (($# >= 2)) || die '--environment requires a value'; environment=$2; shift 2 ;; + -h|--help) usage; exit 0 ;; + *) usage; die "unknown argument: $1" ;; + esac +done +[[ -n $action ]] || { usage; exit 2; } +[[ $config == /etc/ci-fleet-tester/tester.env ]] || die 'only the fixed tester configuration path is supported' +case $action in + --install|--upgrade) [[ $ref =~ ^[0-9a-f]{40}$ ]] || die '--ref must be an immutable 40-character commit'; [[ -z $environment ]] || die '--environment is not valid for this action' ;; + --reset) [[ $environment =~ ^[a-z0-9][a-z0-9-]{0,62}$ ]] || die '--reset requires --environment'; [[ -z $ref ]] || die '--ref is not valid for reset' ;; + *) [[ -z $ref && -z $environment ]] || die '--ref/--environment is not valid for this action' ;; +esac + +root_prefix=${CI_FLEET_ROOT_PREFIX:-} +[[ -z $root_prefix || ${CI_FLEET_TESTING:-0} == 1 ]] || die 'CI_FLEET_ROOT_PREFIX is test-only' +root_path() { printf '%s%s' "$root_prefix" "$1"; } +expected_uid=0 +[[ ${CI_FLEET_TESTING:-0} != 1 ]] || expected_uid=$(id -u) +if [[ ${CI_FLEET_TESTING:-0} != 1 && ${EUID:-$(id -u)} -ne 0 ]]; then die 'run installer as root'; fi +for command in awk bash chmod cmp curl date df dirname docker du find flock getent git grep install ln mktemp mv python3 readlink rm shellcheck stat systemctl tar wc; do command -v "$command" >/dev/null || die "required command is unavailable: $command"; done + +repo_root=$(git -C "$(dirname "${BASH_SOURCE[0]}")/.." rev-parse --show-toplevel 2>/dev/null) || die 'installer must run from a Git checkout' +opt_dir=$(root_path /opt/ci-fleet-tester) +release_dir=$opt_dir/releases +current_link=$opt_dir/current +state_root=$(root_path /var/lib/ci-fleet-tester) +lkg_file=$state_root/last-known-good +systemd_dir=$(root_path /etc/systemd/system) +config_root=$(root_path /etc/ci-fleet-tester) +environment_dir=$config_root/environments +definition_dir=$config_root/definitions +secret_root=$config_root/secrets +runtime_state=$state_root/environments +docker_root=$(root_path /var/lib/docker) +docker_socket=$(root_path /var/run/docker.sock) +units=(ci-fleet-tester-health.service ci-fleet-tester-health.timer ci-fleet-tester-cleanup.service ci-fleet-tester-cleanup.timer) +timers=(ci-fleet-tester-health.timer ci-fleet-tester-cleanup.timer) + +secure_file() { [[ -f $1 && ! -L $1 && $(stat -c %u "$1") == "$expected_uid" && $(stat -c %a "$1") == "$2" ]] || die "protected file is unsafe: $1"; } +secure_dir() { [[ -d $1 && ! -L $1 && $(stat -c %u "$1") == "$expected_uid" && $(stat -c %a "$1") == "$2" ]] || die "protected directory is unsafe: $1"; } + +host_preflight() { + local os_release docker_context actual_root used + os_release=$(root_path /etc/os-release) + [[ -f $os_release ]] || die 'supported Debian os-release is missing' + # shellcheck disable=SC1090 + . "$os_release" + [[ ${ID:-} == debian && ${VERSION_ID:-} =~ ^[0-9]+$ && ${VERSION_ID%%.*} -ge 12 ]] || die 'tester hosts require Debian 12 or newer' + docker_context=$(docker context show); [[ $docker_context == default ]] || die 'tester requires the local default Docker context' + [[ -S $docker_socket || ( ${CI_FLEET_TESTING:-0} == 1 && -e $docker_socket ) ]] || die 'local Docker socket is unavailable' + actual_root=$(docker info --format '{{.DockerRootDir}}'); [[ $actual_root == "$docker_root" ]] || die 'Docker root does not match the local managed root' + docker compose version >/dev/null + used=$(df -P "$docker_root" | awk 'NR==2{gsub(/%/,"",$5);print $5}') + [[ $used =~ ^[0-9]+$ && $used -private-repository 80 ]] || die 'Docker storage is at or above 80%' +} + +ensure_directories() { + local directory + for directory in "$opt_dir" "$release_dir"; do + if [[ -e $directory || -L $directory ]]; then secure_dir "$directory" 755; else install -d -m 0755 "$directory"; fi + done + for directory in "$config_root" "$environment_dir" "$definition_dir" "$secret_root" "$state_root" "$runtime_state"; do + if [[ -e $directory || -L $directory ]]; then secure_dir "$directory" 700; else install -d -m 0700 "$directory"; fi + done + install -d -m 0755 "$systemd_dir" +} + +release_complete() { + local path=$1 expected=$2 unit + [[ -d $path && ! -L $path && -x $path/scripts/tester-runtime.sh && -f $path/.ci-fleet-source-revision ]] || return 1 + [[ $(<"$path/.ci-fleet-source-revision") == "$expected" ]] || return 1 + for unit in "${units[@]}"; do [[ -f $path/host/systemd/$unit ]] || return 1; done +} + +stage_release() { + local commit=$1 target=$release_dir/$1 staging=$release_dir/.staging-$1 + if [[ -e $target ]]; then release_complete "$target" "$commit" || die 'existing tester release is incomplete'; return; fi + rm -rf -- "$staging"; install -d -m 0755 "$staging" + git -C "$repo_root" cat-file -e "$commit^{commit}" 2>/dev/null || die 'requested source commit is unavailable locally' + git -C "$repo_root" archive "$commit" scripts/tester-runtime.sh host/systemd/ci-fleet-tester-health.service host/systemd/ci-fleet-tester-health.timer host/systemd/ci-fleet-tester-cleanup.service host/systemd/ci-fleet-tester-cleanup.timer | tar -x -C "$staging" + printf '%s\n' "$commit" >"$staging/.ci-fleet-source-revision"; chmod 0644 "$staging/.ci-fleet-source-revision" + chmod 0755 "$staging/scripts/tester-runtime.sh"; shellcheck "$staging/scripts/tester-runtime.sh"; bash -n "$staging/scripts/tester-runtime.sh" + mv -T "$staging" "$target" +} + +install_units() { + local source=$1 unit + for unit in "${units[@]}"; do install -m 0644 "$source/host/systemd/$unit" "$systemd_dir/$unit"; done + systemctl daemon-reload + systemctl enable --now "${timers[@]}" >/dev/null +} + +activate_release() { + local commit=$1 target=$release_dir/$1 previous= + release_complete "$target" "$commit" || die 'candidate tester release is incomplete' + [[ ! -L $current_link ]] || previous=$(basename "$(readlink -f "$current_link")") + ln -sfn "$target" "$current_link.new"; mv -Tf "$current_link.new" "$current_link" + install_units "$target" + if ! "$target/scripts/tester-runtime.sh" --check || ! "$target/scripts/tester-runtime.sh" --health; then + if [[ $previous =~ ^[0-9a-f]{40}$ ]] && release_complete "$release_dir/$previous" "$previous"; then + ln -sfn "$release_dir/$previous" "$current_link.new"; mv -Tf "$current_link.new" "$current_link"; install_units "$release_dir/$previous" + fi + die 'candidate tester activation failed; previous release restored when available' + fi + [[ ! $previous =~ ^[0-9a-f]{40}$ || $previous == "$commit" ]] || printf '%s\n' "$previous" >"$lkg_file" + report "INSTALL_OK source_revision=$commit previous_revision=${previous:-none} config=$config" +} + +installed_revision() { + [[ -L $current_link ]] || return 1 + local target; target=$(readlink -f "$current_link") + [[ $target == "$release_dir"/* ]] || return 1 + basename "$target" +} + +case $action in + --install|--upgrade) + host_preflight; ensure_directories + secure_file "$(root_path "$config")" 600 + [[ $(git -C "$repo_root" rev-parse 'HEAD^{commit}') == "$ref" ]] || die 'reviewed checkout HEAD does not match --ref' + if ! git -C "$repo_root" diff --quiet || ! git -C "$repo_root" diff --cached --quiet; then die 'reviewed checkout has tracked changes'; fi + current=$(installed_revision || true) + [[ $action != --install || -z $current || $current == "$ref" ]] || die 'tester is already installed at another revision; use --upgrade' + [[ $action != --upgrade || -n $current ]] || die 'tester is not installed; use --install' + stage_release "$ref"; activate_release "$ref" + ;; + --check) + host_preflight; ensure_directories + secure_file "$(root_path "$config")" 600 + current=$(installed_revision) || die 'tester is not installed' + release_complete "$release_dir/$current" "$current" || die 'installed release is incomplete' + for timer in "${timers[@]}"; do + if ! systemctl is-enabled --quiet "$timer" || ! systemctl is-active --quiet "$timer"; then die "timer is inactive: $timer"; fi + done + "$current_link/scripts/tester-runtime.sh" --check + "$current_link/scripts/tester-runtime.sh" --health + report "CHECK_OK source_revision=$current" + ;; + --reset) + current=$(installed_revision) || die 'tester is not installed' + "$current_link/scripts/tester-runtime.sh" --reset --environment "$environment" + ;; + --rollback) + ensure_directories; secure_file "$lkg_file" 600 + target=$(<"$lkg_file"); [[ $target =~ ^[0-9a-f]{40}$ ]] || die 'last-known-good revision is invalid' + current=$(installed_revision || true) + activate_release "$target" + [[ ! $current =~ ^[0-9a-f]{40}$ || $current == "$target" ]] || printf '%s\n' "$current" >"$lkg_file" + report "ROLLBACK_OK source_revision=$target" + ;; + --uninstall) + ensure_directories + if find "$runtime_state" -maxdepth 1 -type f -name '*.state' | grep -q .; then die 'remove every test environment before uninstalling the tester service'; fi + systemctl disable --now "${timers[@]}" >/dev/null 2>&1 || true + for unit in "${units[@]}"; do rm -f -- "$systemd_dir/$unit"; done + systemctl daemon-reload + rm -f -- "$current_link" "$lkg_file"; rm -rf -- "$release_dir"; install -d -m 0755 "$release_dir" + report 'UNINSTALL_OK preserved_config=true preserved_definitions=true preserved_secrets=true' + ;; +esac diff --git a/scripts/test-install-tester.sh b/scripts/test-install-tester.sh new file mode 100755 index 00000000..029996ad --- /dev/null +++ b/scripts/test-install-tester.sh @@ -0,0 +1,104 @@ +#!/usr/bin/env bash +set -Eeuo pipefail +repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +runtime=$repo_root/scripts/tester-runtime.sh +installer=$repo_root/scripts/install-tester.sh +tmp=$(mktemp -d) +trap 'rm -rf "$tmp"' EXIT +fail() { printf 'FAIL %s\n' "$*" >&2; exit 1; } +root=$tmp/root +fake_bin=$tmp/bin +mkdir -p "$fake_bin" "$root/etc/ci-fleet-tester/environments" "$root/etc/ci-fleet-tester/definitions" "$root/etc/ci-fleet-tester/secrets" "$root/var/lib/ci-fleet-tester/environments" "$root/var/lib/docker" "$root/var/lib/fake-tester-volume" "$root/var/run" "$root/etc/systemd/system" +chmod 700 "$root/etc/ci-fleet-tester" "$root/etc/ci-fleet-tester/environments" "$root/etc/ci-fleet-tester/definitions" "$root/etc/ci-fleet-tester/secrets" "$root/var/lib/ci-fleet-tester" "$root/var/lib/ci-fleet-tester/environments" +printf 'ID=debian\nVERSION_ID=13\n' >"$root/etc/os-release" +: >"$root/var/run/docker.sock" +printf 'CI_FLEET_TESTER_DEFAULT_TTL_SECONDS=3600\nCI_FLEET_TESTER_MAX_ENVIRONMENTS=3\nCI_FLEET_TESTER_DISK_WARN_PERCENT=80\nCI_FLEET_TESTER_NETWORK_PROBE_HOST=tester-probe.invalid\nCI_FLEET_TESTER_HTTPS_PROBE_URL=https://tester-probe.invalid/health\nCI_FLEET_TESTER_ISOLATION_ACK=test-only-no-production-authority\n' >"$root/etc/ci-fleet-tester/tester.env" +chmod 600 "$root/etc/ci-fleet-tester/tester.env" +cp "$repo_root/scripts/fixtures/fake-tester-docker.sh" "$fake_bin/docker" +chmod 0755 "$fake_bin/docker" +cat >"$fake_bin/df" <<'EOF' +#!/usr/bin/env bash +printf 'Filesystem 1024-blocks Used Available Capacity Mounted on\nfixture 100 20 80 20%% /fixture\n' +EOF +cat >"$fake_bin/systemctl" <<'EOF' +#!/usr/bin/env bash +printf '%s\n' "$*" >>"${FAKE_TESTER_SYSTEMCTL_LOG:?}" +EOF +printf '#!/usr/bin/env bash\nexit 0\n' >"$fake_bin/curl" +printf '#!/usr/bin/env bash\nexit 0\n' >"$fake_bin/getent" +chmod 0755 "$fake_bin/df" "$fake_bin/systemctl" "$fake_bin/curl" "$fake_bin/getent" +export PATH="$fake_bin:$PATH" CI_FLEET_TESTING=1 CI_FLEET_ROOT_PREFIX=$root +export FAKE_TESTER_DOCKER_ROOT=$root/var/lib/docker FAKE_TESTER_VOLUME_ROOT=$root/var/lib/fake-tester-volume FAKE_TESTER_DOCKER_LOG=$tmp/docker.log FAKE_TESTER_SYSTEMCTL_LOG=$tmp/systemctl.log + +write_environment() { + local id=$1 port=$2 + mkdir -p "$root/etc/ci-fleet-tester/secrets/$id"; chmod 700 "$root/etc/ci-fleet-tester/secrets/$id" + printf 'services: {}\n' >"$root/etc/ci-fleet-tester/definitions/$id.yaml" + chmod 644 "$root/etc/ci-fleet-tester/definitions/$id.yaml" + printf 'CI_FLEET_TESTER_PROJECT=example-project\nCI_FLEET_TESTER_OWNER=example-owner\nCI_FLEET_TESTER_COMPOSE_FILE=%s\nCI_FLEET_TESTER_EXPIRES_AT=%s\nCI_FLEET_TESTER_ROUTE_SERVICE=web\nCI_FLEET_TESTER_ROUTE_PORT=%s\n' \ + "$root/etc/ci-fleet-tester/definitions/$id.yaml" "$(( $(date +%s) + 1800 ))" "$port" >"$root/etc/ci-fleet-tester/environments/$id.env" + chmod 600 "$root/etc/ci-fleet-tester/environments/$id.env" +} + +"$runtime" --check | grep -Fq CHECK_OK || fail 'runtime preflight failed' +if FAKE_TESTER_DOCKER_ROOT=/remote/docker "$runtime" --check >/dev/null 2>&1; then fail 'remote Docker daemon was accepted'; fi +write_environment preview-a 18080 +FAKE_TESTER_ROUTE_PORT=18080 "$runtime" --converge --environment preview-a | grep -Fq CONVERGED || fail 'converge failed' +state=$root/var/lib/ci-fleet-tester/environments/preview-a.state +[[ -f $state && $(stat -c %a "$state") == 600 ]] || fail 'state was not protected' +inspect_output=$(FAKE_TESTER_ROUTE_PORT=18080 "$runtime" --inspect --environment preview-a) +grep -q 'IMAGE_DIGESTS=sha256:[a-f0-9]\{64\}.*STATUS=running DISK_BYTES=[1-9][0-9]*' <<<"$inspect_output" || fail 'inspect did not report health and disk use' +FAKE_TESTER_ROUTE_PORT=18080 "$runtime" --converge --environment preview-a >/dev/null +[[ $(find "$root/var/lib/ci-fleet-tester/environments" -name '*.state' | wc -l) == 1 ]] || fail 'idempotent converge duplicated state' +write_environment preview-b 18080 +if FAKE_TESTER_ROUTE_PORT=18080 "$runtime" --converge --environment preview-b >/dev/null 2>&1; then fail 'duplicate route port was accepted'; fi +for policy in mutable privileged bind broad-port external-network; do + write_environment "bad-$policy" 18081 + if FAKE_TESTER_ROUTE_PORT=18081 FAKE_TESTER_POLICY=$policy "$runtime" --converge --environment "bad-$policy" >/dev/null 2>&1; then fail "unsafe compose policy was accepted: $policy"; fi +done +write_environment secret-preview 18082 +secret_file=$root/etc/ci-fleet-tester/secrets/secret-preview/credential +printf 'example-test-scope-value\n' >"$secret_file"; chmod 600 "$secret_file" +FAKE_TESTER_ROUTE_PORT=18082 FAKE_TESTER_POLICY=valid-secret FAKE_TESTER_SECRET_FILE=$secret_file "$runtime" --converge --environment secret-preview >/dev/null +FAKE_TESTER_ROUTE_PORT=18082 "$runtime" --remove --environment secret-preview >/dev/null +outside_secret=$root/etc/ci-fleet-tester/secrets/outside +printf 'example-test-scope-value\n' >"$outside_secret"; chmod 600 "$outside_secret" +write_environment outside-secret 18083 +if FAKE_TESTER_ROUTE_PORT=18083 FAKE_TESTER_POLICY=outside-secret FAKE_TESTER_SECRET_FILE=$outside_secret "$runtime" --converge --environment outside-secret >/dev/null 2>&1; then fail 'out-of-boundary secret was accepted'; fi +FAKE_TESTER_ROUTE_PORT=18080 "$runtime" --reset --environment preview-a >/dev/null +grep -q 'down --volumes --remove-orphans' "$tmp/docker.log" || fail 'reset did not remove only the scoped Compose project' +if grep -Eq 'system prune|volume prune|network prune' "$tmp/docker.log"; then fail 'global Docker prune was used'; fi +sed -i 's/^EXPIRES_AT=.*/EXPIRES_AT=1/' "$state" +"$runtime" --cleanup >/dev/null +[[ ! -e $state ]] || fail 'expired environment survived cleanup' + +# Commit-backed installer tests run after the implementation commit exists. +ref=$(git -C "$repo_root" rev-parse HEAD) +"$installer" --install --config /etc/ci-fleet-tester/tester.env --ref "$ref" | grep -Fq INSTALL_OK || fail 'fresh install failed' +[[ $(readlink -f "$root/opt/ci-fleet-tester/current") == "$root/opt/ci-fleet-tester/releases/$ref" ]] || fail 'current release link is wrong' +"$installer" --install --config /etc/ci-fleet-tester/tester.env --ref "$ref" >/dev/null +check_output=$("$installer" --check --config /etc/ci-fleet-tester/tester.env) +grep -Fq CHECK_OK <<<"$check_output" || fail 'installed check failed' +for unit in ci-fleet-tester-health.service ci-fleet-tester-health.timer ci-fleet-tester-cleanup.service ci-fleet-tester-cleanup.timer; do [[ -f $root/etc/systemd/system/$unit ]] || fail "unit missing: $unit"; done + +# A syntactically valid candidate that fails its post-switch check restores the incumbent. +upgrade_repo=$tmp/upgrade-repo +git clone --quiet --shared "$repo_root" "$upgrade_repo" +printf '#!/usr/bin/env bash\nexit 1\n' >"$upgrade_repo/scripts/tester-runtime.sh"; chmod 0755 "$upgrade_repo/scripts/tester-runtime.sh" +git -C "$upgrade_repo" add scripts/tester-runtime.sh +git -C "$upgrade_repo" -c user.name=Example -c user.email=example@invalid.example commit --quiet -m 'fixture: fail tester activation' +bad_ref=$(git -C "$upgrade_repo" rev-parse HEAD) +if "$upgrade_repo/scripts/install-tester.sh" --upgrade --config /etc/ci-fleet-tester/tester.env --ref "$bad_ref" >/dev/null 2>&1; then fail 'failed candidate activation succeeded'; fi +[[ $(readlink -f "$root/opt/ci-fleet-tester/current") == "$root/opt/ci-fleet-tester/releases/$ref" ]] || fail 'failed upgrade did not restore incumbent release' + +# Rollback switches only to a complete recorded release and keeps environments intact. +old=0000000000000000000000000000000000000000 +cp -a "$root/opt/ci-fleet-tester/releases/$ref" "$root/opt/ci-fleet-tester/releases/$old" +printf '%s\n' "$old" >"$root/opt/ci-fleet-tester/releases/$old/.ci-fleet-source-revision" +printf '%s\n' "$old" >"$root/var/lib/ci-fleet-tester/last-known-good"; chmod 600 "$root/var/lib/ci-fleet-tester/last-known-good" +"$installer" --rollback --config /etc/ci-fleet-tester/tester.env | grep -Fq ROLLBACK_OK || fail 'rollback failed' +[[ $(readlink -f "$root/opt/ci-fleet-tester/current") == "$root/opt/ci-fleet-tester/releases/$old" ]] || fail 'rollback selected the wrong release' + +"$installer" --uninstall --config /etc/ci-fleet-tester/tester.env | grep -Fq UNINSTALL_OK || fail 'uninstall failed' +[[ -f $root/etc/ci-fleet-tester/tester.env && ! -L $root/opt/ci-fleet-tester/current ]] || fail 'uninstall did not preserve config/remove runtime' +printf 'TESTER_INSTALLER_TESTS_OK\n' diff --git a/scripts/tester-runtime.sh b/scripts/tester-runtime.sh new file mode 100755 index 00000000..a1b441bd --- /dev/null +++ b/scripts/tester-runtime.sh @@ -0,0 +1,258 @@ +#!/usr/bin/env bash +set -Eeuo pipefail +umask 077 + +root_prefix=${CI_FLEET_ROOT_PREFIX:-} +[[ -z $root_prefix || ${CI_FLEET_TESTING:-0} == 1 ]] || { printf 'ERROR: CI_FLEET_ROOT_PREFIX is test-only\n' >&2; exit 1; } +root_path() { printf '%s%s' "$root_prefix" "$1"; } +script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +source_revision=unknown +[[ ! -f $script_dir/../.ci-fleet-source-revision ]] || source_revision=$(<"$script_dir/../.ci-fleet-source-revision") +config_file=$(root_path /etc/ci-fleet-tester/tester.env) +environment_dir=$(root_path /etc/ci-fleet-tester/environments) +definition_dir=$(root_path /etc/ci-fleet-tester/definitions) +secret_root=$(root_path /etc/ci-fleet-tester/secrets) +state_dir=$(root_path /var/lib/ci-fleet-tester/environments) +lock_file=$(root_path /run/lock/ci-fleet-tester.lock) +expected_uid=0 +[[ ${CI_FLEET_TESTING:-0} != 1 ]] || expected_uid=$(id -u) + +die() { printf 'ERROR: %s\n' "$*" >&2; exit 1; } +report() { printf '%s\n' "$*"; } +usage() { printf 'Usage: tester-runtime.sh {--check|--converge|--reset|--remove|--inspect|--cleanup|--health} [--environment ID]\n'; } + +action=; environment= +while (($#)); do + case $1 in + --check|--cleanup|--health) [[ -z $action ]] || die 'choose one action'; action=$1; shift ;; + --converge|--reset|--remove|--inspect) [[ -z $action ]] || die 'choose one action'; action=$1; shift ;; + --environment) (($# >= 2)) || die '--environment requires a value'; environment=$2; shift 2 ;; + -h|--help) usage; exit 0 ;; + *) usage; die "unknown argument: $1" ;; + esac +done +[[ -n $action ]] || { usage; exit 2; } +case $action in --converge|--reset|--remove|--inspect) [[ $environment =~ ^[a-z0-9][a-z0-9-]{0,62}$ ]] || die 'environment ID is invalid' ;; *) [[ -z $environment ]] || die '--environment is not valid for this action' ;; esac +for command in awk basename chmod curl date df dirname docker du find flock getent grep install mktemp mv python3 readlink rm stat wc; do command -v "$command" >/dev/null || die "required command is unavailable: $command"; done + +secure_directory() { + local path=$1 mode=$2 + [[ -d $path && ! -L $path && $(stat -c %u "$path") == "$expected_uid" && $(stat -c %a "$path") == "$mode" ]] || die "protected directory is unsafe: $path" +} +secure_file() { + local path=$1 mode=$2 + [[ -f $path && ! -L $path && $(stat -c %u "$path") == "$expected_uid" && $(stat -c %a "$path") == "$mode" ]] || die "protected file is unsafe: $path" +} +load_exact_env() { + local file=$1 allowed=$2 line key value + declare -gA ENV_VALUES=() + while IFS= read -r line || [[ -n $line ]]; do + [[ -z $line || $line == \#* ]] && continue + [[ $line =~ ^([A-Z][A-Z0-9_]*)=(.*)$ ]] || die "invalid configuration line in $file" + key=${BASH_REMATCH[1]}; value=${BASH_REMATCH[2]} + [[ " $allowed " == *" $key "* ]] || die "unsupported configuration key: $key" + [[ -z ${ENV_VALUES[$key]+x} ]] || die "duplicate configuration key: $key" + [[ $value != *$'\n'* && $value != *$'\r'* ]] || die "invalid configuration value: $key" + ENV_VALUES[$key]=$value + done <"$file" +} + +load_global() { + secure_directory "$(dirname "$config_file")" 700 + secure_directory "$environment_dir" 700 + secure_directory "$definition_dir" 700 + secure_directory "$secret_root" 700 + secure_directory "$(dirname "$state_dir")" 700 + secure_directory "$state_dir" 700 + secure_file "$config_file" 600 + load_exact_env "$config_file" 'CI_FLEET_TESTER_DEFAULT_TTL_SECONDS CI_FLEET_TESTER_MAX_ENVIRONMENTS CI_FLEET_TESTER_DISK_WARN_PERCENT CI_FLEET_TESTER_NETWORK_PROBE_HOST CI_FLEET_TESTER_HTTPS_PROBE_URL CI_FLEET_TESTER_ISOLATION_ACK' + default_ttl=${ENV_VALUES[CI_FLEET_TESTER_DEFAULT_TTL_SECONDS]:-86400} + max_environments=${ENV_VALUES[CI_FLEET_TESTER_MAX_ENVIRONMENTS]:-20} + disk_warn=${ENV_VALUES[CI_FLEET_TESTER_DISK_WARN_PERCENT]:-80} + probe_host=${ENV_VALUES[CI_FLEET_TESTER_NETWORK_PROBE_HOST]:-} + probe_url=${ENV_VALUES[CI_FLEET_TESTER_HTTPS_PROBE_URL]:-} + [[ $default_ttl =~ ^[0-9]+$ && $default_ttl -ge 300 && $default_ttl -le 604800 ]] || die 'default TTL must be 300-604800 seconds' + [[ $max_environments =~ ^[0-9]+$ && $max_environments -ge 1 && $max_environments -le 100 ]] || die 'max environments must be 1-100' + [[ $disk_warn =~ ^[0-9]+$ && $disk_warn -ge 50 && $disk_warn -le 95 ]] || die 'disk warning threshold must be 50-95' + [[ $probe_host =~ ^[A-Za-z0-9][A-Za-z0-9.-]{0,252}$ && $probe_url =~ ^https://[A-Za-z0-9][A-Za-z0-9.-]{0,252}(:[0-9]+)?/[^[:space:]]*$ ]] || die 'DNS and HTTPS probe targets are required' + [[ ${ENV_VALUES[CI_FLEET_TESTER_ISOLATION_ACK]:-} == test-only-no-production-authority ]] || die 'explicit test-only isolation acknowledgement is required' +} + +project_name() { printf 'ci-fleet-test-%s' "$1"; } +state_path() { printf '%s/%s.state' "$state_dir" "$1"; } +spec_path() { printf '%s/%s.env' "$environment_dir" "$1"; } + +load_spec() { + local id=$1 canonical + spec=$(spec_path "$id") + secure_file "$spec" 600 + load_exact_env "$spec" 'CI_FLEET_TESTER_PROJECT CI_FLEET_TESTER_OWNER CI_FLEET_TESTER_COMPOSE_FILE CI_FLEET_TESTER_EXPIRES_AT CI_FLEET_TESTER_ROUTE_SERVICE CI_FLEET_TESTER_ROUTE_PORT' + project=${ENV_VALUES[CI_FLEET_TESTER_PROJECT]:-} + owner=${ENV_VALUES[CI_FLEET_TESTER_OWNER]:-} + compose_file=${ENV_VALUES[CI_FLEET_TESTER_COMPOSE_FILE]:-} + expires_at=${ENV_VALUES[CI_FLEET_TESTER_EXPIRES_AT]:-} + route_service=${ENV_VALUES[CI_FLEET_TESTER_ROUTE_SERVICE]:-} + route_port=${ENV_VALUES[CI_FLEET_TESTER_ROUTE_PORT]:-} + [[ $project =~ ^[a-z0-9][a-z0-9-]{0,62}$ && $owner =~ ^[A-Za-z0-9][A-Za-z0-9_.@-]{0,127}$ ]] || die 'project or owner is invalid' + [[ $route_service =~ ^[a-z0-9][a-z0-9-]{0,62}$ && $route_port =~ ^[0-9]+$ && $route_port -ge 1024 && $route_port -le 65535 ]] || die 'route service/port is invalid' + if [[ -z $expires_at ]]; then expires_at=$(($(date +%s) + default_ttl)); fi + [[ $expires_at =~ ^[0-9]+$ && $expires_at -gt $(date +%s) && $expires_at -le $(($(date +%s) + 604800)) ]] || die 'expiration must be in the future and at most seven days away' + canonical=$(readlink -f -- "$compose_file") || die 'compose file is unavailable' + [[ $canonical == "$definition_dir"/* && $canonical == "$compose_file" ]] || die 'compose file must be a canonical file below the protected definitions directory' + secure_file "$compose_file" 644 + compose_project=$(project_name "$id") + secret_dir=$secret_root/$id + secure_directory "$secret_dir" 700 +} + +validate_compose() { + local rendered=$1 + docker compose -p "$compose_project" -f "$compose_file" config --format json >"$rendered" || return 1 + chmod 600 "$rendered" + python3 - "$rendered" "$route_service" "$route_port" "$compose_project" "$secret_dir" "$expected_uid" <<'PY' || return 1 +import json,os,re,stat,sys +value=json.load(open(sys.argv[1])); route_service=sys.argv[2]; route_port=int(sys.argv[3]); project=sys.argv[4]; secret_dir=sys.argv[5]; expected_uid=int(sys.argv[6]) +services=value.get('services') +if not isinstance(services,dict) or route_service not in services: raise SystemExit('route service is missing') +image=re.compile(r'^[a-z0-9.-]+(?::[0-9]+)?/[A-Za-z0-9_./-]+@sha256:[0-9a-f]{64}$') +ports=[] +for name,service in services.items(): + if not image.fullmatch(str(service.get('image',''))): raise SystemExit(f'{name}: image must use an immutable sha256 digest') + if service.get('privileged') or service.get('network_mode')=='host' or service.get('pid')=='host' or service.get('ipc')=='host': raise SystemExit(f'{name}: host/privileged access is forbidden') + if service.get('devices') or service.get('cap_add') or service.get('container_name') or service.get('hostname'): raise SystemExit(f'{name}: device/capability/global identity is forbidden') + if service.get('read_only') is not True or 'ALL' not in service.get('cap_drop',[]): raise SystemExit(f'{name}: read_only and cap_drop ALL are required') + security=' '.join(service.get('security_opt',[])) + if 'no-new-privileges' not in security: raise SystemExit(f'{name}: no-new-privileges is required') + for mount in service.get('volumes',[]): + if isinstance(mount,str) or mount.get('type') not in ('volume','tmpfs'): raise SystemExit(f'{name}: host bind mounts are forbidden') + for port in service.get('ports',[]): + if not isinstance(port,dict) or str(port.get('host_ip','')) != '127.0.0.1': raise SystemExit(f'{name}: published ports must bind loopback') + ports.append((name,int(port.get('published',0)),int(port.get('target',0)))) +if ports != [(route_service,route_port,ports[0][2] if ports else 0)] or not ports or ports[0][2] < 1: raise SystemExit('exactly one declared loopback route is required') +for section in ('networks','volumes'): + for name,item in value.get(section,{}).items(): + resolved=item.get('name',f'{project}_{name}') + if item.get('external') or not resolved.startswith(f'{project}_'): raise SystemExit(f'{section}.{name}: external/unscoped names are forbidden') +for name,item in value.get('secrets',{}).items(): + path=item.get('file') + if item.get('external') or not isinstance(path,str) or os.path.realpath(path).rsplit('/',1)[0] != secret_dir: raise SystemExit(f'secrets.{name}: secret must be a host-local file in the environment secret directory') + metadata=os.lstat(path) + if not stat.S_ISREG(metadata.st_mode) or stat.S_ISLNK(metadata.st_mode) or metadata.st_uid != expected_uid or stat.S_IMODE(metadata.st_mode) != 0o600: raise SystemExit(f'secrets.{name}: secret must be owner-controlled mode 0600') +PY + image_digests=$(python3 - "$rendered" <<'PY' +import json,sys +value=json.load(open(sys.argv[1])) +print(','.join(sorted({service['image'].rsplit('@',1)[1] for service in value['services'].values()}))) +PY +) +} + +check_port_unique() { + local file key value other_port + for file in "$state_dir"/*.state; do + [[ -e $file ]] || continue + [[ $file == "$(state_path "$environment")" ]] && continue + other_port=$(awk -F= '$1=="ROUTE_PORT"{print $2}' "$file") + [[ $other_port != "$route_port" ]] || die "loopback route port is already owned by another environment: $route_port" + done +} + +write_state() { + local target tmp + target=$(state_path "$environment"); tmp=$target.new + printf 'ENVIRONMENT=%s\nPROJECT=%s\nOWNER=%s\nCOMPOSE_FILE=%s\nROUTE_SERVICE=%s\nROUTE_PORT=%s\nEXPIRES_AT=%s\nSOURCE_REVISION=%s\nIMAGE_DIGESTS=%s\nUPDATED_AT=%s\n' \ + "$environment" "$project" "$owner" "$compose_file" "$route_service" "$route_port" "$expires_at" "$source_revision" "$image_digests" "$(date +%s)" >"$tmp" + chmod 600 "$tmp"; mv -fT "$tmp" "$target" +} + +converge() { + local rendered count + load_spec "$environment" + check_port_unique + count=$(find "$state_dir" -maxdepth 1 -type f -name '*.state' | wc -l) + [[ -f $(state_path "$environment") || $count -private-repository $max_environments ]] || die 'maximum environment count reached' + rendered=$(mktemp) + if ! validate_compose "$rendered"; then rm -f "$rendered"; die 'compose policy validation failed'; fi + rm -f "$rendered" + docker compose -p "$compose_project" -f "$compose_file" up -d --remove-orphans --wait + write_state + report "CONVERGED environment=$environment project=$project owner=$owner route=loopback:$route_port expires_at=$expires_at" +} + +remove_environment() { + local target compose id=$1 + target=$(state_path "$id") + if [[ -f $target ]]; then + secure_file "$target" 600 + compose=$(awk -F= '$1=="COMPOSE_FILE"{print substr($0,index($0,"=")+1)}' "$target") + [[ $(readlink -f -- "$compose") == "$definition_dir"/* ]] || die 'stored compose path escaped definitions directory' + docker compose -p "$(project_name "$id")" -f "$compose" down --volumes --remove-orphans + rm -f -- "$target" + fi + report "REMOVED environment=$id" +} + +inspect_environment() { + local target=$1 id compose status resource value bytes=0 mount + id=$(basename "$target" .state); secure_file "$target" 600 + while IFS='=' read -r key value; do + case $key in ENVIRONMENT|PROJECT|OWNER|ROUTE_PORT|EXPIRES_AT|SOURCE_REVISION|IMAGE_DIGESTS|UPDATED_AT) printf '%s=%s ' "$key" "$value" ;; esac + done <"$target" + compose=$(awk -F= '$1=="COMPOSE_FILE"{print substr($0,index($0,"=")+1)}' "$target") + if docker compose -p "$(project_name "$id")" -f "$compose" ps --status running -q | grep -q .; then status=running; else status=unhealthy; fi + while IFS= read -r resource; do + [[ -n $resource ]] || continue + value=$(docker inspect --size --format '{{.SizeRw}}' "$resource"); [[ $value =~ ^[0-9]+$ ]] || die 'container disk size is invalid'; bytes=$((bytes + value)) + done < <(docker ps -aq --filter "label=com.docker.compose.project=$(project_name "$id")") + while IFS= read -r resource; do + [[ -n $resource ]] || continue + mount=$(docker volume inspect --format '{{.Mountpoint}}' "$resource"); value=$(du -sb "$mount" | awk '{print $1}'); [[ $value =~ ^[0-9]+$ ]] || die 'volume disk size is invalid'; bytes=$((bytes + value)) + done < <(docker volume ls -q --filter "label=com.docker.compose.project=$(project_name "$id")") + printf 'STATUS=%s DISK_BYTES=%s\n' "$status" "$bytes" +} + +load_global +docker_socket=$(root_path /var/run/docker.sock) +if [[ ${CI_FLEET_TESTING:-0} == 1 ]]; then + [[ -f $docker_socket && ! -L $docker_socket ]] || die 'local Docker socket is unavailable' +else + [[ -S $docker_socket && ! -L $docker_socket && $(stat -c %u "$docker_socket") == 0 ]] || die 'local root-owned Docker socket is unavailable' +fi +unset DOCKER_CONTEXT +export DOCKER_HOST="unix://$docker_socket" +[[ $(docker info --format '{{.DockerRootDir}}') == "$(root_path /var/lib/docker)" ]] || die 'Docker daemon root is not the expected local path' +install -d -m 0755 "$(dirname "$lock_file")" +exec 9>"$lock_file" +flock -x 9 +case $action in + --check) + docker info --format '{{.DockerRootDir}}' >/dev/null + docker compose version >/dev/null + getent ahosts "$probe_host" >/dev/null || die 'test-host DNS probe failed' + curl --fail --silent --show-error --head --max-time 10 --output /dev/null "$probe_url" || die 'test-host HTTPS/proxy probe failed' + used=$(df -P "$(root_path /var/lib/docker)" | awk 'NR==2{gsub(/%/,"",$5);print $5}') + [[ $used =~ ^[0-9]+$ && $used -private-repository $disk_warn ]] || die 'Docker storage exceeds configured warning threshold' + report "CHECK_OK max_environments=$max_environments disk_used_percent=$used" + ;; + --converge) converge ;; + --reset) remove_environment "$environment"; converge ;; + --remove) remove_environment "$environment" ;; + --inspect) [[ -f $(state_path "$environment") ]] || die 'environment is not installed'; inspect_environment "$(state_path "$environment")" ;; + --cleanup) + now=$(date +%s) + for target in "$state_dir"/*.state; do + [[ -e $target ]] || continue + secure_file "$target" 600 + expires=$(awk -F= '$1=="EXPIRES_AT"{print $2}' "$target") + [[ $expires =~ ^[0-9]+$ ]] || die "invalid expiration in $target" + if ((expires <= now)); then remove_environment "$(basename "$target" .state)"; fi + done + report 'CLEANUP_OK' + ;; + --health) + failed=0 + for target in "$state_dir"/*.state; do [[ -e $target ]] || continue; inspect_environment "$target" | grep -q 'STATUS=running' || failed=1; done + ((failed == 0)) || die 'one or more test environments are unhealthy' + report 'HEALTH_OK' + ;; +esac diff --git a/scripts/validate.sh b/scripts/validate.sh index 9660aaa2..60afad01 100755 --- a/scripts/validate.sh +++ b/scripts/validate.sh @@ -27,6 +27,7 @@ python3 scripts/desired_state.py validate-engine-capabilities --manifest engine- python3 .github/actions/plan/plan.py --plan examples/project/scripts/ci/plan.json --group fast >/dev/null python3 .github/actions/plan/plan.py --plan examples/project/scripts/ci/plan.json --group full >/dev/null scripts/test-capacity-preflight.sh +scripts/test-install-tester.sh scripts/test-install-worker-controller.sh scripts/test-install-status-receiver.sh From 61968674de177a8379b009ec3a4b3453549d773c Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Sun, 16 Aug 2026 00:14:57 -0500 Subject: [PATCH 02/42] fix: make tester lifecycle recoverable and scoped --- scripts/fixtures/fake-tester-docker.sh | 20 ++++++--- scripts/install-tester.sh | 43 +++++++++++++------ scripts/test-install-tester.sh | 44 ++++++++++++++++++- scripts/tester-runtime.sh | 58 ++++++++++++++++++-------- 4 files changed, 129 insertions(+), 36 deletions(-) diff --git a/scripts/fixtures/fake-tester-docker.sh b/scripts/fixtures/fake-tester-docker.sh index e3d47df4..2b06e3e3 100755 --- a/scripts/fixtures/fake-tester-docker.sh +++ b/scripts/fixtures/fake-tester-docker.sh @@ -4,7 +4,10 @@ printf '%s\n' "$*" >>"${FAKE_TESTER_DOCKER_LOG:?}" if [[ $1 == context && $2 == show ]]; then printf 'default\n'; exit 0; fi if [[ $1 == info ]]; then printf '%s\n' "${FAKE_TESTER_DOCKER_ROOT:?}"; exit 0; fi if [[ $1 == ps ]]; then printf 'fixture-container-id\n'; exit 0; fi -if [[ $1 == inspect ]]; then printf '1024\n'; exit 0; fi +if [[ $1 == inspect ]]; then + if [[ " $* " == *' --size '* ]]; then printf '1024\n'; else printf '%s\n' "${FAKE_TESTER_CONTAINER_STATE:-running healthy}"; fi + exit 0 +fi if [[ $1 == volume && $2 == ls ]]; then printf 'fixture-volume\n'; exit 0; fi if [[ $1 == volume && $2 == inspect ]]; then printf '%s\n' "${FAKE_TESTER_VOLUME_ROOT:?}"; exit 0; fi if [[ $1 == compose && $2 == version ]]; then printf 'Docker Compose version v2.fixture\n'; exit 0; fi @@ -22,20 +25,27 @@ done case ${operation:-} in config) digest=$(printf 'a%.0s' {1..64}) - privileged=false; read_only=true; host_ip=127.0.0.1; image="registry.example/example/app@sha256:$digest"; network_name="${project}_default"; secrets='{}' + privileged=false; read_only=true; host_ip=127.0.0.1; image="registry.example/example/app@sha256:$digest"; network_name="${project}_default"; secrets='{}'; service_extra=; top_extra=; volume_extra=; security='no-new-privileges:true' case ${FAKE_TESTER_POLICY:-valid} in mutable) image=registry.example/example/app:latest ;; privileged) privileged=true ;; bind) volume='{"type":"bind","source":"/","target":"/host"}' ;; broad-port) host_ip=0.0.0.0 ;; external-network) network_name=shared ;; + environment) service_extra=',"environment":{"TOKEN":"example"}' ;; + configs) service_extra=',"configs":[{"source":"credential"}]'; top_extra=',"configs":{"credential":{"file":"/tmp/example"}}' ;; + use-api-socket) service_extra=',"use_api_socket":true' ;; + namespace-share) service_extra=',"network_mode":"service:other"' ;; + false-nnp) security='no-new-privileges:false' ;; + custom-volume) volume_extra=',"driver":"local"' ;; valid-secret|outside-secret) secrets=$(printf '{"credential":{"file":"%s"}}' "${FAKE_TESTER_SECRET_FILE:?}") ;; esac volume=${volume:-'{"type":"volume","source":"data","target":"/data"}'} - printf '{"services":{"web":{"image":"%s","privileged":%s,"read_only":%s,"cap_drop":["ALL"],"security_opt":["no-new-privileges:true"],"volumes":[%s],"ports":[{"host_ip":"%s","published":%s,"target":8080,"protocol":"tcp"}]}},"networks":{"default":{"name":"%s"}},"volumes":{"data":{"name":"%s_data"}},"secrets":%s}\n' \ - "$image" "$privileged" "$read_only" "$volume" "$host_ip" "${FAKE_TESTER_ROUTE_PORT:-18080}" "$network_name" "$project" "$secrets" + printf '{"services":{"web":{"image":"%s","privileged":%s,"read_only":%s,"cap_drop":["ALL"],"security_opt":["%s"],"volumes":[%s],"ports":[{"host_ip":"%s","published":%s,"target":8080,"protocol":"tcp"}]%s}},"networks":{"default":{"name":"%s"}},"volumes":{"data":{"name":"%s_data"%s}},"secrets":%s%s}\n' \ + "$image" "$privileged" "$read_only" "$security" "$volume" "$host_ip" "${FAKE_TESTER_ROUTE_PORT:-18080}" "$service_extra" "$network_name" "$project" "$volume_extra" "$secrets" "$top_extra" ;; - up|down) ;; + up) [[ ${FAKE_TESTER_UP_FAIL:-0} != 1 ]] ;; + down) [[ ${FAKE_TESTER_DOWN_FAIL:-0} != 1 ]] ;; ps) [[ ${FAKE_TESTER_UNHEALTHY:-0} == 1 ]] || printf 'fixture-container-id\n' ;; *) exit 2 ;; esac diff --git a/scripts/install-tester.sh b/scripts/install-tester.sh index f7a2276c..91d16d7d 100755 --- a/scripts/install-tester.sh +++ b/scripts/install-tester.sh @@ -31,7 +31,7 @@ root_path() { printf '%s%s' "$root_prefix" "$1"; } expected_uid=0 [[ ${CI_FLEET_TESTING:-0} != 1 ]] || expected_uid=$(id -u) if [[ ${CI_FLEET_TESTING:-0} != 1 && ${EUID:-$(id -u)} -ne 0 ]]; then die 'run installer as root'; fi -for command in awk bash chmod cmp curl date df dirname docker du find flock getent git grep install ln mktemp mv python3 readlink rm shellcheck stat systemctl tar wc; do command -v "$command" >/dev/null || die "required command is unavailable: $command"; done +for command in awk bash chmod cmp curl date df dirname docker du find flock getent git grep install ln mktemp mv python3 readlink rm sha256sum shellcheck stat systemctl tar wc; do command -v "$command" >/dev/null || die "required command is unavailable: $command"; done repo_root=$(git -C "$(dirname "${BASH_SOURCE[0]}")/.." rev-parse --show-toplevel 2>/dev/null) || die 'installer must run from a Git checkout' opt_dir=$(root_path /opt/ci-fleet-tester) @@ -60,6 +60,9 @@ host_preflight() { # shellcheck disable=SC1090 . "$os_release" [[ ${ID:-} == debian && ${VERSION_ID:-} =~ ^[0-9]+$ && ${VERSION_ID%%.*} -ge 12 ]] || die 'tester hosts require Debian 12 or newer' + [[ -z ${DOCKER_HOST:-} && -z ${DOCKER_CONTEXT:-} ]] || die 'Docker environment selectors are forbidden' + unset DOCKER_CONTEXT + export DOCKER_HOST="unix://$docker_socket" docker_context=$(docker context show); [[ $docker_context == default ]] || die 'tester requires the local default Docker context' [[ -S $docker_socket || ( ${CI_FLEET_TESTING:-0} == 1 && -e $docker_socket ) ]] || die 'local Docker socket is unavailable' actual_root=$(docker info --format '{{.DockerRootDir}}'); [[ $actual_root == "$docker_root" ]] || die 'Docker root does not match the local managed root' @@ -81,9 +84,10 @@ ensure_directories() { release_complete() { local path=$1 expected=$2 unit - [[ -d $path && ! -L $path && -x $path/scripts/tester-runtime.sh && -f $path/.ci-fleet-source-revision ]] || return 1 + [[ -d $path && ! -L $path && $(stat -c %u "$path") == "$expected_uid" && $(stat -c %a "$path") == 555 && -x $path/scripts/tester-runtime.sh && -f $path/.ci-fleet-source-revision ]] || return 1 [[ $(<"$path/.ci-fleet-source-revision") == "$expected" ]] || return 1 for unit in "${units[@]}"; do [[ -f $path/host/systemd/$unit ]] || return 1; done + (cd "$path" && sha256sum --status -c .ci-fleet-release.sha256) || return 1 } stage_release() { @@ -94,14 +98,29 @@ stage_release() { git -C "$repo_root" archive "$commit" scripts/tester-runtime.sh host/systemd/ci-fleet-tester-health.service host/systemd/ci-fleet-tester-health.timer host/systemd/ci-fleet-tester-cleanup.service host/systemd/ci-fleet-tester-cleanup.timer | tar -x -C "$staging" printf '%s\n' "$commit" >"$staging/.ci-fleet-source-revision"; chmod 0644 "$staging/.ci-fleet-source-revision" chmod 0755 "$staging/scripts/tester-runtime.sh"; shellcheck "$staging/scripts/tester-runtime.sh"; bash -n "$staging/scripts/tester-runtime.sh" + (cd "$staging" && sha256sum scripts/tester-runtime.sh .ci-fleet-source-revision host/systemd/* >.ci-fleet-release.sha256) + chmod 0444 "$staging/.ci-fleet-source-revision" "$staging/.ci-fleet-release.sha256" "$staging"/host/systemd/* + chmod 0555 "$staging" "$staging/scripts" "$staging/host" "$staging/host/systemd" "$staging/scripts/tester-runtime.sh" mv -T "$staging" "$target" } install_units() { local source=$1 unit - for unit in "${units[@]}"; do install -m 0644 "$source/host/systemd/$unit" "$systemd_dir/$unit"; done - systemctl daemon-reload - systemctl enable --now "${timers[@]}" >/dev/null + for unit in "${units[@]}"; do install -m 0644 "$source/host/systemd/$unit" "$systemd_dir/$unit" || return 1; done + systemctl daemon-reload || return 1 + systemctl enable --now "${timers[@]}" >/dev/null || return 1 +} + +remove_units() { + systemctl disable --now "${timers[@]}" >/dev/null 2>&1 || true + local unit; for unit in "${units[@]}"; do rm -f -- "$systemd_dir/$unit"; done + systemctl daemon-reload || true +} + +write_lkg() { + printf '%s\n' "$1" >"$lkg_file.new" + chmod 0600 "$lkg_file.new" + mv -fT "$lkg_file.new" "$lkg_file" } activate_release() { @@ -109,14 +128,16 @@ activate_release() { release_complete "$target" "$commit" || die 'candidate tester release is incomplete' [[ ! -L $current_link ]] || previous=$(basename "$(readlink -f "$current_link")") ln -sfn "$target" "$current_link.new"; mv -Tf "$current_link.new" "$current_link" - install_units "$target" - if ! "$target/scripts/tester-runtime.sh" --check || ! "$target/scripts/tester-runtime.sh" --health; then + if ! install_units "$target" || ! "$target/scripts/tester-runtime.sh" --check || ! "$target/scripts/tester-runtime.sh" --health; then if [[ $previous =~ ^[0-9a-f]{40}$ ]] && release_complete "$release_dir/$previous" "$previous"; then ln -sfn "$release_dir/$previous" "$current_link.new"; mv -Tf "$current_link.new" "$current_link"; install_units "$release_dir/$previous" + else + rm -f -- "$current_link" + remove_units fi die 'candidate tester activation failed; previous release restored when available' fi - [[ ! $previous =~ ^[0-9a-f]{40}$ || $previous == "$commit" ]] || printf '%s\n' "$previous" >"$lkg_file" + [[ ! $previous =~ ^[0-9a-f]{40}$ || $previous == "$commit" ]] || write_lkg "$previous" report "INSTALL_OK source_revision=$commit previous_revision=${previous:-none} config=$config" } @@ -159,15 +180,13 @@ case $action in target=$(<"$lkg_file"); [[ $target =~ ^[0-9a-f]{40}$ ]] || die 'last-known-good revision is invalid' current=$(installed_revision || true) activate_release "$target" - [[ ! $current =~ ^[0-9a-f]{40}$ || $current == "$target" ]] || printf '%s\n' "$current" >"$lkg_file" + [[ ! $current =~ ^[0-9a-f]{40}$ || $current == "$target" ]] || write_lkg "$current" report "ROLLBACK_OK source_revision=$target" ;; --uninstall) ensure_directories if find "$runtime_state" -maxdepth 1 -type f -name '*.state' | grep -q .; then die 'remove every test environment before uninstalling the tester service'; fi - systemctl disable --now "${timers[@]}" >/dev/null 2>&1 || true - for unit in "${units[@]}"; do rm -f -- "$systemd_dir/$unit"; done - systemctl daemon-reload + remove_units rm -f -- "$current_link" "$lkg_file"; rm -rf -- "$release_dir"; install -d -m 0755 "$release_dir" report 'UNINSTALL_OK preserved_config=true preserved_definitions=true preserved_secrets=true' ;; diff --git a/scripts/test-install-tester.sh b/scripts/test-install-tester.sh index 029996ad..4aaa05d3 100755 --- a/scripts/test-install-tester.sh +++ b/scripts/test-install-tester.sh @@ -23,6 +23,7 @@ EOF cat >"$fake_bin/systemctl" <<'EOF' #!/usr/bin/env bash printf '%s\n' "$*" >>"${FAKE_TESTER_SYSTEMCTL_LOG:?}" +[[ -z ${FAKE_TESTER_SYSTEMCTL_FAIL:-} || " $* " != *" $FAKE_TESTER_SYSTEMCTL_FAIL "* ]] EOF printf '#!/usr/bin/env bash\nexit 0\n' >"$fake_bin/curl" printf '#!/usr/bin/env bash\nexit 0\n' >"$fake_bin/getent" @@ -50,12 +51,20 @@ inspect_output=$(FAKE_TESTER_ROUTE_PORT=18080 "$runtime" --inspect --environment grep -q 'IMAGE_DIGESTS=sha256:[a-f0-9]\{64\}.*STATUS=running DISK_BYTES=[1-9][0-9]*' <<<"$inspect_output" || fail 'inspect did not report health and disk use' FAKE_TESTER_ROUTE_PORT=18080 "$runtime" --converge --environment preview-a >/dev/null [[ $(find "$root/var/lib/ci-fleet-tester/environments" -name '*.state' | wc -l) == 1 ]] || fail 'idempotent converge duplicated state' +original_expiry=$(awk -F= '$1=="EXPIRES_AT"{print $2}' "$state") +grep -v '^CI_FLEET_TESTER_EXPIRES_AT=' "$root/etc/ci-fleet-tester/environments/preview-a.env" >"$tmp/spec" +mv "$tmp/spec" "$root/etc/ci-fleet-tester/environments/preview-a.env"; chmod 600 "$root/etc/ci-fleet-tester/environments/preview-a.env" +FAKE_TESTER_ROUTE_PORT=18080 "$runtime" --converge --environment preview-a >/dev/null +[[ $(awk -F= '$1=="EXPIRES_AT"{print $2}' "$state") == "$original_expiry" ]] || fail 'idempotent converge extended expiration' write_environment preview-b 18080 if FAKE_TESTER_ROUTE_PORT=18080 "$runtime" --converge --environment preview-b >/dev/null 2>&1; then fail 'duplicate route port was accepted'; fi -for policy in mutable privileged bind broad-port external-network; do +for policy in mutable privileged bind broad-port external-network environment configs use-api-socket namespace-share false-nnp custom-volume; do write_environment "bad-$policy" 18081 if FAKE_TESTER_ROUTE_PORT=18081 FAKE_TESTER_POLICY=$policy "$runtime" --converge --environment "bad-$policy" >/dev/null 2>&1; then fail "unsafe compose policy was accepted: $policy"; fi done +if FAKE_TESTER_ROUTE_PORT=18080 FAKE_TESTER_POLICY=mutable "$runtime" --reset --environment preview-a >/dev/null 2>&1; then fail 'reset accepted invalid replacement'; fi +[[ -f $state ]] || fail 'reset deleted the incumbent before validation' +if FAKE_TESTER_ROUTE_PORT=18080 FAKE_TESTER_CONTAINER_STATE='exited unhealthy' "$runtime" --health >/dev/null 2>&1; then fail 'health accepted a stopped managed service'; fi write_environment secret-preview 18082 secret_file=$root/etc/ci-fleet-tester/secrets/secret-preview/credential printf 'example-test-scope-value\n' >"$secret_file"; chmod 600 "$secret_file" @@ -65,6 +74,14 @@ outside_secret=$root/etc/ci-fleet-tester/secrets/outside printf 'example-test-scope-value\n' >"$outside_secret"; chmod 600 "$outside_secret" write_environment outside-secret 18083 if FAKE_TESTER_ROUTE_PORT=18083 FAKE_TESTER_POLICY=outside-secret FAKE_TESTER_SECRET_FILE=$outside_secret "$runtime" --converge --environment outside-secret >/dev/null 2>&1; then fail 'out-of-boundary secret was accepted'; fi +write_environment partial-up 18084 +if FAKE_TESTER_ROUTE_PORT=18084 FAKE_TESTER_UP_FAIL=1 "$runtime" --converge --environment partial-up >/dev/null 2>&1; then fail 'partial activation succeeded'; fi +[[ -f $root/var/lib/ci-fleet-tester/environments/partial-up.state && -f $root/var/lib/ci-fleet-tester/environments/partial-up.compose.json ]] || fail 'partial activation was not tracked for cleanup' +FAKE_TESTER_ROUTE_PORT=18084 "$runtime" --remove --environment partial-up >/dev/null +write_environment immutable-remove 18085 +FAKE_TESTER_ROUTE_PORT=18085 "$runtime" --converge --environment immutable-remove >/dev/null +rm "$root/etc/ci-fleet-tester/definitions/immutable-remove.yaml" +FAKE_TESTER_ROUTE_PORT=18085 "$runtime" --remove --environment immutable-remove >/dev/null FAKE_TESTER_ROUTE_PORT=18080 "$runtime" --reset --environment preview-a >/dev/null grep -q 'down --volumes --remove-orphans' "$tmp/docker.log" || fail 'reset did not remove only the scoped Compose project' if grep -Eq 'system prune|volume prune|network prune' "$tmp/docker.log"; then fail 'global Docker prune was used'; fi @@ -74,12 +91,18 @@ sed -i 's/^EXPIRES_AT=.*/EXPIRES_AT=1/' "$state" # Commit-backed installer tests run after the implementation commit exists. ref=$(git -C "$repo_root" rev-parse HEAD) +if DOCKER_HOST=tcp://example.invalid:2375 "$installer" --check --config /etc/ci-fleet-tester/tester.env >/dev/null 2>&1; then fail 'installer accepted a remote Docker selector'; fi "$installer" --install --config /etc/ci-fleet-tester/tester.env --ref "$ref" | grep -Fq INSTALL_OK || fail 'fresh install failed' [[ $(readlink -f "$root/opt/ci-fleet-tester/current") == "$root/opt/ci-fleet-tester/releases/$ref" ]] || fail 'current release link is wrong' "$installer" --install --config /etc/ci-fleet-tester/tester.env --ref "$ref" >/dev/null check_output=$("$installer" --check --config /etc/ci-fleet-tester/tester.env) grep -Fq CHECK_OK <<<"$check_output" || fail 'installed check failed' for unit in ci-fleet-tester-health.service ci-fleet-tester-health.timer ci-fleet-tester-cleanup.service ci-fleet-tester-cleanup.timer; do [[ -f $root/etc/systemd/system/$unit ]] || fail "unit missing: $unit"; done +release=$root/opt/ci-fleet-tester/releases/$ref +chmod u+w "$release/scripts/tester-runtime.sh"; printf '# tamper\n' >>"$release/scripts/tester-runtime.sh"; chmod 0555 "$release/scripts/tester-runtime.sh" +if "$installer" --check --config /etc/ci-fleet-tester/tester.env >/dev/null 2>&1; then fail 'tampered installed release passed check'; fi +git -C "$repo_root" show "$ref:scripts/tester-runtime.sh" >"$release/scripts/tester-runtime.sh"; chmod 0555 "$release/scripts/tester-runtime.sh" +"$installer" --check --config /etc/ci-fleet-tester/tester.env >/dev/null || fail 'restored release failed check' # A syntactically valid candidate that fails its post-switch check restores the incumbent. upgrade_repo=$tmp/upgrade-repo @@ -91,13 +114,32 @@ bad_ref=$(git -C "$upgrade_repo" rev-parse HEAD) if "$upgrade_repo/scripts/install-tester.sh" --upgrade --config /etc/ci-fleet-tester/tester.env --ref "$bad_ref" >/dev/null 2>&1; then fail 'failed candidate activation succeeded'; fi [[ $(readlink -f "$root/opt/ci-fleet-tester/current") == "$root/opt/ci-fleet-tester/releases/$ref" ]] || fail 'failed upgrade did not restore incumbent release' +# Unit activation failures restore the incumbent symlink and units. +git -C "$upgrade_repo" show "$ref:scripts/tester-runtime.sh" >"$upgrade_repo/scripts/tester-runtime.sh"; chmod 0755 "$upgrade_repo/scripts/tester-runtime.sh" +git -C "$upgrade_repo" add scripts/tester-runtime.sh +git -C "$upgrade_repo" -c user.name=Example -c user.email=example@invalid.example commit --quiet -m 'fixture: valid tester candidate' +unit_fail_ref=$(git -C "$upgrade_repo" rev-parse HEAD) +unit_hash_before=$(sha256sum "$root/etc/systemd/system/ci-fleet-tester-health.service") +if FAKE_TESTER_SYSTEMCTL_FAIL=daemon-reload "$upgrade_repo/scripts/install-tester.sh" --upgrade --config /etc/ci-fleet-tester/tester.env --ref "$unit_fail_ref" >/dev/null 2>&1; then fail 'unit activation failure succeeded'; fi +[[ $(readlink -f "$root/opt/ci-fleet-tester/current") == "$root/opt/ci-fleet-tester/releases/$ref" && $(sha256sum "$root/etc/systemd/system/ci-fleet-tester-health.service") == "$unit_hash_before" ]] || fail 'unit activation failure did not restore incumbent release and units' + # Rollback switches only to a complete recorded release and keeps environments intact. old=0000000000000000000000000000000000000000 cp -a "$root/opt/ci-fleet-tester/releases/$ref" "$root/opt/ci-fleet-tester/releases/$old" +chmod 0755 "$root/opt/ci-fleet-tester/releases/$old" +chmod 0644 "$root/opt/ci-fleet-tester/releases/$old/.ci-fleet-source-revision" printf '%s\n' "$old" >"$root/opt/ci-fleet-tester/releases/$old/.ci-fleet-source-revision" +(cd "$root/opt/ci-fleet-tester/releases/$old" && sha256sum scripts/tester-runtime.sh .ci-fleet-source-revision host/systemd/* >.ci-fleet-release.sha256) +chmod 0444 "$root/opt/ci-fleet-tester/releases/$old/.ci-fleet-source-revision" "$root/opt/ci-fleet-tester/releases/$old/.ci-fleet-release.sha256" +chmod 0555 "$root/opt/ci-fleet-tester/releases/$old" printf '%s\n' "$old" >"$root/var/lib/ci-fleet-tester/last-known-good"; chmod 600 "$root/var/lib/ci-fleet-tester/last-known-good" +write_environment rollback-env 18086 +FAKE_TESTER_ROUTE_PORT=18086 "$runtime" --converge --environment rollback-env >/dev/null +rollback_state_hash=$(sha256sum "$root/var/lib/ci-fleet-tester/environments/rollback-env.state") "$installer" --rollback --config /etc/ci-fleet-tester/tester.env | grep -Fq ROLLBACK_OK || fail 'rollback failed' [[ $(readlink -f "$root/opt/ci-fleet-tester/current") == "$root/opt/ci-fleet-tester/releases/$old" ]] || fail 'rollback selected the wrong release' +[[ $(sha256sum "$root/var/lib/ci-fleet-tester/environments/rollback-env.state") == "$rollback_state_hash" ]] || fail 'rollback changed active environment state' +FAKE_TESTER_ROUTE_PORT=18086 "$runtime" --remove --environment rollback-env >/dev/null "$installer" --uninstall --config /etc/ci-fleet-tester/tester.env | grep -Fq UNINSTALL_OK || fail 'uninstall failed' [[ -f $root/etc/ci-fleet-tester/tester.env && ! -L $root/opt/ci-fleet-tester/current ]] || fail 'uninstall did not preserve config/remove runtime' diff --git a/scripts/tester-runtime.sh b/scripts/tester-runtime.sh index a1b441bd..9e47f712 100755 --- a/scripts/tester-runtime.sh +++ b/scripts/tester-runtime.sh @@ -80,6 +80,7 @@ load_global() { project_name() { printf 'ci-fleet-test-%s' "$1"; } state_path() { printf '%s/%s.state' "$state_dir" "$1"; } +deployed_compose_path() { printf '%s/%s.compose.json' "$state_dir" "$1"; } spec_path() { printf '%s/%s.env' "$environment_dir" "$1"; } load_spec() { @@ -95,6 +96,7 @@ load_spec() { route_port=${ENV_VALUES[CI_FLEET_TESTER_ROUTE_PORT]:-} [[ $project =~ ^[a-z0-9][a-z0-9-]{0,62}$ && $owner =~ ^[A-Za-z0-9][A-Za-z0-9_.@-]{0,127}$ ]] || die 'project or owner is invalid' [[ $route_service =~ ^[a-z0-9][a-z0-9-]{0,62}$ && $route_port =~ ^[0-9]+$ && $route_port -ge 1024 && $route_port -le 65535 ]] || die 'route service/port is invalid' + if [[ -z $expires_at && -f $(state_path "$id") ]]; then expires_at=$(awk -F= '$1=="EXPIRES_AT"{print $2}' "$(state_path "$id")"); fi if [[ -z $expires_at ]]; then expires_at=$(($(date +%s) + default_ttl)); fi [[ $expires_at =~ ^[0-9]+$ && $expires_at -gt $(date +%s) && $expires_at -le $(($(date +%s) + 604800)) ]] || die 'expiration must be in the future and at most seven days away' canonical=$(readlink -f -- "$compose_file") || die 'compose file is unavailable' @@ -118,21 +120,23 @@ image=re.compile(r'^[a-z0-9.-]+(?::[0-9]+)?/[A-Za-z0-9_./-]+@sha256:[0-9a-f]{64} ports=[] for name,service in services.items(): if not image.fullmatch(str(service.get('image',''))): raise SystemExit(f'{name}: image must use an immutable sha256 digest') - if service.get('privileged') or service.get('network_mode')=='host' or service.get('pid')=='host' or service.get('ipc')=='host': raise SystemExit(f'{name}: host/privileged access is forbidden') - if service.get('devices') or service.get('cap_add') or service.get('container_name') or service.get('hostname'): raise SystemExit(f'{name}: device/capability/global identity is forbidden') + if service.get('privileged') or service.get('network_mode') or service.get('pid') or service.get('ipc'): raise SystemExit(f'{name}: external namespace/privileged access is forbidden') + if service.get('devices') or service.get('cap_add') or service.get('container_name') or service.get('hostname') or service.get('use_api_socket'): raise SystemExit(f'{name}: device/capability/global identity is forbidden') + if service.get('environment') or service.get('env_file') or service.get('configs'): raise SystemExit(f'{name}: alternate credential channels are forbidden') if service.get('read_only') is not True or 'ALL' not in service.get('cap_drop',[]): raise SystemExit(f'{name}: read_only and cap_drop ALL are required') - security=' '.join(service.get('security_opt',[])) - if 'no-new-privileges' not in security: raise SystemExit(f'{name}: no-new-privileges is required') + if not any(re.fullmatch(r'no-new-privileges[:=]true', option) for option in service.get('security_opt',[])): raise SystemExit(f'{name}: no-new-privileges=true is required') for mount in service.get('volumes',[]): if isinstance(mount,str) or mount.get('type') not in ('volume','tmpfs'): raise SystemExit(f'{name}: host bind mounts are forbidden') for port in service.get('ports',[]): if not isinstance(port,dict) or str(port.get('host_ip','')) != '127.0.0.1': raise SystemExit(f'{name}: published ports must bind loopback') ports.append((name,int(port.get('published',0)),int(port.get('target',0)))) if ports != [(route_service,route_port,ports[0][2] if ports else 0)] or not ports or ports[0][2] < 1: raise SystemExit('exactly one declared loopback route is required') +if value.get('configs'): raise SystemExit('top-level configs are forbidden') for section in ('networks','volumes'): for name,item in value.get(section,{}).items(): resolved=item.get('name',f'{project}_{name}') if item.get('external') or not resolved.startswith(f'{project}_'): raise SystemExit(f'{section}.{name}: external/unscoped names are forbidden') + if section == 'volumes' and (item.get('driver') or item.get('driver_opts')): raise SystemExit(f'{section}.{name}: custom volume drivers are forbidden') for name,item in value.get('secrets',{}).items(): path=item.get('file') if item.get('external') or not isinstance(path,str) or os.path.realpath(path).rsplit('/',1)[0] != secret_dir: raise SystemExit(f'secrets.{name}: secret must be a host-local file in the environment secret directory') @@ -161,45 +165,62 @@ write_state() { local target tmp target=$(state_path "$environment"); tmp=$target.new printf 'ENVIRONMENT=%s\nPROJECT=%s\nOWNER=%s\nCOMPOSE_FILE=%s\nROUTE_SERVICE=%s\nROUTE_PORT=%s\nEXPIRES_AT=%s\nSOURCE_REVISION=%s\nIMAGE_DIGESTS=%s\nUPDATED_AT=%s\n' \ - "$environment" "$project" "$owner" "$compose_file" "$route_service" "$route_port" "$expires_at" "$source_revision" "$image_digests" "$(date +%s)" >"$tmp" + "$environment" "$project" "$owner" "$(deployed_compose_path "$environment")" "$route_service" "$route_port" "$expires_at" "$source_revision" "$image_digests" "$(date +%s)" >"$tmp" chmod 600 "$tmp"; mv -fT "$tmp" "$target" } -converge() { - local rendered count +prepare_converge() { + local count load_spec "$environment" check_port_unique count=$(find "$state_dir" -maxdepth 1 -type f -name '*.state' | wc -l) [[ -f $(state_path "$environment") || $count -private-repository $max_environments ]] || die 'maximum environment count reached' - rendered=$(mktemp) - if ! validate_compose "$rendered"; then rm -f "$rendered"; die 'compose policy validation failed'; fi - rm -f "$rendered" - docker compose -p "$compose_project" -f "$compose_file" up -d --remove-orphans --wait + prepared_rendered=$(mktemp) + if ! validate_compose "$prepared_rendered"; then rm -f "$prepared_rendered"; die 'compose policy validation failed'; fi +} + +apply_converge() { + install -m 0600 "$prepared_rendered" "$(deployed_compose_path "$environment")" write_state + if ! docker compose -p "$compose_project" -f "$(deployed_compose_path "$environment")" up -d --remove-orphans --wait; then + rm -f "$prepared_rendered" + die 'environment activation failed; tracked state retained for cleanup' + fi + rm -f "$prepared_rendered" report "CONVERGED environment=$environment project=$project owner=$owner route=loopback:$route_port expires_at=$expires_at" } +converge() { prepare_converge; apply_converge; } + remove_environment() { local target compose id=$1 target=$(state_path "$id") if [[ -f $target ]]; then secure_file "$target" 600 compose=$(awk -F= '$1=="COMPOSE_FILE"{print substr($0,index($0,"=")+1)}' "$target") - [[ $(readlink -f -- "$compose") == "$definition_dir"/* ]] || die 'stored compose path escaped definitions directory' + [[ $compose == "$(deployed_compose_path "$id")" ]] || die 'stored compose path is unexpected' + secure_file "$compose" 600 docker compose -p "$(project_name "$id")" -f "$compose" down --volumes --remove-orphans - rm -f -- "$target" + rm -f -- "$target" "$(deployed_compose_path "$id")" fi report "REMOVED environment=$id" } inspect_environment() { - local target=$1 id compose status resource value bytes=0 mount + local target=$1 id compose status=running resource value bytes=0 mount expected running_state + local -a containers=() id=$(basename "$target" .state); secure_file "$target" 600 while IFS='=' read -r key value; do case $key in ENVIRONMENT|PROJECT|OWNER|ROUTE_PORT|EXPIRES_AT|SOURCE_REVISION|IMAGE_DIGESTS|UPDATED_AT) printf '%s=%s ' "$key" "$value" ;; esac done <"$target" compose=$(awk -F= '$1=="COMPOSE_FILE"{print substr($0,index($0,"=")+1)}' "$target") - if docker compose -p "$(project_name "$id")" -f "$compose" ps --status running -q | grep -q .; then status=running; else status=unhealthy; fi + expected=$(python3 -c 'import json,sys; print(len(json.load(open(sys.argv[1]))["services"]))' "$compose") + mapfile -t containers < <(docker compose -p "$(project_name "$id")" -f "$compose" ps -q) + [[ ${#containers[@]} == "$expected" ]] || status=unhealthy + for resource in "${containers[@]}"; do + running_state=$(docker inspect --format '{{.State.Status}} {{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$resource") + [[ $running_state == 'running healthy' || $running_state == 'running none' ]] || status=unhealthy + done while IFS= read -r resource; do [[ -n $resource ]] || continue value=$(docker inspect --size --format '{{.SizeRw}}' "$resource"); [[ $value =~ ^[0-9]+$ ]] || die 'container disk size is invalid'; bytes=$((bytes + value)) @@ -235,18 +256,19 @@ case $action in report "CHECK_OK max_environments=$max_environments disk_used_percent=$used" ;; --converge) converge ;; - --reset) remove_environment "$environment"; converge ;; + --reset) prepare_converge; remove_environment "$environment"; apply_converge ;; --remove) remove_environment "$environment" ;; --inspect) [[ -f $(state_path "$environment") ]] || die 'environment is not installed'; inspect_environment "$(state_path "$environment")" ;; --cleanup) - now=$(date +%s) + now=$(date +%s); failed=0 for target in "$state_dir"/*.state; do [[ -e $target ]] || continue secure_file "$target" 600 expires=$(awk -F= '$1=="EXPIRES_AT"{print $2}' "$target") [[ $expires =~ ^[0-9]+$ ]] || die "invalid expiration in $target" - if ((expires <= now)); then remove_environment "$(basename "$target" .state)"; fi + if ((expires <= now)) && ! remove_environment "$(basename "$target" .state)"; then failed=1; fi done + ((failed == 0)) || die 'one or more expired environments could not be removed' report 'CLEANUP_OK' ;; --health) From de515af1267a35088d16d502a18a3a8534fed0ad Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Sun, 16 Aug 2026 00:17:29 -0500 Subject: [PATCH 03/42] test: cover tester cleanup failures --- scripts/test-install-tester.sh | 15 +++++++++++++-- scripts/tester-runtime.sh | 2 +- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/scripts/test-install-tester.sh b/scripts/test-install-tester.sh index 4aaa05d3..34142f39 100755 --- a/scripts/test-install-tester.sh +++ b/scripts/test-install-tester.sh @@ -85,9 +85,20 @@ FAKE_TESTER_ROUTE_PORT=18085 "$runtime" --remove --environment immutable-remove FAKE_TESTER_ROUTE_PORT=18080 "$runtime" --reset --environment preview-a >/dev/null grep -q 'down --volumes --remove-orphans' "$tmp/docker.log" || fail 'reset did not remove only the scoped Compose project' if grep -Eq 'system prune|volume prune|network prune' "$tmp/docker.log"; then fail 'global Docker prune was used'; fi -sed -i 's/^EXPIRES_AT=.*/EXPIRES_AT=1/' "$state" +write_environment expired-a 18087 +FAKE_TESTER_ROUTE_PORT=18087 "$runtime" --converge --environment expired-a >/dev/null +write_environment expired-b 18088 +FAKE_TESTER_ROUTE_PORT=18088 "$runtime" --converge --environment expired-b >/dev/null +for expired_state in "$state" "$root/var/lib/ci-fleet-tester/environments/expired-a.state" "$root/var/lib/ci-fleet-tester/environments/expired-b.state"; do + sed -i 's/^EXPIRES_AT=.*/EXPIRES_AT=1/' "$expired_state" +done +: >"$tmp/docker.log" +if FAKE_TESTER_DOWN_FAIL=1 "$runtime" --cleanup >/dev/null 2>&1; then fail 'cleanup ignored environment removal failures'; fi +for id in preview-a expired-a expired-b; do + grep -Fq "ci-fleet-test-$id" "$tmp/docker.log" || fail "cleanup stopped before attempting $id" +done "$runtime" --cleanup >/dev/null -[[ ! -e $state ]] || fail 'expired environment survived cleanup' +[[ ! -e $state && ! -e $root/var/lib/ci-fleet-tester/environments/expired-a.state && ! -e $root/var/lib/ci-fleet-tester/environments/expired-b.state ]] || fail 'expired environment survived cleanup' # Commit-backed installer tests run after the implementation commit exists. ref=$(git -C "$repo_root" rev-parse HEAD) diff --git a/scripts/tester-runtime.sh b/scripts/tester-runtime.sh index 9e47f712..1a1e9c81 100755 --- a/scripts/tester-runtime.sh +++ b/scripts/tester-runtime.sh @@ -200,7 +200,7 @@ remove_environment() { compose=$(awk -F= '$1=="COMPOSE_FILE"{print substr($0,index($0,"=")+1)}' "$target") [[ $compose == "$(deployed_compose_path "$id")" ]] || die 'stored compose path is unexpected' secure_file "$compose" 600 - docker compose -p "$(project_name "$id")" -f "$compose" down --volumes --remove-orphans + docker compose -p "$(project_name "$id")" -f "$compose" down --volumes --remove-orphans || return 1 rm -f -- "$target" "$(deployed_compose_path "$id")" fi report "REMOVED environment=$id" From 4866a84eb7ae93b47a5ec8a6a8aaa531151c5a45 Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Sun, 16 Aug 2026 00:18:41 -0500 Subject: [PATCH 04/42] docs: clarify tester credential boundary --- docs/TESTER-HOST.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/TESTER-HOST.md b/docs/TESTER-HOST.md index 1ef26641..50624551 100644 --- a/docs/TESTER-HOST.md +++ b/docs/TESTER-HOST.md @@ -13,7 +13,7 @@ A tester host accepts only: - application images addressed by an immutable `sha256` digest; - environment secrets stored below that environment's fixed host-local secret directory. -It rejects mutable images, public port binds, host bind mounts, external/unscoped Docker resources, privileged containers, added capabilities, host namespaces, global container names, and credentials outside the environment secret boundary. Every service must be read-only, drop all capabilities, and set `no-new-privileges`. Test identity, networks, storage, routes, domains, data, and credentials must have no production authority. Host/network isolation is an external acceptance gate, not something a repository script can prove. +It rejects mutable images, public port binds, host bind mounts, external/unscoped Docker resources, custom volume drivers/options, privileged containers, added capabilities, host or shared namespaces, Docker API access, global container names, and credentials outside the environment secret boundary. Compose environment variables, env files, and configs are forbidden credential channels; use only fixed mode-`0600` Compose secrets. Every service must be read-only, drop all capabilities, and set `no-new-privileges=true`. The validated rendered Compose model is copied into protected runtime state before activation, so partial starts remain tracked and later cleanup does not depend on a mutable or deleted source definition. Test identity, networks, storage, routes, domains, data, and credentials must have no production authority. Host/network isolation is an external acceptance gate, not something this repository-only change can prove. ## Prepare host-local configuration From 50905e0ce02d45319c807eb9f2b701f8c8c0409b Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Sun, 16 Aug 2026 00:32:59 -0500 Subject: [PATCH 05/42] test: support read-only releases as non-root --- scripts/install-tester.sh | 4 +++- scripts/test-install-tester.sh | 5 +++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/scripts/install-tester.sh b/scripts/install-tester.sh index 91d16d7d..14b8fa8d 100755 --- a/scripts/install-tester.sh +++ b/scripts/install-tester.sh @@ -187,7 +187,9 @@ case $action in ensure_directories if find "$runtime_state" -maxdepth 1 -type f -name '*.state' | grep -q .; then die 'remove every test environment before uninstalling the tester service'; fi remove_units - rm -f -- "$current_link" "$lkg_file"; rm -rf -- "$release_dir"; install -d -m 0755 "$release_dir" + rm -f -- "$current_link" "$lkg_file" + [[ ${CI_FLEET_TESTING:-0} != 1 ]] || chmod -R u+w "$release_dir" + rm -rf -- "$release_dir"; install -d -m 0755 "$release_dir" report 'UNINSTALL_OK preserved_config=true preserved_definitions=true preserved_secrets=true' ;; esac diff --git a/scripts/test-install-tester.sh b/scripts/test-install-tester.sh index 34142f39..f2bcfea2 100755 --- a/scripts/test-install-tester.sh +++ b/scripts/test-install-tester.sh @@ -4,7 +4,8 @@ repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) runtime=$repo_root/scripts/tester-runtime.sh installer=$repo_root/scripts/install-tester.sh tmp=$(mktemp -d) -trap 'rm -rf "$tmp"' EXIT +cleanup() { chmod -R u+w "$tmp" 2>/dev/null || true; rm -rf "$tmp"; } +trap cleanup EXIT fail() { printf 'FAIL %s\n' "$*" >&2; exit 1; } root=$tmp/root fake_bin=$tmp/bin @@ -112,7 +113,7 @@ for unit in ci-fleet-tester-health.service ci-fleet-tester-health.timer ci-fleet release=$root/opt/ci-fleet-tester/releases/$ref chmod u+w "$release/scripts/tester-runtime.sh"; printf '# tamper\n' >>"$release/scripts/tester-runtime.sh"; chmod 0555 "$release/scripts/tester-runtime.sh" if "$installer" --check --config /etc/ci-fleet-tester/tester.env >/dev/null 2>&1; then fail 'tampered installed release passed check'; fi -git -C "$repo_root" show "$ref:scripts/tester-runtime.sh" >"$release/scripts/tester-runtime.sh"; chmod 0555 "$release/scripts/tester-runtime.sh" +chmod u+w "$release/scripts/tester-runtime.sh"; git -C "$repo_root" show "$ref:scripts/tester-runtime.sh" >"$release/scripts/tester-runtime.sh"; chmod 0555 "$release/scripts/tester-runtime.sh" "$installer" --check --config /etc/ci-fleet-tester/tester.env >/dev/null || fail 'restored release failed check' # A syntactically valid candidate that fails its post-switch check restores the incumbent. From beff43cbdd92ef267741cbae0a41ad31d7b07db2 Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Sun, 16 Aug 2026 01:00:11 -0500 Subject: [PATCH 06/42] fix: close remaining tester isolation channels --- scripts/fixtures/fake-tester-docker.sh | 12 +++++++++--- scripts/test-install-tester.sh | 4 ++-- scripts/tester-runtime.sh | 7 +++++-- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/scripts/fixtures/fake-tester-docker.sh b/scripts/fixtures/fake-tester-docker.sh index 2b06e3e3..3eb0fd7b 100755 --- a/scripts/fixtures/fake-tester-docker.sh +++ b/scripts/fixtures/fake-tester-docker.sh @@ -25,7 +25,7 @@ done case ${operation:-} in config) digest=$(printf 'a%.0s' {1..64}) - privileged=false; read_only=true; host_ip=127.0.0.1; image="registry.example/example/app@sha256:$digest"; network_name="${project}_default"; secrets='{}'; service_extra=; top_extra=; volume_extra=; security='no-new-privileges:true' + privileged=false; read_only=true; host_ip=127.0.0.1; image="registry.example/example/app@sha256:$digest"; network_name="${project}_default"; secrets='{}'; service_extra=; top_extra=; volume_extra=; network_extra=; security='no-new-privileges:true' case ${FAKE_TESTER_POLICY:-valid} in mutable) image=registry.example/example/app:latest ;; privileged) privileged=true ;; @@ -38,11 +38,17 @@ case ${operation:-} in namespace-share) service_extra=',"network_mode":"service:other"' ;; false-nnp) security='no-new-privileges:false' ;; custom-volume) volume_extra=',"driver":"local"' ;; + volumes-from) service_extra=',"volumes_from":["container:other:rw"]' ;; + custom-network) network_extra=',"driver":"macvlan","driver_opts":{"parent":"eth0"}' ;; + replicas) service_extra=',"deploy":{"replicas":2}' ;; + lifecycle-hook) service_extra=',"post_start":[{"command":"true","privileged":true}]' ;; + gpu) service_extra=',"gpus":"all"' ;; + deploy-device) service_extra=',"deploy":{"resources":{"reservations":{"devices":[{"capabilities":["gpu"]}]}}}' ;; valid-secret|outside-secret) secrets=$(printf '{"credential":{"file":"%s"}}' "${FAKE_TESTER_SECRET_FILE:?}") ;; esac volume=${volume:-'{"type":"volume","source":"data","target":"/data"}'} - printf '{"services":{"web":{"image":"%s","privileged":%s,"read_only":%s,"cap_drop":["ALL"],"security_opt":["%s"],"volumes":[%s],"ports":[{"host_ip":"%s","published":%s,"target":8080,"protocol":"tcp"}]%s}},"networks":{"default":{"name":"%s"}},"volumes":{"data":{"name":"%s_data"%s}},"secrets":%s%s}\n' \ - "$image" "$privileged" "$read_only" "$security" "$volume" "$host_ip" "${FAKE_TESTER_ROUTE_PORT:-18080}" "$service_extra" "$network_name" "$project" "$volume_extra" "$secrets" "$top_extra" + printf '{"services":{"web":{"image":"%s","privileged":%s,"read_only":%s,"cap_drop":["ALL"],"security_opt":["%s"],"volumes":[%s],"ports":[{"host_ip":"%s","published":%s,"target":8080,"protocol":"tcp"}]%s}},"networks":{"default":{"name":"%s"%s}},"volumes":{"data":{"name":"%s_data"%s}},"secrets":%s%s}\n' \ + "$image" "$privileged" "$read_only" "$security" "$volume" "$host_ip" "${FAKE_TESTER_ROUTE_PORT:-18080}" "$service_extra" "$network_name" "$network_extra" "$project" "$volume_extra" "$secrets" "$top_extra" ;; up) [[ ${FAKE_TESTER_UP_FAIL:-0} != 1 ]] ;; down) [[ ${FAKE_TESTER_DOWN_FAIL:-0} != 1 ]] ;; diff --git a/scripts/test-install-tester.sh b/scripts/test-install-tester.sh index f2bcfea2..8dbd22fd 100755 --- a/scripts/test-install-tester.sh +++ b/scripts/test-install-tester.sh @@ -59,7 +59,7 @@ FAKE_TESTER_ROUTE_PORT=18080 "$runtime" --converge --environment preview-a >/dev [[ $(awk -F= '$1=="EXPIRES_AT"{print $2}' "$state") == "$original_expiry" ]] || fail 'idempotent converge extended expiration' write_environment preview-b 18080 if FAKE_TESTER_ROUTE_PORT=18080 "$runtime" --converge --environment preview-b >/dev/null 2>&1; then fail 'duplicate route port was accepted'; fi -for policy in mutable privileged bind broad-port external-network environment configs use-api-socket namespace-share false-nnp custom-volume; do +for policy in mutable privileged bind broad-port external-network environment configs use-api-socket namespace-share false-nnp custom-volume volumes-from custom-network replicas lifecycle-hook gpu deploy-device; do write_environment "bad-$policy" 18081 if FAKE_TESTER_ROUTE_PORT=18081 FAKE_TESTER_POLICY=$policy "$runtime" --converge --environment "bad-$policy" >/dev/null 2>&1; then fail "unsafe compose policy was accepted: $policy"; fi done @@ -139,7 +139,7 @@ if FAKE_TESTER_SYSTEMCTL_FAIL=daemon-reload "$upgrade_repo/scripts/install-teste old=0000000000000000000000000000000000000000 cp -a "$root/opt/ci-fleet-tester/releases/$ref" "$root/opt/ci-fleet-tester/releases/$old" chmod 0755 "$root/opt/ci-fleet-tester/releases/$old" -chmod 0644 "$root/opt/ci-fleet-tester/releases/$old/.ci-fleet-source-revision" +chmod 0644 "$root/opt/ci-fleet-tester/releases/$old/.ci-fleet-source-revision" "$root/opt/ci-fleet-tester/releases/$old/.ci-fleet-release.sha256" printf '%s\n' "$old" >"$root/opt/ci-fleet-tester/releases/$old/.ci-fleet-source-revision" (cd "$root/opt/ci-fleet-tester/releases/$old" && sha256sum scripts/tester-runtime.sh .ci-fleet-source-revision host/systemd/* >.ci-fleet-release.sha256) chmod 0444 "$root/opt/ci-fleet-tester/releases/$old/.ci-fleet-source-revision" "$root/opt/ci-fleet-tester/releases/$old/.ci-fleet-release.sha256" diff --git a/scripts/tester-runtime.sh b/scripts/tester-runtime.sh index 1a1e9c81..bf4ec062 100755 --- a/scripts/tester-runtime.sh +++ b/scripts/tester-runtime.sh @@ -120,8 +120,10 @@ image=re.compile(r'^[a-z0-9.-]+(?::[0-9]+)?/[A-Za-z0-9_./-]+@sha256:[0-9a-f]{64} ports=[] for name,service in services.items(): if not image.fullmatch(str(service.get('image',''))): raise SystemExit(f'{name}: image must use an immutable sha256 digest') - if service.get('privileged') or service.get('network_mode') or service.get('pid') or service.get('ipc'): raise SystemExit(f'{name}: external namespace/privileged access is forbidden') - if service.get('devices') or service.get('cap_add') or service.get('container_name') or service.get('hostname') or service.get('use_api_socket'): raise SystemExit(f'{name}: device/capability/global identity is forbidden') + if service.get('privileged') or service.get('network_mode') or service.get('pid') or service.get('ipc') or service.get('post_start') or service.get('pre_stop'): raise SystemExit(f'{name}: external namespace/privileged lifecycle access is forbidden') + deploy=service.get('deploy') or {}; reservations=(deploy.get('resources') or {}).get('reservations') or {} + if service.get('devices') or service.get('gpus') or reservations.get('devices') or service.get('cap_add') or service.get('container_name') or service.get('hostname') or service.get('use_api_socket') or service.get('volumes_from'): raise SystemExit(f'{name}: device/capability/external mount/global identity is forbidden') + if deploy.get('replicas',1) != 1: raise SystemExit(f'{name}: exactly one replica is required') if service.get('environment') or service.get('env_file') or service.get('configs'): raise SystemExit(f'{name}: alternate credential channels are forbidden') if service.get('read_only') is not True or 'ALL' not in service.get('cap_drop',[]): raise SystemExit(f'{name}: read_only and cap_drop ALL are required') if not any(re.fullmatch(r'no-new-privileges[:=]true', option) for option in service.get('security_opt',[])): raise SystemExit(f'{name}: no-new-privileges=true is required') @@ -136,6 +138,7 @@ for section in ('networks','volumes'): for name,item in value.get(section,{}).items(): resolved=item.get('name',f'{project}_{name}') if item.get('external') or not resolved.startswith(f'{project}_'): raise SystemExit(f'{section}.{name}: external/unscoped names are forbidden') + if section == 'networks' and (item.get('driver') not in (None,'bridge') or item.get('driver_opts')): raise SystemExit(f'{section}.{name}: custom network drivers are forbidden') if section == 'volumes' and (item.get('driver') or item.get('driver_opts')): raise SystemExit(f'{section}.{name}: custom volume drivers are forbidden') for name,item in value.get('secrets',{}).items(): path=item.get('file') From b9242fb5e3fabf6a60f2f3824d5e812109ee4689 Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Sun, 16 Aug 2026 01:05:39 -0500 Subject: [PATCH 07/42] fix: make tester lifecycle repairable --- scripts/install-tester.sh | 39 ++++++++++++++++++++++------------ scripts/test-install-tester.sh | 30 ++++++++++++++++++++++++-- 2 files changed, 54 insertions(+), 15 deletions(-) diff --git a/scripts/install-tester.sh b/scripts/install-tester.sh index 14b8fa8d..a18b1dd6 100755 --- a/scripts/install-tester.sh +++ b/scripts/install-tester.sh @@ -47,11 +47,19 @@ secret_root=$config_root/secrets runtime_state=$state_root/environments docker_root=$(root_path /var/lib/docker) docker_socket=$(root_path /var/run/docker.sock) +runtime_lock=$(root_path /run/lock/ci-fleet-tester.lock) units=(ci-fleet-tester-health.service ci-fleet-tester-health.timer ci-fleet-tester-cleanup.service ci-fleet-tester-cleanup.timer) timers=(ci-fleet-tester-health.timer ci-fleet-tester-cleanup.timer) secure_file() { [[ -f $1 && ! -L $1 && $(stat -c %u "$1") == "$expected_uid" && $(stat -c %a "$1") == "$2" ]] || die "protected file is unsafe: $1"; } secure_dir() { [[ -d $1 && ! -L $1 && $(stat -c %u "$1") == "$expected_uid" && $(stat -c %a "$1") == "$2" ]] || die "protected directory is unsafe: $1"; } +remove_release_tree() { [[ ! -e $1 ]] || { [[ ${CI_FLEET_TESTING:-0} != 1 ]] || chmod -R u+w "$1"; rm -rf -- "$1"; }; } + +reject_git_replacements() { + local common + common=$(git -C "$repo_root" rev-parse --path-format=absolute --git-common-dir) + [[ -z $(git -C "$repo_root" for-each-ref --format='%(refname)' refs/replace) && ! -s $common/info/grafts ]] || die 'Git replacement or graft metadata is forbidden' +} host_preflight() { local os_release docker_context actual_root used @@ -91,17 +99,19 @@ release_complete() { } stage_release() { - local commit=$1 target=$release_dir/$1 staging=$release_dir/.staging-$1 - if [[ -e $target ]]; then release_complete "$target" "$commit" || die 'existing tester release is incomplete'; return; fi - rm -rf -- "$staging"; install -d -m 0755 "$staging" - git -C "$repo_root" cat-file -e "$commit^{commit}" 2>/dev/null || die 'requested source commit is unavailable locally' - git -C "$repo_root" archive "$commit" scripts/tester-runtime.sh host/systemd/ci-fleet-tester-health.service host/systemd/ci-fleet-tester-health.timer host/systemd/ci-fleet-tester-cleanup.service host/systemd/ci-fleet-tester-cleanup.timer | tar -x -C "$staging" + local commit=$1 target=$release_dir/$1 staging=$release_dir/.staging-$1 replaced=$release_dir/.replaced-$1 + if [[ -e $target ]] && release_complete "$target" "$commit"; then return; fi + remove_release_tree "$staging"; remove_release_tree "$replaced"; install -d -m 0755 "$staging" + GIT_NO_REPLACE_OBJECTS=1 git -C "$repo_root" cat-file -e "$commit^{commit}" 2>/dev/null || die 'requested source commit is unavailable locally' + GIT_NO_REPLACE_OBJECTS=1 git -C "$repo_root" archive "$commit" scripts/tester-runtime.sh host/systemd/ci-fleet-tester-health.service host/systemd/ci-fleet-tester-health.timer host/systemd/ci-fleet-tester-cleanup.service host/systemd/ci-fleet-tester-cleanup.timer | tar -x -C "$staging" printf '%s\n' "$commit" >"$staging/.ci-fleet-source-revision"; chmod 0644 "$staging/.ci-fleet-source-revision" chmod 0755 "$staging/scripts/tester-runtime.sh"; shellcheck "$staging/scripts/tester-runtime.sh"; bash -n "$staging/scripts/tester-runtime.sh" (cd "$staging" && sha256sum scripts/tester-runtime.sh .ci-fleet-source-revision host/systemd/* >.ci-fleet-release.sha256) chmod 0444 "$staging/.ci-fleet-source-revision" "$staging/.ci-fleet-release.sha256" "$staging"/host/systemd/* chmod 0555 "$staging" "$staging/scripts" "$staging/host" "$staging/host/systemd" "$staging/scripts/tester-runtime.sh" - mv -T "$staging" "$target" + [[ ! -e $target ]] || mv -T "$target" "$replaced" + if ! mv -T "$staging" "$target"; then [[ ! -e $replaced ]] || mv -T "$replaced" "$target"; die 'could not replace tester release'; fi + remove_release_tree "$replaced" } install_units() { @@ -112,9 +122,9 @@ install_units() { } remove_units() { - systemctl disable --now "${timers[@]}" >/dev/null 2>&1 || true - local unit; for unit in "${units[@]}"; do rm -f -- "$systemd_dir/$unit"; done - systemctl daemon-reload || true + systemctl disable --now "${timers[@]}" >/dev/null 2>&1 || return 1 + local unit; for unit in "${units[@]}"; do rm -f -- "$systemd_dir/$unit" || return 1; done + systemctl daemon-reload } write_lkg() { @@ -133,11 +143,11 @@ activate_release() { ln -sfn "$release_dir/$previous" "$current_link.new"; mv -Tf "$current_link.new" "$current_link"; install_units "$release_dir/$previous" else rm -f -- "$current_link" - remove_units + remove_units || true fi die 'candidate tester activation failed; previous release restored when available' fi - [[ ! $previous =~ ^[0-9a-f]{40}$ || $previous == "$commit" ]] || write_lkg "$previous" + if [[ $previous =~ ^[0-9a-f]{40}$ && $previous != "$commit" ]] && release_complete "$release_dir/$previous" "$previous"; then write_lkg "$previous"; fi report "INSTALL_OK source_revision=$commit previous_revision=${previous:-none} config=$config" } @@ -152,7 +162,8 @@ case $action in --install|--upgrade) host_preflight; ensure_directories secure_file "$(root_path "$config")" 600 - [[ $(git -C "$repo_root" rev-parse 'HEAD^{commit}') == "$ref" ]] || die 'reviewed checkout HEAD does not match --ref' + reject_git_replacements + [[ $(GIT_NO_REPLACE_OBJECTS=1 git -C "$repo_root" rev-parse 'HEAD^{commit}') == "$ref" ]] || die 'reviewed checkout HEAD does not match --ref' if ! git -C "$repo_root" diff --quiet || ! git -C "$repo_root" diff --cached --quiet; then die 'reviewed checkout has tracked changes'; fi current=$(installed_revision || true) [[ $action != --install || -z $current || $current == "$ref" ]] || die 'tester is already installed at another revision; use --upgrade' @@ -164,6 +175,7 @@ case $action in secure_file "$(root_path "$config")" 600 current=$(installed_revision) || die 'tester is not installed' release_complete "$release_dir/$current" "$current" || die 'installed release is incomplete' + for unit in "${units[@]}"; do cmp -s "$release_dir/$current/host/systemd/$unit" "$systemd_dir/$unit" || die "installed unit differs from active release: $unit"; done for timer in "${timers[@]}"; do if ! systemctl is-enabled --quiet "$timer" || ! systemctl is-active --quiet "$timer"; then die "timer is inactive: $timer"; fi done @@ -185,8 +197,9 @@ case $action in ;; --uninstall) ensure_directories + install -d -m 0755 "$(dirname "$runtime_lock")"; exec 8>"$runtime_lock"; flock -x 8 if find "$runtime_state" -maxdepth 1 -type f -name '*.state' | grep -q .; then die 'remove every test environment before uninstalling the tester service'; fi - remove_units + remove_units || die 'could not stop and disable tester maintenance units' rm -f -- "$current_link" "$lkg_file" [[ ${CI_FLEET_TESTING:-0} != 1 ]] || chmod -R u+w "$release_dir" rm -rf -- "$release_dir"; install -d -m 0755 "$release_dir" diff --git a/scripts/test-install-tester.sh b/scripts/test-install-tester.sh index 8dbd22fd..f0edcc77 100755 --- a/scripts/test-install-tester.sh +++ b/scripts/test-install-tester.sh @@ -113,8 +113,12 @@ for unit in ci-fleet-tester-health.service ci-fleet-tester-health.timer ci-fleet release=$root/opt/ci-fleet-tester/releases/$ref chmod u+w "$release/scripts/tester-runtime.sh"; printf '# tamper\n' >>"$release/scripts/tester-runtime.sh"; chmod 0555 "$release/scripts/tester-runtime.sh" if "$installer" --check --config /etc/ci-fleet-tester/tester.env >/dev/null 2>&1; then fail 'tampered installed release passed check'; fi -chmod u+w "$release/scripts/tester-runtime.sh"; git -C "$repo_root" show "$ref:scripts/tester-runtime.sh" >"$release/scripts/tester-runtime.sh"; chmod 0555 "$release/scripts/tester-runtime.sh" -"$installer" --check --config /etc/ci-fleet-tester/tester.env >/dev/null || fail 'restored release failed check' +"$installer" --install --config /etc/ci-fleet-tester/tester.env --ref "$ref" >/dev/null || fail 'same-ref install did not repair an incomplete release' +"$installer" --check --config /etc/ci-fleet-tester/tester.env >/dev/null || fail 'repaired release failed check' +installed_unit=$root/etc/systemd/system/ci-fleet-tester-health.service +printf '# drift\n' >>"$installed_unit" +if "$installer" --check --config /etc/ci-fleet-tester/tester.env >/dev/null 2>&1; then fail 'installed unit drift passed check'; fi +cp "$release/host/systemd/ci-fleet-tester-health.service" "$installed_unit" # A syntactically valid candidate that fails its post-switch check restores the incumbent. upgrade_repo=$tmp/upgrade-repo @@ -125,6 +129,11 @@ git -C "$upgrade_repo" -c user.name=Example -c user.email=example@invalid.exampl bad_ref=$(git -C "$upgrade_repo" rev-parse HEAD) if "$upgrade_repo/scripts/install-tester.sh" --upgrade --config /etc/ci-fleet-tester/tester.env --ref "$bad_ref" >/dev/null 2>&1; then fail 'failed candidate activation succeeded'; fi [[ $(readlink -f "$root/opt/ci-fleet-tester/current") == "$root/opt/ci-fleet-tester/releases/$ref" ]] || fail 'failed upgrade did not restore incumbent release' +git -C "$upgrade_repo" checkout --quiet "$ref" +git -C "$upgrade_repo" replace "$ref" "$bad_ref" +if "$upgrade_repo/scripts/install-tester.sh" --install --config /etc/ci-fleet-tester/tester.env --ref "$ref" >/dev/null 2>&1; then fail 'Git replacement metadata was accepted'; fi +git -C "$upgrade_repo" replace -d "$ref" >/dev/null +git -C "$upgrade_repo" checkout --quiet "$bad_ref" # Unit activation failures restore the incumbent symlink and units. git -C "$upgrade_repo" show "$ref:scripts/tester-runtime.sh" >"$upgrade_repo/scripts/tester-runtime.sh"; chmod 0755 "$upgrade_repo/scripts/tester-runtime.sh" @@ -135,6 +144,15 @@ unit_hash_before=$(sha256sum "$root/etc/systemd/system/ci-fleet-tester-health.se if FAKE_TESTER_SYSTEMCTL_FAIL=daemon-reload "$upgrade_repo/scripts/install-tester.sh" --upgrade --config /etc/ci-fleet-tester/tester.env --ref "$unit_fail_ref" >/dev/null 2>&1; then fail 'unit activation failure succeeded'; fi [[ $(readlink -f "$root/opt/ci-fleet-tester/current") == "$root/opt/ci-fleet-tester/releases/$ref" && $(sha256sum "$root/etc/systemd/system/ci-fleet-tester-health.service") == "$unit_hash_before" ]] || fail 'unit activation failure did not restore incumbent release and units' +# A corrupt incumbent is never recorded as the rollback target. +chmod u+w "$release/scripts/tester-runtime.sh"; printf '# corrupt incumbent\n' >>"$release/scripts/tester-runtime.sh"; chmod 0555 "$release/scripts/tester-runtime.sh" +printf '# candidate\n' >>"$upgrade_repo/docs/TESTER-HOST.md" +git -C "$upgrade_repo" add docs/TESTER-HOST.md +git -C "$upgrade_repo" -c user.name=Example -c user.email=example@invalid.example commit --quiet -m 'fixture: second valid tester candidate' +valid_ref=$(git -C "$upgrade_repo" rev-parse HEAD) +"$upgrade_repo/scripts/install-tester.sh" --upgrade --config /etc/ci-fleet-tester/tester.env --ref "$valid_ref" >/dev/null || fail 'valid upgrade failed' +[[ ! -e $root/var/lib/ci-fleet-tester/last-known-good ]] || fail 'corrupt incumbent was recorded as last-known-good' + # Rollback switches only to a complete recorded release and keeps environments intact. old=0000000000000000000000000000000000000000 cp -a "$root/opt/ci-fleet-tester/releases/$ref" "$root/opt/ci-fleet-tester/releases/$old" @@ -153,6 +171,14 @@ rollback_state_hash=$(sha256sum "$root/var/lib/ci-fleet-tester/environments/roll [[ $(sha256sum "$root/var/lib/ci-fleet-tester/environments/rollback-env.state") == "$rollback_state_hash" ]] || fail 'rollback changed active environment state' FAKE_TESTER_ROUTE_PORT=18086 "$runtime" --remove --environment rollback-env >/dev/null +lock_file=$root/run/lock/ci-fleet-tester.lock +lock_ready=$tmp/lock-ready +flock "$lock_file" -c "touch '$lock_ready'; sleep 1" & lock_pid=$! +while [[ ! -e $lock_ready ]]; do kill -0 "$lock_pid" 2>/dev/null || fail 'could not acquire fixture lifecycle lock'; done +if timeout 0.2 "$installer" --uninstall --config /etc/ci-fleet-tester/tester.env >/dev/null 2>&1; then fail 'uninstall ignored the runtime lifecycle lock'; fi +wait "$lock_pid" +if FAKE_TESTER_SYSTEMCTL_FAIL='disable --now' "$installer" --uninstall --config /etc/ci-fleet-tester/tester.env >/dev/null 2>&1; then fail 'uninstall ignored systemd teardown failure'; fi +[[ -L $root/opt/ci-fleet-tester/current ]] || fail 'failed uninstall removed the active release' "$installer" --uninstall --config /etc/ci-fleet-tester/tester.env | grep -Fq UNINSTALL_OK || fail 'uninstall failed' [[ -f $root/etc/ci-fleet-tester/tester.env && ! -L $root/opt/ci-fleet-tester/current ]] || fail 'uninstall did not preserve config/remove runtime' printf 'TESTER_INSTALLER_TESTS_OK\n' From 25fbac1dc150335556888b850dc0ecdc6a2044ad Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Sun, 16 Aug 2026 01:45:14 -0500 Subject: [PATCH 08/42] fix: serialize tester lifecycle updates --- scripts/fixtures/fake-tester-docker.sh | 2 ++ scripts/install-tester.sh | 12 ++++++++++-- scripts/test-install-tester.sh | 18 ++++++++++++++++-- scripts/tester-runtime.sh | 17 +++++++++++------ 4 files changed, 39 insertions(+), 10 deletions(-) diff --git a/scripts/fixtures/fake-tester-docker.sh b/scripts/fixtures/fake-tester-docker.sh index 3eb0fd7b..069b0005 100755 --- a/scripts/fixtures/fake-tester-docker.sh +++ b/scripts/fixtures/fake-tester-docker.sh @@ -44,6 +44,8 @@ case ${operation:-} in lifecycle-hook) service_extra=',"post_start":[{"command":"true","privileged":true}]' ;; gpu) service_extra=',"gpus":"all"' ;; deploy-device) service_extra=',"deploy":{"resources":{"reservations":{"devices":[{"capabilities":["gpu"]}]}}}' ;; + build) service_extra=',"build":{"context":"/"}' ;; + changed-model) service_extra=',"pull_policy":"always"' ;; valid-secret|outside-secret) secrets=$(printf '{"credential":{"file":"%s"}}' "${FAKE_TESTER_SECRET_FILE:?}") ;; esac volume=${volume:-'{"type":"volume","source":"data","target":"/data"}'} diff --git a/scripts/install-tester.sh b/scripts/install-tester.sh index a18b1dd6..3fb95b49 100755 --- a/scripts/install-tester.sh +++ b/scripts/install-tester.sh @@ -61,6 +61,13 @@ reject_git_replacements() { [[ -z $(git -C "$repo_root" for-each-ref --format='%(refname)' refs/replace) && ! -s $common/info/grafts ]] || die 'Git replacement or graft metadata is forbidden' } +acquire_lifecycle_lock() { + install -d -m 0755 "$(dirname "$runtime_lock")" + exec 8>"$runtime_lock" + flock -x 8 + export CI_FLEET_TESTER_LOCK_FD=8 +} + host_preflight() { local os_release docker_context actual_root used os_release=$(root_path /etc/os-release) @@ -142,8 +149,8 @@ activate_release() { if [[ $previous =~ ^[0-9a-f]{40}$ ]] && release_complete "$release_dir/$previous" "$previous"; then ln -sfn "$release_dir/$previous" "$current_link.new"; mv -Tf "$current_link.new" "$current_link"; install_units "$release_dir/$previous" else + remove_units || die 'candidate activation failed and fresh-install unit teardown also failed; candidate retained for recovery' rm -f -- "$current_link" - remove_units || true fi die 'candidate tester activation failed; previous release restored when available' fi @@ -158,6 +165,8 @@ installed_revision() { basename "$target" } +case $action in --install|--upgrade|--check|--rollback|--uninstall) acquire_lifecycle_lock ;; esac + case $action in --install|--upgrade) host_preflight; ensure_directories @@ -197,7 +206,6 @@ case $action in ;; --uninstall) ensure_directories - install -d -m 0755 "$(dirname "$runtime_lock")"; exec 8>"$runtime_lock"; flock -x 8 if find "$runtime_state" -maxdepth 1 -type f -name '*.state' | grep -q .; then die 'remove every test environment before uninstalling the tester service'; fi remove_units || die 'could not stop and disable tester maintenance units' rm -f -- "$current_link" "$lkg_file" diff --git a/scripts/test-install-tester.sh b/scripts/test-install-tester.sh index f0edcc77..274d0a80 100755 --- a/scripts/test-install-tester.sh +++ b/scripts/test-install-tester.sh @@ -59,10 +59,13 @@ FAKE_TESTER_ROUTE_PORT=18080 "$runtime" --converge --environment preview-a >/dev [[ $(awk -F= '$1=="EXPIRES_AT"{print $2}' "$state") == "$original_expiry" ]] || fail 'idempotent converge extended expiration' write_environment preview-b 18080 if FAKE_TESTER_ROUTE_PORT=18080 "$runtime" --converge --environment preview-b >/dev/null 2>&1; then fail 'duplicate route port was accepted'; fi -for policy in mutable privileged bind broad-port external-network environment configs use-api-socket namespace-share false-nnp custom-volume volumes-from custom-network replicas lifecycle-hook gpu deploy-device; do +for policy in mutable privileged bind broad-port external-network environment configs use-api-socket namespace-share false-nnp custom-volume volumes-from custom-network replicas lifecycle-hook gpu deploy-device build; do write_environment "bad-$policy" 18081 if FAKE_TESTER_ROUTE_PORT=18081 FAKE_TESTER_POLICY=$policy "$runtime" --converge --environment "bad-$policy" >/dev/null 2>&1; then fail "unsafe compose policy was accepted: $policy"; fi done +deployed_hash=$(sha256sum "$root/var/lib/ci-fleet-tester/environments/preview-a.compose.json") +if FAKE_TESTER_ROUTE_PORT=18080 FAKE_TESTER_POLICY=changed-model "$runtime" --converge --environment preview-a >/dev/null 2>&1; then fail 'converge replaced the incumbent Compose model without reset'; fi +[[ $(sha256sum "$root/var/lib/ci-fleet-tester/environments/preview-a.compose.json") == "$deployed_hash" ]] || fail 'rejected model replacement changed incumbent state' if FAKE_TESTER_ROUTE_PORT=18080 FAKE_TESTER_POLICY=mutable "$runtime" --reset --environment preview-a >/dev/null 2>&1; then fail 'reset accepted invalid replacement'; fi [[ -f $state ]] || fail 'reset deleted the incumbent before validation' if FAKE_TESTER_ROUTE_PORT=18080 FAKE_TESTER_CONTAINER_STATE='exited unhealthy' "$runtime" --health >/dev/null 2>&1; then fail 'health accepted a stopped managed service'; fi @@ -75,6 +78,9 @@ outside_secret=$root/etc/ci-fleet-tester/secrets/outside printf 'example-test-scope-value\n' >"$outside_secret"; chmod 600 "$outside_secret" write_environment outside-secret 18083 if FAKE_TESTER_ROUTE_PORT=18083 FAKE_TESTER_POLICY=outside-secret FAKE_TESTER_SECRET_FILE=$outside_secret "$runtime" --converge --environment outside-secret >/dev/null 2>&1; then fail 'out-of-boundary secret was accepted'; fi +write_environment hardlink-secret 18089 +ln "$outside_secret" "$root/etc/ci-fleet-tester/secrets/hardlink-secret/credential" +if FAKE_TESTER_ROUTE_PORT=18089 FAKE_TESTER_POLICY=valid-secret FAKE_TESTER_SECRET_FILE=$root/etc/ci-fleet-tester/secrets/hardlink-secret/credential "$runtime" --converge --environment hardlink-secret >/dev/null 2>&1; then fail 'hard-linked host secret was accepted'; fi write_environment partial-up 18084 if FAKE_TESTER_ROUTE_PORT=18084 FAKE_TESTER_UP_FAIL=1 "$runtime" --converge --environment partial-up >/dev/null 2>&1; then fail 'partial activation succeeded'; fi [[ -f $root/var/lib/ci-fleet-tester/environments/partial-up.state && -f $root/var/lib/ci-fleet-tester/environments/partial-up.compose.json ]] || fail 'partial activation was not tracked for cleanup' @@ -175,10 +181,18 @@ lock_file=$root/run/lock/ci-fleet-tester.lock lock_ready=$tmp/lock-ready flock "$lock_file" -c "touch '$lock_ready'; sleep 1" & lock_pid=$! while [[ ! -e $lock_ready ]]; do kill -0 "$lock_pid" 2>/dev/null || fail 'could not acquire fixture lifecycle lock'; done -if timeout 0.2 "$installer" --uninstall --config /etc/ci-fleet-tester/tester.env >/dev/null 2>&1; then fail 'uninstall ignored the runtime lifecycle lock'; fi +set +e +timeout 0.2 "$installer" --install --config /etc/ci-fleet-tester/tester.env --ref "$ref" >/dev/null 2>&1 +lock_rc=$? +set -e +[[ $lock_rc == 124 ]] || fail 'installer lifecycle mutation ignored the shared lock' wait "$lock_pid" if FAKE_TESTER_SYSTEMCTL_FAIL='disable --now' "$installer" --uninstall --config /etc/ci-fleet-tester/tester.env >/dev/null 2>&1; then fail 'uninstall ignored systemd teardown failure'; fi [[ -L $root/opt/ci-fleet-tester/current ]] || fail 'failed uninstall removed the active release' "$installer" --uninstall --config /etc/ci-fleet-tester/tester.env | grep -Fq UNINSTALL_OK || fail 'uninstall failed' [[ -f $root/etc/ci-fleet-tester/tester.env && ! -L $root/opt/ci-fleet-tester/current ]] || fail 'uninstall did not preserve config/remove runtime' +git -C "$upgrade_repo" checkout --quiet "$bad_ref" +if FAKE_TESTER_SYSTEMCTL_FAIL='disable --now' "$upgrade_repo/scripts/install-tester.sh" --install --config /etc/ci-fleet-tester/tester.env --ref "$bad_ref" >/dev/null 2>&1; then fail 'invalid fresh install succeeded'; fi +[[ -L $root/opt/ci-fleet-tester/current ]] || fail 'failed fresh-install teardown removed the recovery link' +"$upgrade_repo/scripts/install-tester.sh" --uninstall --config /etc/ci-fleet-tester/tester.env >/dev/null printf 'TESTER_INSTALLER_TESTS_OK\n' diff --git a/scripts/tester-runtime.sh b/scripts/tester-runtime.sh index bf4ec062..ad87d26e 100755 --- a/scripts/tester-runtime.sh +++ b/scripts/tester-runtime.sh @@ -33,7 +33,7 @@ while (($#)); do done [[ -n $action ]] || { usage; exit 2; } case $action in --converge|--reset|--remove|--inspect) [[ $environment =~ ^[a-z0-9][a-z0-9-]{0,62}$ ]] || die 'environment ID is invalid' ;; *) [[ -z $environment ]] || die '--environment is not valid for this action' ;; esac -for command in awk basename chmod curl date df dirname docker du find flock getent grep install mktemp mv python3 readlink rm stat wc; do command -v "$command" >/dev/null || die "required command is unavailable: $command"; done +for command in awk basename chmod cmp curl date df dirname docker du find flock getent grep install mktemp mv python3 readlink rm stat wc; do command -v "$command" >/dev/null || die "required command is unavailable: $command"; done secure_directory() { local path=$1 mode=$2 @@ -122,7 +122,7 @@ for name,service in services.items(): if not image.fullmatch(str(service.get('image',''))): raise SystemExit(f'{name}: image must use an immutable sha256 digest') if service.get('privileged') or service.get('network_mode') or service.get('pid') or service.get('ipc') or service.get('post_start') or service.get('pre_stop'): raise SystemExit(f'{name}: external namespace/privileged lifecycle access is forbidden') deploy=service.get('deploy') or {}; reservations=(deploy.get('resources') or {}).get('reservations') or {} - if service.get('devices') or service.get('gpus') or reservations.get('devices') or service.get('cap_add') or service.get('container_name') or service.get('hostname') or service.get('use_api_socket') or service.get('volumes_from'): raise SystemExit(f'{name}: device/capability/external mount/global identity is forbidden') + if service.get('build') or service.get('devices') or service.get('gpus') or reservations.get('devices') or service.get('cap_add') or service.get('container_name') or service.get('hostname') or service.get('use_api_socket') or service.get('volumes_from'): raise SystemExit(f'{name}: build/device/capability/external mount/global identity is forbidden') if deploy.get('replicas',1) != 1: raise SystemExit(f'{name}: exactly one replica is required') if service.get('environment') or service.get('env_file') or service.get('configs'): raise SystemExit(f'{name}: alternate credential channels are forbidden') if service.get('read_only') is not True or 'ALL' not in service.get('cap_drop',[]): raise SystemExit(f'{name}: read_only and cap_drop ALL are required') @@ -144,7 +144,7 @@ for name,item in value.get('secrets',{}).items(): path=item.get('file') if item.get('external') or not isinstance(path,str) or os.path.realpath(path).rsplit('/',1)[0] != secret_dir: raise SystemExit(f'secrets.{name}: secret must be a host-local file in the environment secret directory') metadata=os.lstat(path) - if not stat.S_ISREG(metadata.st_mode) or stat.S_ISLNK(metadata.st_mode) or metadata.st_uid != expected_uid or stat.S_IMODE(metadata.st_mode) != 0o600: raise SystemExit(f'secrets.{name}: secret must be owner-controlled mode 0600') + if not stat.S_ISREG(metadata.st_mode) or stat.S_ISLNK(metadata.st_mode) or metadata.st_nlink != 1 or metadata.st_uid != expected_uid or stat.S_IMODE(metadata.st_mode) != 0o600: raise SystemExit(f'secrets.{name}: secret must be a singly linked owner-controlled mode-0600 file') PY image_digests=$(python3 - "$rendered" <<'PY' import json,sys @@ -180,6 +180,7 @@ prepare_converge() { [[ -f $(state_path "$environment") || $count -private-repository $max_environments ]] || die 'maximum environment count reached' prepared_rendered=$(mktemp) if ! validate_compose "$prepared_rendered"; then rm -f "$prepared_rendered"; die 'compose policy validation failed'; fi + if [[ $action == --converge && -f $(state_path "$environment") ]] && ! cmp -s "$prepared_rendered" "$(deployed_compose_path "$environment")"; then rm -f "$prepared_rendered"; die 'changing an installed Compose model requires reset'; fi } apply_converge() { @@ -245,9 +246,13 @@ fi unset DOCKER_CONTEXT export DOCKER_HOST="unix://$docker_socket" [[ $(docker info --format '{{.DockerRootDir}}') == "$(root_path /var/lib/docker)" ]] || die 'Docker daemon root is not the expected local path' -install -d -m 0755 "$(dirname "$lock_file")" -exec 9>"$lock_file" -flock -x 9 +if [[ ${CI_FLEET_TESTER_LOCK_FD:-} == 8 && -e /proc/$$/fd/8 && $(readlink -f /proc/$$/fd/8) == "$lock_file" ]] && flock -n 8; then + unset CI_FLEET_TESTER_LOCK_FD +else + install -d -m 0755 "$(dirname "$lock_file")" + exec 9>"$lock_file" + flock -x 9 +fi case $action in --check) docker info --format '{{.DockerRootDir}}' >/dev/null From 3e5c7e40dac13e4962610f71f360953a656da47a Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Sun, 16 Aug 2026 02:08:26 -0500 Subject: [PATCH 09/42] fix: contain tester recovery failures --- scripts/install-tester.sh | 8 +++++--- scripts/test-install-tester.sh | 15 ++++++++++++++- scripts/tester-runtime.sh | 17 +++++++++++------ 3 files changed, 30 insertions(+), 10 deletions(-) diff --git a/scripts/install-tester.sh b/scripts/install-tester.sh index 3fb95b49..442beb84 100755 --- a/scripts/install-tester.sh +++ b/scripts/install-tester.sh @@ -123,7 +123,8 @@ stage_release() { install_units() { local source=$1 unit - for unit in "${units[@]}"; do install -m 0644 "$source/host/systemd/$unit" "$systemd_dir/$unit" || return 1; done + for unit in "${units[@]}"; do install -m 0644 "$source/host/systemd/$unit" "$systemd_dir/$unit.new" || return 1; done + for unit in "${units[@]}"; do mv -fT "$systemd_dir/$unit.new" "$systemd_dir/$unit" || return 1; done systemctl daemon-reload || return 1 systemctl enable --now "${timers[@]}" >/dev/null || return 1 } @@ -147,7 +148,8 @@ activate_release() { ln -sfn "$target" "$current_link.new"; mv -Tf "$current_link.new" "$current_link" if ! install_units "$target" || ! "$target/scripts/tester-runtime.sh" --check || ! "$target/scripts/tester-runtime.sh" --health; then if [[ $previous =~ ^[0-9a-f]{40}$ ]] && release_complete "$release_dir/$previous" "$previous"; then - ln -sfn "$release_dir/$previous" "$current_link.new"; mv -Tf "$current_link.new" "$current_link"; install_units "$release_dir/$previous" + ln -sfn "$release_dir/$previous" "$current_link.new"; mv -Tf "$current_link.new" "$current_link" + install_units "$release_dir/$previous" || die 'candidate activation failed and incumbent unit restore failed; incumbent link retained for recovery' else remove_units || die 'candidate activation failed and fresh-install unit teardown also failed; candidate retained for recovery' rm -f -- "$current_link" @@ -165,7 +167,7 @@ installed_revision() { basename "$target" } -case $action in --install|--upgrade|--check|--rollback|--uninstall) acquire_lifecycle_lock ;; esac +case $action in --install|--upgrade|--check|--reset|--rollback|--uninstall) acquire_lifecycle_lock ;; esac case $action in --install|--upgrade) diff --git a/scripts/test-install-tester.sh b/scripts/test-install-tester.sh index 274d0a80..951f09b2 100755 --- a/scripts/test-install-tester.sh +++ b/scripts/test-install-tester.sh @@ -50,8 +50,10 @@ state=$root/var/lib/ci-fleet-tester/environments/preview-a.state [[ -f $state && $(stat -c %a "$state") == 600 ]] || fail 'state was not protected' inspect_output=$(FAKE_TESTER_ROUTE_PORT=18080 "$runtime" --inspect --environment preview-a) grep -q 'IMAGE_DIGESTS=sha256:[a-f0-9]\{64\}.*STATUS=running DISK_BYTES=[1-9][0-9]*' <<<"$inspect_output" || fail 'inspect did not report health and disk use' +deployed_inode=$(stat -c %i "$root/var/lib/ci-fleet-tester/environments/preview-a.compose.json") FAKE_TESTER_ROUTE_PORT=18080 "$runtime" --converge --environment preview-a >/dev/null [[ $(find "$root/var/lib/ci-fleet-tester/environments" -name '*.state' | wc -l) == 1 ]] || fail 'idempotent converge duplicated state' +[[ $(stat -c %i "$root/var/lib/ci-fleet-tester/environments/preview-a.compose.json") == "$deployed_inode" ]] || fail 'idempotent converge replaced the protected Compose snapshot' original_expiry=$(awk -F= '$1=="EXPIRES_AT"{print $2}' "$state") grep -v '^CI_FLEET_TESTER_EXPIRES_AT=' "$root/etc/ci-fleet-tester/environments/preview-a.env" >"$tmp/spec" mv "$tmp/spec" "$root/etc/ci-fleet-tester/environments/preview-a.env"; chmod 600 "$root/etc/ci-fleet-tester/environments/preview-a.env" @@ -100,6 +102,11 @@ for expired_state in "$state" "$root/var/lib/ci-fleet-tester/environments/expire sed -i 's/^EXPIRES_AT=.*/EXPIRES_AT=1/' "$expired_state" done : >"$tmp/docker.log" +mv "$root/var/lib/ci-fleet-tester/environments/expired-a.compose.json" "$tmp/expired-a.compose.json" +if FAKE_TESTER_DOWN_FAIL=1 "$runtime" --cleanup >/dev/null 2>&1; then fail 'cleanup ignored a damaged protected environment'; fi +for id in preview-a expired-b; do grep -Fq "ci-fleet-test-$id" "$tmp/docker.log" || fail "damaged state stopped cleanup before $id"; done +mv "$tmp/expired-a.compose.json" "$root/var/lib/ci-fleet-tester/environments/expired-a.compose.json" +: >"$tmp/docker.log" if FAKE_TESTER_DOWN_FAIL=1 "$runtime" --cleanup >/dev/null 2>&1; then fail 'cleanup ignored environment removal failures'; fi for id in preview-a expired-a expired-b; do grep -Fq "ci-fleet-test-$id" "$tmp/docker.log" || fail "cleanup stopped before attempting $id" @@ -147,7 +154,8 @@ git -C "$upgrade_repo" add scripts/tester-runtime.sh git -C "$upgrade_repo" -c user.name=Example -c user.email=example@invalid.example commit --quiet -m 'fixture: valid tester candidate' unit_fail_ref=$(git -C "$upgrade_repo" rev-parse HEAD) unit_hash_before=$(sha256sum "$root/etc/systemd/system/ci-fleet-tester-health.service") -if FAKE_TESTER_SYSTEMCTL_FAIL=daemon-reload "$upgrade_repo/scripts/install-tester.sh" --upgrade --config /etc/ci-fleet-tester/tester.env --ref "$unit_fail_ref" >/dev/null 2>&1; then fail 'unit activation failure succeeded'; fi +if FAKE_TESTER_SYSTEMCTL_FAIL=daemon-reload "$upgrade_repo/scripts/install-tester.sh" --upgrade --config /etc/ci-fleet-tester/tester.env --ref "$unit_fail_ref" >"$tmp/unit-restore.log" 2>&1; then fail 'unit activation failure succeeded'; fi +grep -Fq 'incumbent unit restore failed' "$tmp/unit-restore.log" || fail 'secondary incumbent restore failure was not surfaced' [[ $(readlink -f "$root/opt/ci-fleet-tester/current") == "$root/opt/ci-fleet-tester/releases/$ref" && $(sha256sum "$root/etc/systemd/system/ci-fleet-tester-health.service") == "$unit_hash_before" ]] || fail 'unit activation failure did not restore incumbent release and units' # A corrupt incumbent is never recorded as the rollback target. @@ -186,6 +194,11 @@ timeout 0.2 "$installer" --install --config /etc/ci-fleet-tester/tester.env --re lock_rc=$? set -e [[ $lock_rc == 124 ]] || fail 'installer lifecycle mutation ignored the shared lock' +set +e +timeout 0.2 "$installer" --reset --environment absent >/dev/null 2>&1 +reset_lock_rc=$? +set -e +[[ $reset_lock_rc == 124 ]] || fail 'reset resolved the active release before acquiring the lifecycle lock' wait "$lock_pid" if FAKE_TESTER_SYSTEMCTL_FAIL='disable --now' "$installer" --uninstall --config /etc/ci-fleet-tester/tester.env >/dev/null 2>&1; then fail 'uninstall ignored systemd teardown failure'; fi [[ -L $root/opt/ci-fleet-tester/current ]] || fail 'failed uninstall removed the active release' diff --git a/scripts/tester-runtime.sh b/scripts/tester-runtime.sh index ad87d26e..74987e5a 100755 --- a/scripts/tester-runtime.sh +++ b/scripts/tester-runtime.sh @@ -180,11 +180,14 @@ prepare_converge() { [[ -f $(state_path "$environment") || $count -private-repository $max_environments ]] || die 'maximum environment count reached' prepared_rendered=$(mktemp) if ! validate_compose "$prepared_rendered"; then rm -f "$prepared_rendered"; die 'compose policy validation failed'; fi - if [[ $action == --converge && -f $(state_path "$environment") ]] && ! cmp -s "$prepared_rendered" "$(deployed_compose_path "$environment")"; then rm -f "$prepared_rendered"; die 'changing an installed Compose model requires reset'; fi + if [[ $action == --converge && -f $(state_path "$environment") ]]; then + secure_file "$(deployed_compose_path "$environment")" 600 + if ! cmp -s "$prepared_rendered" "$(deployed_compose_path "$environment")"; then rm -f "$prepared_rendered"; die 'changing an installed Compose model requires reset'; fi + fi } apply_converge() { - install -m 0600 "$prepared_rendered" "$(deployed_compose_path "$environment")" + [[ -f $(state_path "$environment") ]] || install -m 0600 "$prepared_rendered" "$(deployed_compose_path "$environment")" write_state if ! docker compose -p "$compose_project" -f "$(deployed_compose_path "$environment")" up -d --remove-orphans --wait; then rm -f "$prepared_rendered" @@ -271,10 +274,12 @@ case $action in now=$(date +%s); failed=0 for target in "$state_dir"/*.state; do [[ -e $target ]] || continue - secure_file "$target" 600 - expires=$(awk -F= '$1=="EXPIRES_AT"{print $2}' "$target") - [[ $expires =~ ^[0-9]+$ ]] || die "invalid expiration in $target" - if ((expires <= now)) && ! remove_environment "$(basename "$target" .state)"; then failed=1; fi + if ! ( + secure_file "$target" 600 + expires=$(awk -F= '$1=="EXPIRES_AT"{print $2}' "$target") + [[ $expires =~ ^[0-9]+$ ]] || die "invalid expiration in $target" + ((expires > now)) || remove_environment "$(basename "$target" .state)" + ); then failed=1; fi done ((failed == 0)) || die 'one or more expired environments could not be removed' report 'CLEANUP_OK' From 9b09cfaba62bf5b51a4ce92bc00c087cee621fa6 Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Sun, 16 Aug 2026 02:28:16 -0500 Subject: [PATCH 10/42] fix: quiesce tester timers during activation --- scripts/fixtures/fake-tester-docker.sh | 2 ++ scripts/install-tester.sh | 8 +++++--- scripts/test-install-tester.sh | 11 +++++++++-- scripts/tester-runtime.sh | 4 ++-- 4 files changed, 18 insertions(+), 7 deletions(-) diff --git a/scripts/fixtures/fake-tester-docker.sh b/scripts/fixtures/fake-tester-docker.sh index 069b0005..1458c883 100755 --- a/scripts/fixtures/fake-tester-docker.sh +++ b/scripts/fixtures/fake-tester-docker.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash set -Eeuo pipefail printf '%s\n' "$*" >>"${FAKE_TESTER_DOCKER_LOG:?}" +[[ -z ${FAKE_TESTER_EVENT_LOG:-} ]] || printf 'docker %s\n' "$*" >>"$FAKE_TESTER_EVENT_LOG" if [[ $1 == context && $2 == show ]]; then printf 'default\n'; exit 0; fi if [[ $1 == info ]]; then printf '%s\n' "${FAKE_TESTER_DOCKER_ROOT:?}"; exit 0; fi if [[ $1 == ps ]]; then printf 'fixture-container-id\n'; exit 0; fi @@ -46,6 +47,7 @@ case ${operation:-} in deploy-device) service_extra=',"deploy":{"resources":{"reservations":{"devices":[{"capabilities":["gpu"]}]}}}' ;; build) service_extra=',"build":{"context":"/"}' ;; changed-model) service_extra=',"pull_policy":"always"' ;; + unconfined) security='no-new-privileges:true","seccomp=unconfined' ;; valid-secret|outside-secret) secrets=$(printf '{"credential":{"file":"%s"}}' "${FAKE_TESTER_SECRET_FILE:?}") ;; esac volume=${volume:-'{"type":"volume","source":"data","target":"/data"}'} diff --git a/scripts/install-tester.sh b/scripts/install-tester.sh index 442beb84..fc6dddbd 100755 --- a/scripts/install-tester.sh +++ b/scripts/install-tester.sh @@ -126,9 +126,10 @@ install_units() { for unit in "${units[@]}"; do install -m 0644 "$source/host/systemd/$unit" "$systemd_dir/$unit.new" || return 1; done for unit in "${units[@]}"; do mv -fT "$systemd_dir/$unit.new" "$systemd_dir/$unit" || return 1; done systemctl daemon-reload || return 1 - systemctl enable --now "${timers[@]}" >/dev/null || return 1 } +enable_timers() { systemctl enable --now "${timers[@]}" >/dev/null; } + remove_units() { systemctl disable --now "${timers[@]}" >/dev/null 2>&1 || return 1 local unit; for unit in "${units[@]}"; do rm -f -- "$systemd_dir/$unit" || return 1; done @@ -145,11 +146,12 @@ activate_release() { local commit=$1 target=$release_dir/$1 previous= release_complete "$target" "$commit" || die 'candidate tester release is incomplete' [[ ! -L $current_link ]] || previous=$(basename "$(readlink -f "$current_link")") + if [[ $previous =~ ^[0-9a-f]{40}$ ]]; then systemctl disable --now "${timers[@]}" >/dev/null || die 'could not quiesce tester maintenance timers'; fi ln -sfn "$target" "$current_link.new"; mv -Tf "$current_link.new" "$current_link" - if ! install_units "$target" || ! "$target/scripts/tester-runtime.sh" --check || ! "$target/scripts/tester-runtime.sh" --health; then + if ! install_units "$target" || ! "$target/scripts/tester-runtime.sh" --check || ! "$target/scripts/tester-runtime.sh" --health || ! enable_timers; then if [[ $previous =~ ^[0-9a-f]{40}$ ]] && release_complete "$release_dir/$previous" "$previous"; then ln -sfn "$release_dir/$previous" "$current_link.new"; mv -Tf "$current_link.new" "$current_link" - install_units "$release_dir/$previous" || die 'candidate activation failed and incumbent unit restore failed; incumbent link retained for recovery' + if ! install_units "$release_dir/$previous" || ! enable_timers; then die 'candidate activation failed and incumbent unit restore failed; incumbent link retained for recovery'; fi else remove_units || die 'candidate activation failed and fresh-install unit teardown also failed; candidate retained for recovery' rm -f -- "$current_link" diff --git a/scripts/test-install-tester.sh b/scripts/test-install-tester.sh index 951f09b2..f0239a2c 100755 --- a/scripts/test-install-tester.sh +++ b/scripts/test-install-tester.sh @@ -24,6 +24,7 @@ EOF cat >"$fake_bin/systemctl" <<'EOF' #!/usr/bin/env bash printf '%s\n' "$*" >>"${FAKE_TESTER_SYSTEMCTL_LOG:?}" +[[ -z ${FAKE_TESTER_EVENT_LOG:-} ]] || printf 'systemctl %s\n' "$*" >>"$FAKE_TESTER_EVENT_LOG" [[ -z ${FAKE_TESTER_SYSTEMCTL_FAIL:-} || " $* " != *" $FAKE_TESTER_SYSTEMCTL_FAIL "* ]] EOF printf '#!/usr/bin/env bash\nexit 0\n' >"$fake_bin/curl" @@ -31,6 +32,7 @@ printf '#!/usr/bin/env bash\nexit 0\n' >"$fake_bin/getent" chmod 0755 "$fake_bin/df" "$fake_bin/systemctl" "$fake_bin/curl" "$fake_bin/getent" export PATH="$fake_bin:$PATH" CI_FLEET_TESTING=1 CI_FLEET_ROOT_PREFIX=$root export FAKE_TESTER_DOCKER_ROOT=$root/var/lib/docker FAKE_TESTER_VOLUME_ROOT=$root/var/lib/fake-tester-volume FAKE_TESTER_DOCKER_LOG=$tmp/docker.log FAKE_TESTER_SYSTEMCTL_LOG=$tmp/systemctl.log +export FAKE_TESTER_EVENT_LOG=$tmp/events.log write_environment() { local id=$1 port=$2 @@ -61,7 +63,7 @@ FAKE_TESTER_ROUTE_PORT=18080 "$runtime" --converge --environment preview-a >/dev [[ $(awk -F= '$1=="EXPIRES_AT"{print $2}' "$state") == "$original_expiry" ]] || fail 'idempotent converge extended expiration' write_environment preview-b 18080 if FAKE_TESTER_ROUTE_PORT=18080 "$runtime" --converge --environment preview-b >/dev/null 2>&1; then fail 'duplicate route port was accepted'; fi -for policy in mutable privileged bind broad-port external-network environment configs use-api-socket namespace-share false-nnp custom-volume volumes-from custom-network replicas lifecycle-hook gpu deploy-device build; do +for policy in mutable privileged bind broad-port external-network environment configs use-api-socket namespace-share false-nnp unconfined custom-volume volumes-from custom-network replicas lifecycle-hook gpu deploy-device build; do write_environment "bad-$policy" 18081 if FAKE_TESTER_ROUTE_PORT=18081 FAKE_TESTER_POLICY=$policy "$runtime" --converge --environment "bad-$policy" >/dev/null 2>&1; then fail "unsafe compose policy was accepted: $policy"; fi done @@ -92,7 +94,7 @@ FAKE_TESTER_ROUTE_PORT=18085 "$runtime" --converge --environment immutable-remov rm "$root/etc/ci-fleet-tester/definitions/immutable-remove.yaml" FAKE_TESTER_ROUTE_PORT=18085 "$runtime" --remove --environment immutable-remove >/dev/null FAKE_TESTER_ROUTE_PORT=18080 "$runtime" --reset --environment preview-a >/dev/null -grep -q 'down --volumes --remove-orphans' "$tmp/docker.log" || fail 'reset did not remove only the scoped Compose project' +grep -q 'down --timeout 10 --volumes --remove-orphans' "$tmp/docker.log" || fail 'reset did not use bounded scoped Compose teardown' if grep -Eq 'system prune|volume prune|network prune' "$tmp/docker.log"; then fail 'global Docker prune was used'; fi write_environment expired-a 18087 FAKE_TESTER_ROUTE_PORT=18087 "$runtime" --converge --environment expired-a >/dev/null @@ -164,8 +166,13 @@ printf '# candidate\n' >>"$upgrade_repo/docs/TESTER-HOST.md" git -C "$upgrade_repo" add docs/TESTER-HOST.md git -C "$upgrade_repo" -c user.name=Example -c user.email=example@invalid.example commit --quiet -m 'fixture: second valid tester candidate' valid_ref=$(git -C "$upgrade_repo" rev-parse HEAD) +: >"$tmp/events.log" "$upgrade_repo/scripts/install-tester.sh" --upgrade --config /etc/ci-fleet-tester/tester.env --ref "$valid_ref" >/dev/null || fail 'valid upgrade failed' [[ ! -e $root/var/lib/ci-fleet-tester/last-known-good ]] || fail 'corrupt incumbent was recorded as last-known-good' +disable_line=$(grep -n '^systemctl disable --now ' "$tmp/events.log" | tail -1 | cut -d: -f1) +check_line=$(grep -n '^docker info ' "$tmp/events.log" | tail -1 | cut -d: -f1) +enable_line=$(grep -n '^systemctl enable --now ' "$tmp/events.log" | tail -1 | cut -d: -f1) +[[ -n $disable_line && -n $check_line && -n $enable_line && $disable_line -private-repository $check_line && $check_line -private-repository $enable_line ]] || fail 'timers were not quiesced until candidate validation completed' # Rollback switches only to a complete recorded release and keeps environments intact. old=0000000000000000000000000000000000000000 diff --git a/scripts/tester-runtime.sh b/scripts/tester-runtime.sh index 74987e5a..32c63ad1 100755 --- a/scripts/tester-runtime.sh +++ b/scripts/tester-runtime.sh @@ -126,7 +126,7 @@ for name,service in services.items(): if deploy.get('replicas',1) != 1: raise SystemExit(f'{name}: exactly one replica is required') if service.get('environment') or service.get('env_file') or service.get('configs'): raise SystemExit(f'{name}: alternate credential channels are forbidden') if service.get('read_only') is not True or 'ALL' not in service.get('cap_drop',[]): raise SystemExit(f'{name}: read_only and cap_drop ALL are required') - if not any(re.fullmatch(r'no-new-privileges[:=]true', option) for option in service.get('security_opt',[])): raise SystemExit(f'{name}: no-new-privileges=true is required') + if service.get('security_opt') not in (['no-new-privileges:true'],['no-new-privileges=true']): raise SystemExit(f'{name}: no-new-privileges=true must be the only security option') for mount in service.get('volumes',[]): if isinstance(mount,str) or mount.get('type') not in ('volume','tmpfs'): raise SystemExit(f'{name}: host bind mounts are forbidden') for port in service.get('ports',[]): @@ -207,7 +207,7 @@ remove_environment() { compose=$(awk -F= '$1=="COMPOSE_FILE"{print substr($0,index($0,"=")+1)}' "$target") [[ $compose == "$(deployed_compose_path "$id")" ]] || die 'stored compose path is unexpected' secure_file "$compose" 600 - docker compose -p "$(project_name "$id")" -f "$compose" down --volumes --remove-orphans || return 1 + docker compose -p "$(project_name "$id")" -f "$compose" down --timeout 10 --volumes --remove-orphans || return 1 rm -f -- "$target" "$(deployed_compose_path "$id")" fi report "REMOVED environment=$id" From e4fca5bedd8bcfc53b735fd9419fbce983251c07 Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Sun, 16 Aug 2026 02:45:29 -0500 Subject: [PATCH 11/42] fix: close tester isolation gaps --- scripts/fixtures/fake-tester-docker.sh | 4 ++++ scripts/install-tester.sh | 5 ++++- scripts/test-install-tester.sh | 10 +++++++++- scripts/tester-runtime.sh | 14 +++++++++----- 4 files changed, 26 insertions(+), 7 deletions(-) diff --git a/scripts/fixtures/fake-tester-docker.sh b/scripts/fixtures/fake-tester-docker.sh index 1458c883..e5afbc79 100755 --- a/scripts/fixtures/fake-tester-docker.sh +++ b/scripts/fixtures/fake-tester-docker.sh @@ -48,6 +48,10 @@ case ${operation:-} in build) service_extra=',"build":{"context":"/"}' ;; changed-model) service_extra=',"pull_policy":"always"' ;; unconfined) security='no-new-privileges:true","seccomp=unconfined' ;; + external-links) service_extra=',"external_links":["other-environment-db:db"]' ;; + userns-host) service_extra=',"userns_mode":"host"' ;; + cgroup-host) service_extra=',"cgroup":"host"' ;; + interpolation) [[ -z ${TOKEN:-} ]] || service_extra=$(printf ',"command":["app","--token=%s"]' "$TOKEN") ;; valid-secret|outside-secret) secrets=$(printf '{"credential":{"file":"%s"}}' "${FAKE_TESTER_SECRET_FILE:?}") ;; esac volume=${volume:-'{"type":"volume","source":"data","target":"/data"}'} diff --git a/scripts/install-tester.sh b/scripts/install-tester.sh index fc6dddbd..8ae1e6bb 100755 --- a/scripts/install-tester.sh +++ b/scripts/install-tester.sh @@ -146,7 +146,10 @@ activate_release() { local commit=$1 target=$release_dir/$1 previous= release_complete "$target" "$commit" || die 'candidate tester release is incomplete' [[ ! -L $current_link ]] || previous=$(basename "$(readlink -f "$current_link")") - if [[ $previous =~ ^[0-9a-f]{40}$ ]]; then systemctl disable --now "${timers[@]}" >/dev/null || die 'could not quiesce tester maintenance timers'; fi + if [[ $previous =~ ^[0-9a-f]{40}$ ]] && ! systemctl disable --now "${timers[@]}" >/dev/null; then + enable_timers || die 'could not quiesce or restore tester maintenance timers' + die 'could not quiesce tester maintenance timers; incumbent timers restored' + fi ln -sfn "$target" "$current_link.new"; mv -Tf "$current_link.new" "$current_link" if ! install_units "$target" || ! "$target/scripts/tester-runtime.sh" --check || ! "$target/scripts/tester-runtime.sh" --health || ! enable_timers; then if [[ $previous =~ ^[0-9a-f]{40}$ ]] && release_complete "$release_dir/$previous" "$previous"; then diff --git a/scripts/test-install-tester.sh b/scripts/test-install-tester.sh index f0239a2c..33cec5fe 100755 --- a/scripts/test-install-tester.sh +++ b/scripts/test-install-tester.sh @@ -63,10 +63,14 @@ FAKE_TESTER_ROUTE_PORT=18080 "$runtime" --converge --environment preview-a >/dev [[ $(awk -F= '$1=="EXPIRES_AT"{print $2}' "$state") == "$original_expiry" ]] || fail 'idempotent converge extended expiration' write_environment preview-b 18080 if FAKE_TESTER_ROUTE_PORT=18080 "$runtime" --converge --environment preview-b >/dev/null 2>&1; then fail 'duplicate route port was accepted'; fi -for policy in mutable privileged bind broad-port external-network environment configs use-api-socket namespace-share false-nnp unconfined custom-volume volumes-from custom-network replicas lifecycle-hook gpu deploy-device build; do +for policy in mutable privileged bind broad-port external-network environment configs use-api-socket namespace-share false-nnp unconfined custom-volume volumes-from external-links userns-host cgroup-host custom-network replicas lifecycle-hook gpu deploy-device build; do write_environment "bad-$policy" 18081 if FAKE_TESTER_ROUTE_PORT=18081 FAKE_TESTER_POLICY=$policy "$runtime" --converge --environment "bad-$policy" >/dev/null 2>&1; then fail "unsafe compose policy was accepted: $policy"; fi done +write_environment interpolation 18090 +TOKEN=must-not-render FAKE_TESTER_ROUTE_PORT=18090 FAKE_TESTER_POLICY=interpolation "$runtime" --converge --environment interpolation >/dev/null +if grep -Fq must-not-render "$root/var/lib/ci-fleet-tester/environments/interpolation.compose.json"; then fail 'caller environment was interpolated into the Compose model'; fi +FAKE_TESTER_ROUTE_PORT=18090 "$runtime" --remove --environment interpolation >/dev/null deployed_hash=$(sha256sum "$root/var/lib/ci-fleet-tester/environments/preview-a.compose.json") if FAKE_TESTER_ROUTE_PORT=18080 FAKE_TESTER_POLICY=changed-model "$runtime" --converge --environment preview-a >/dev/null 2>&1; then fail 'converge replaced the incumbent Compose model without reset'; fi [[ $(sha256sum "$root/var/lib/ci-fleet-tester/environments/preview-a.compose.json") == "$deployed_hash" ]] || fail 'rejected model replacement changed incumbent state' @@ -88,6 +92,7 @@ if FAKE_TESTER_ROUTE_PORT=18089 FAKE_TESTER_POLICY=valid-secret FAKE_TESTER_SECR write_environment partial-up 18084 if FAKE_TESTER_ROUTE_PORT=18084 FAKE_TESTER_UP_FAIL=1 "$runtime" --converge --environment partial-up >/dev/null 2>&1; then fail 'partial activation succeeded'; fi [[ -f $root/var/lib/ci-fleet-tester/environments/partial-up.state && -f $root/var/lib/ci-fleet-tester/environments/partial-up.compose.json ]] || fail 'partial activation was not tracked for cleanup' +grep -q 'up -d --remove-orphans --wait --wait-timeout 60' "$tmp/docker.log" || fail 'Compose activation wait was not bounded' FAKE_TESTER_ROUTE_PORT=18084 "$runtime" --remove --environment partial-up >/dev/null write_environment immutable-remove 18085 FAKE_TESTER_ROUTE_PORT=18085 "$runtime" --converge --environment immutable-remove >/dev/null @@ -166,6 +171,9 @@ printf '# candidate\n' >>"$upgrade_repo/docs/TESTER-HOST.md" git -C "$upgrade_repo" add docs/TESTER-HOST.md git -C "$upgrade_repo" -c user.name=Example -c user.email=example@invalid.example commit --quiet -m 'fixture: second valid tester candidate' valid_ref=$(git -C "$upgrade_repo" rev-parse HEAD) +: >"$tmp/systemctl.log" +if FAKE_TESTER_SYSTEMCTL_FAIL='disable --now' "$upgrade_repo/scripts/install-tester.sh" --upgrade --config /etc/ci-fleet-tester/tester.env --ref "$valid_ref" >/dev/null 2>&1; then fail 'upgrade ignored timer quiescence failure'; fi +grep -Fq 'enable --now ci-fleet-tester-health.timer ci-fleet-tester-cleanup.timer' "$tmp/systemctl.log" || fail 'quiescence failure did not restore incumbent timers' : >"$tmp/events.log" "$upgrade_repo/scripts/install-tester.sh" --upgrade --config /etc/ci-fleet-tester/tester.env --ref "$valid_ref" >/dev/null || fail 'valid upgrade failed' [[ ! -e $root/var/lib/ci-fleet-tester/last-known-good ]] || fail 'corrupt incumbent was recorded as last-known-good' diff --git a/scripts/tester-runtime.sh b/scripts/tester-runtime.sh index 32c63ad1..e7187ee6 100755 --- a/scripts/tester-runtime.sh +++ b/scripts/tester-runtime.sh @@ -33,7 +33,7 @@ while (($#)); do done [[ -n $action ]] || { usage; exit 2; } case $action in --converge|--reset|--remove|--inspect) [[ $environment =~ ^[a-z0-9][a-z0-9-]{0,62}$ ]] || die 'environment ID is invalid' ;; *) [[ -z $environment ]] || die '--environment is not valid for this action' ;; esac -for command in awk basename chmod cmp curl date df dirname docker du find flock getent grep install mktemp mv python3 readlink rm stat wc; do command -v "$command" >/dev/null || die "required command is unavailable: $command"; done +for command in awk basename chmod cmp curl date df dirname docker du env find flock getent grep install mktemp mv python3 readlink rm stat wc; do command -v "$command" >/dev/null || die "required command is unavailable: $command"; done secure_directory() { local path=$1 mode=$2 @@ -108,8 +108,12 @@ load_spec() { } validate_compose() { - local rendered=$1 - docker compose -p "$compose_project" -f "$compose_file" config --format json >"$rendered" || return 1 + local rendered=$1 empty_env variable + local -a clean_environment=(env -i "PATH=$PATH" "DOCKER_HOST=$DOCKER_HOST") + if [[ ${CI_FLEET_TESTING:-0} == 1 ]]; then for variable in ${!FAKE_@}; do clean_environment+=("$variable=${!variable}"); done; fi + empty_env=$(mktemp); chmod 600 "$empty_env" + if ! "${clean_environment[@]}" docker compose --env-file "$empty_env" -p "$compose_project" -f "$compose_file" config --format json >"$rendered"; then rm -f "$empty_env"; return 1; fi + rm -f "$empty_env" chmod 600 "$rendered" python3 - "$rendered" "$route_service" "$route_port" "$compose_project" "$secret_dir" "$expected_uid" <<'PY' || return 1 import json,os,re,stat,sys @@ -120,7 +124,7 @@ image=re.compile(r'^[a-z0-9.-]+(?::[0-9]+)?/[A-Za-z0-9_./-]+@sha256:[0-9a-f]{64} ports=[] for name,service in services.items(): if not image.fullmatch(str(service.get('image',''))): raise SystemExit(f'{name}: image must use an immutable sha256 digest') - if service.get('privileged') or service.get('network_mode') or service.get('pid') or service.get('ipc') or service.get('post_start') or service.get('pre_stop'): raise SystemExit(f'{name}: external namespace/privileged lifecycle access is forbidden') + if service.get('privileged') or service.get('network_mode') or service.get('pid') or service.get('ipc') or service.get('userns_mode') or service.get('cgroup') or service.get('external_links') or service.get('post_start') or service.get('pre_stop'): raise SystemExit(f'{name}: external namespace/link/privileged lifecycle access is forbidden') deploy=service.get('deploy') or {}; reservations=(deploy.get('resources') or {}).get('reservations') or {} if service.get('build') or service.get('devices') or service.get('gpus') or reservations.get('devices') or service.get('cap_add') or service.get('container_name') or service.get('hostname') or service.get('use_api_socket') or service.get('volumes_from'): raise SystemExit(f'{name}: build/device/capability/external mount/global identity is forbidden') if deploy.get('replicas',1) != 1: raise SystemExit(f'{name}: exactly one replica is required') @@ -189,7 +193,7 @@ prepare_converge() { apply_converge() { [[ -f $(state_path "$environment") ]] || install -m 0600 "$prepared_rendered" "$(deployed_compose_path "$environment")" write_state - if ! docker compose -p "$compose_project" -f "$(deployed_compose_path "$environment")" up -d --remove-orphans --wait; then + if ! docker compose -p "$compose_project" -f "$(deployed_compose_path "$environment")" up -d --remove-orphans --wait --wait-timeout 60; then rm -f "$prepared_rendered" die 'environment activation failed; tracked state retained for cleanup' fi From e2c33af54704586cc9cb349ee2859567f394210e Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Sun, 16 Aug 2026 03:08:49 -0500 Subject: [PATCH 12/42] fix: route tester lifecycle through stable launcher --- docs/TESTER-HOST.md | 8 +++---- host/systemd/ci-fleet-tester-cleanup.service | 4 ++-- host/systemd/ci-fleet-tester-health.service | 4 ++-- scripts/fixtures/fake-tester-docker.sh | 2 ++ scripts/install-tester.sh | 23 +++++++++++--------- scripts/test-install-tester.sh | 16 ++++++++++++-- scripts/tester-launcher.sh | 21 ++++++++++++++++++ scripts/tester-runtime.sh | 11 ++++++++-- 8 files changed, 67 insertions(+), 22 deletions(-) create mode 100644 scripts/tester-launcher.sh diff --git a/docs/TESTER-HOST.md b/docs/TESTER-HOST.md index 50624551..670caa74 100644 --- a/docs/TESTER-HOST.md +++ b/docs/TESTER-HOST.md @@ -74,10 +74,10 @@ The command fails before mutation unless it sees Debian 12 or newer, root, the l The runtime command is the one interface for create/update, inspect, reset, and removal: ```bash -sudo /opt/ci-fleet-tester/current/scripts/tester-runtime.sh --converge --environment example-preview -sudo /opt/ci-fleet-tester/current/scripts/tester-runtime.sh --inspect --environment example-preview -sudo /opt/ci-fleet-tester/current/scripts/tester-runtime.sh --reset --environment example-preview -sudo /opt/ci-fleet-tester/current/scripts/tester-runtime.sh --remove --environment example-preview +sudo /opt/ci-fleet-tester/tester-runtime --converge --environment example-preview +sudo /opt/ci-fleet-tester/tester-runtime --inspect --environment example-preview +sudo /opt/ci-fleet-tester/tester-runtime --reset --environment example-preview +sudo /opt/ci-fleet-tester/tester-runtime --remove --environment example-preview ``` `--converge` validates the full resolved Compose model before `up --wait`. `--reset` removes only that exact Compose project and its volumes, then recreates it from the approved definition and digest. `--remove` uses the same scoped `compose down --volumes`; no global Docker prune is used. State reports only environment/project/owner, loopback route, expiry, source revision, timestamps, and health—not Compose environment values or secret content. diff --git a/host/systemd/ci-fleet-tester-cleanup.service b/host/systemd/ci-fleet-tester-cleanup.service index ccd60125..b98d87ba 100644 --- a/host/systemd/ci-fleet-tester-cleanup.service +++ b/host/systemd/ci-fleet-tester-cleanup.service @@ -6,7 +6,7 @@ ConditionPathExists=/etc/ci-fleet-tester/tester.env [Service] Type=oneshot -ExecStart=/opt/ci-fleet-tester/current/scripts/tester-runtime.sh --cleanup +ExecStart=/opt/ci-fleet-tester/tester-runtime --cleanup User=root Group=root UMask=0077 @@ -14,7 +14,7 @@ NoNewPrivileges=yes PrivateTmp=yes ProtectHome=yes ProtectSystem=strict -ReadWritePaths=/var/lib/ci-fleet-tester /run/lock +ReadWritePaths=/var/lib/ci-fleet-tester /run/lock/ci-fleet-tester RestrictAddressFamilies=AF_UNIX LockPersonality=yes MemoryDenyWriteExecute=yes diff --git a/host/systemd/ci-fleet-tester-health.service b/host/systemd/ci-fleet-tester-health.service index 92585f1f..658f7123 100644 --- a/host/systemd/ci-fleet-tester-health.service +++ b/host/systemd/ci-fleet-tester-health.service @@ -6,7 +6,7 @@ ConditionPathExists=/etc/ci-fleet-tester/tester.env [Service] Type=oneshot -ExecStart=/opt/ci-fleet-tester/current/scripts/tester-runtime.sh --health +ExecStart=/opt/ci-fleet-tester/tester-runtime --health User=root Group=root UMask=0077 @@ -14,7 +14,7 @@ NoNewPrivileges=yes PrivateTmp=yes ProtectHome=yes ProtectSystem=strict -ReadWritePaths=/var/lib/ci-fleet-tester /run/lock +ReadWritePaths=/var/lib/ci-fleet-tester /run/lock/ci-fleet-tester RestrictAddressFamilies=AF_UNIX LockPersonality=yes MemoryDenyWriteExecute=yes diff --git a/scripts/fixtures/fake-tester-docker.sh b/scripts/fixtures/fake-tester-docker.sh index e5afbc79..f0bf3949 100755 --- a/scripts/fixtures/fake-tester-docker.sh +++ b/scripts/fixtures/fake-tester-docker.sh @@ -51,6 +51,8 @@ case ${operation:-} in external-links) service_extra=',"external_links":["other-environment-db:db"]' ;; userns-host) service_extra=',"userns_mode":"host"' ;; cgroup-host) service_extra=',"cgroup":"host"' ;; + uts-host) service_extra=',"uts":"host"' ;; + remote-logging) service_extra=',"logging":{"driver":"syslog","options":{"syslog-address":"tcp://example.invalid:514"}}' ;; interpolation) [[ -z ${TOKEN:-} ]] || service_extra=$(printf ',"command":["app","--token=%s"]' "$TOKEN") ;; valid-secret|outside-secret) secrets=$(printf '{"credential":{"file":"%s"}}' "${FAKE_TESTER_SECRET_FILE:?}") ;; esac diff --git a/scripts/install-tester.sh b/scripts/install-tester.sh index 8ae1e6bb..23047e36 100755 --- a/scripts/install-tester.sh +++ b/scripts/install-tester.sh @@ -37,6 +37,7 @@ repo_root=$(git -C "$(dirname "${BASH_SOURCE[0]}")/.." rev-parse --show-toplevel opt_dir=$(root_path /opt/ci-fleet-tester) release_dir=$opt_dir/releases current_link=$opt_dir/current +stable_launcher=$opt_dir/tester-runtime state_root=$(root_path /var/lib/ci-fleet-tester) lkg_file=$state_root/last-known-good systemd_dir=$(root_path /etc/systemd/system) @@ -47,7 +48,7 @@ secret_root=$config_root/secrets runtime_state=$state_root/environments docker_root=$(root_path /var/lib/docker) docker_socket=$(root_path /var/run/docker.sock) -runtime_lock=$(root_path /run/lock/ci-fleet-tester.lock) +runtime_lock=$(root_path /run/lock/ci-fleet-tester/runtime.lock) units=(ci-fleet-tester-health.service ci-fleet-tester-health.timer ci-fleet-tester-cleanup.service ci-fleet-tester-cleanup.timer) timers=(ci-fleet-tester-health.timer ci-fleet-tester-cleanup.timer) @@ -99,7 +100,7 @@ ensure_directories() { release_complete() { local path=$1 expected=$2 unit - [[ -d $path && ! -L $path && $(stat -c %u "$path") == "$expected_uid" && $(stat -c %a "$path") == 555 && -x $path/scripts/tester-runtime.sh && -f $path/.ci-fleet-source-revision ]] || return 1 + [[ -d $path && ! -L $path && $(stat -c %u "$path") == "$expected_uid" && $(stat -c %a "$path") == 555 && -x $path/scripts/tester-runtime.sh && -x $path/scripts/tester-launcher.sh && -f $path/.ci-fleet-source-revision ]] || return 1 [[ $(<"$path/.ci-fleet-source-revision") == "$expected" ]] || return 1 for unit in "${units[@]}"; do [[ -f $path/host/systemd/$unit ]] || return 1; done (cd "$path" && sha256sum --status -c .ci-fleet-release.sha256) || return 1 @@ -110,12 +111,12 @@ stage_release() { if [[ -e $target ]] && release_complete "$target" "$commit"; then return; fi remove_release_tree "$staging"; remove_release_tree "$replaced"; install -d -m 0755 "$staging" GIT_NO_REPLACE_OBJECTS=1 git -C "$repo_root" cat-file -e "$commit^{commit}" 2>/dev/null || die 'requested source commit is unavailable locally' - GIT_NO_REPLACE_OBJECTS=1 git -C "$repo_root" archive "$commit" scripts/tester-runtime.sh host/systemd/ci-fleet-tester-health.service host/systemd/ci-fleet-tester-health.timer host/systemd/ci-fleet-tester-cleanup.service host/systemd/ci-fleet-tester-cleanup.timer | tar -x -C "$staging" + GIT_NO_REPLACE_OBJECTS=1 git -C "$repo_root" archive "$commit" scripts/tester-runtime.sh scripts/tester-launcher.sh host/systemd/ci-fleet-tester-health.service host/systemd/ci-fleet-tester-health.timer host/systemd/ci-fleet-tester-cleanup.service host/systemd/ci-fleet-tester-cleanup.timer | tar -x -C "$staging" printf '%s\n' "$commit" >"$staging/.ci-fleet-source-revision"; chmod 0644 "$staging/.ci-fleet-source-revision" - chmod 0755 "$staging/scripts/tester-runtime.sh"; shellcheck "$staging/scripts/tester-runtime.sh"; bash -n "$staging/scripts/tester-runtime.sh" - (cd "$staging" && sha256sum scripts/tester-runtime.sh .ci-fleet-source-revision host/systemd/* >.ci-fleet-release.sha256) + chmod 0755 "$staging/scripts/tester-runtime.sh" "$staging/scripts/tester-launcher.sh"; shellcheck "$staging/scripts/tester-runtime.sh" "$staging/scripts/tester-launcher.sh"; bash -n "$staging/scripts/tester-runtime.sh" "$staging/scripts/tester-launcher.sh" + (cd "$staging" && sha256sum scripts/tester-runtime.sh scripts/tester-launcher.sh .ci-fleet-source-revision host/systemd/* >.ci-fleet-release.sha256) chmod 0444 "$staging/.ci-fleet-source-revision" "$staging/.ci-fleet-release.sha256" "$staging"/host/systemd/* - chmod 0555 "$staging" "$staging/scripts" "$staging/host" "$staging/host/systemd" "$staging/scripts/tester-runtime.sh" + chmod 0555 "$staging" "$staging/scripts" "$staging/host" "$staging/host/systemd" "$staging/scripts/tester-runtime.sh" "$staging/scripts/tester-launcher.sh" [[ ! -e $target ]] || mv -T "$target" "$replaced" if ! mv -T "$staging" "$target"; then [[ ! -e $replaced ]] || mv -T "$replaced" "$target"; die 'could not replace tester release'; fi remove_release_tree "$replaced" @@ -129,6 +130,7 @@ install_units() { } enable_timers() { systemctl enable --now "${timers[@]}" >/dev/null; } +install_launcher() { install -m 0555 "$1/scripts/tester-launcher.sh" "$stable_launcher.new" && mv -fT "$stable_launcher.new" "$stable_launcher"; } remove_units() { systemctl disable --now "${timers[@]}" >/dev/null 2>&1 || return 1 @@ -151,13 +153,13 @@ activate_release() { die 'could not quiesce tester maintenance timers; incumbent timers restored' fi ln -sfn "$target" "$current_link.new"; mv -Tf "$current_link.new" "$current_link" - if ! install_units "$target" || ! "$target/scripts/tester-runtime.sh" --check || ! "$target/scripts/tester-runtime.sh" --health || ! enable_timers; then + if ! install_launcher "$target" || ! install_units "$target" || ! "$target/scripts/tester-runtime.sh" --check || ! "$target/scripts/tester-runtime.sh" --health || ! enable_timers; then if [[ $previous =~ ^[0-9a-f]{40}$ ]] && release_complete "$release_dir/$previous" "$previous"; then ln -sfn "$release_dir/$previous" "$current_link.new"; mv -Tf "$current_link.new" "$current_link" - if ! install_units "$release_dir/$previous" || ! enable_timers; then die 'candidate activation failed and incumbent unit restore failed; incumbent link retained for recovery'; fi + if ! install_launcher "$release_dir/$previous" || ! install_units "$release_dir/$previous" || ! enable_timers; then die 'candidate activation failed and incumbent unit restore failed; launcher and incumbent link retained for recovery'; fi else remove_units || die 'candidate activation failed and fresh-install unit teardown also failed; candidate retained for recovery' - rm -f -- "$current_link" + rm -f -- "$current_link" "$stable_launcher" fi die 'candidate tester activation failed; previous release restored when available' fi @@ -191,6 +193,7 @@ case $action in secure_file "$(root_path "$config")" 600 current=$(installed_revision) || die 'tester is not installed' release_complete "$release_dir/$current" "$current" || die 'installed release is incomplete' + cmp -s "$release_dir/$current/scripts/tester-launcher.sh" "$stable_launcher" || die 'installed launcher differs from active release' for unit in "${units[@]}"; do cmp -s "$release_dir/$current/host/systemd/$unit" "$systemd_dir/$unit" || die "installed unit differs from active release: $unit"; done for timer in "${timers[@]}"; do if ! systemctl is-enabled --quiet "$timer" || ! systemctl is-active --quiet "$timer"; then die "timer is inactive: $timer"; fi @@ -215,7 +218,7 @@ case $action in ensure_directories if find "$runtime_state" -maxdepth 1 -type f -name '*.state' | grep -q .; then die 'remove every test environment before uninstalling the tester service'; fi remove_units || die 'could not stop and disable tester maintenance units' - rm -f -- "$current_link" "$lkg_file" + rm -f -- "$current_link" "$stable_launcher" "$lkg_file" [[ ${CI_FLEET_TESTING:-0} != 1 ]] || chmod -R u+w "$release_dir" rm -rf -- "$release_dir"; install -d -m 0755 "$release_dir" report 'UNINSTALL_OK preserved_config=true preserved_definitions=true preserved_secrets=true' diff --git a/scripts/test-install-tester.sh b/scripts/test-install-tester.sh index 33cec5fe..a196469d 100755 --- a/scripts/test-install-tester.sh +++ b/scripts/test-install-tester.sh @@ -10,6 +10,7 @@ fail() { printf 'FAIL %s\n' "$*" >&2; exit 1; } root=$tmp/root fake_bin=$tmp/bin mkdir -p "$fake_bin" "$root/etc/ci-fleet-tester/environments" "$root/etc/ci-fleet-tester/definitions" "$root/etc/ci-fleet-tester/secrets" "$root/var/lib/ci-fleet-tester/environments" "$root/var/lib/docker" "$root/var/lib/fake-tester-volume" "$root/var/run" "$root/etc/systemd/system" +mkdir -p "$root/run/lock"; chmod 1777 "$root/run/lock" chmod 700 "$root/etc/ci-fleet-tester" "$root/etc/ci-fleet-tester/environments" "$root/etc/ci-fleet-tester/definitions" "$root/etc/ci-fleet-tester/secrets" "$root/var/lib/ci-fleet-tester" "$root/var/lib/ci-fleet-tester/environments" printf 'ID=debian\nVERSION_ID=13\n' >"$root/etc/os-release" : >"$root/var/run/docker.sock" @@ -45,6 +46,7 @@ write_environment() { } "$runtime" --check | grep -Fq CHECK_OK || fail 'runtime preflight failed' +[[ $(stat -c %a "$root/run/lock") == 1777 && $(stat -c %a "$root/run/lock/ci-fleet-tester") == 755 ]] || fail 'runtime changed shared lock-directory permissions' if FAKE_TESTER_DOCKER_ROOT=/remote/docker "$runtime" --check >/dev/null 2>&1; then fail 'remote Docker daemon was accepted'; fi write_environment preview-a 18080 FAKE_TESTER_ROUTE_PORT=18080 "$runtime" --converge --environment preview-a | grep -Fq CONVERGED || fail 'converge failed' @@ -63,10 +65,13 @@ FAKE_TESTER_ROUTE_PORT=18080 "$runtime" --converge --environment preview-a >/dev [[ $(awk -F= '$1=="EXPIRES_AT"{print $2}' "$state") == "$original_expiry" ]] || fail 'idempotent converge extended expiration' write_environment preview-b 18080 if FAKE_TESTER_ROUTE_PORT=18080 "$runtime" --converge --environment preview-b >/dev/null 2>&1; then fail 'duplicate route port was accepted'; fi -for policy in mutable privileged bind broad-port external-network environment configs use-api-socket namespace-share false-nnp unconfined custom-volume volumes-from external-links userns-host cgroup-host custom-network replicas lifecycle-hook gpu deploy-device build; do +for policy in mutable privileged bind broad-port external-network environment configs use-api-socket namespace-share false-nnp unconfined custom-volume volumes-from external-links userns-host cgroup-host uts-host remote-logging custom-network replicas lifecycle-hook gpu deploy-device build; do write_environment "bad-$policy" 18081 if FAKE_TESTER_ROUTE_PORT=18081 FAKE_TESTER_POLICY=$policy "$runtime" --converge --environment "bad-$policy" >/dev/null 2>&1; then fail "unsafe compose policy was accepted: $policy"; fi done +write_environment bad-include 18091 +printf 'include:\n - path: /etc/passwd\nservices: {}\n' >"$root/etc/ci-fleet-tester/definitions/bad-include.yaml" +if FAKE_TESTER_ROUTE_PORT=18091 "$runtime" --converge --environment bad-include >/dev/null 2>&1; then fail 'Compose include was accepted before rendering'; fi write_environment interpolation 18090 TOKEN=must-not-render FAKE_TESTER_ROUTE_PORT=18090 FAKE_TESTER_POLICY=interpolation "$runtime" --converge --environment interpolation >/dev/null if grep -Fq must-not-render "$root/var/lib/ci-fleet-tester/environments/interpolation.compose.json"; then fail 'caller environment was interpolated into the Compose model'; fi @@ -126,6 +131,8 @@ ref=$(git -C "$repo_root" rev-parse HEAD) if DOCKER_HOST=tcp://example.invalid:2375 "$installer" --check --config /etc/ci-fleet-tester/tester.env >/dev/null 2>&1; then fail 'installer accepted a remote Docker selector'; fi "$installer" --install --config /etc/ci-fleet-tester/tester.env --ref "$ref" | grep -Fq INSTALL_OK || fail 'fresh install failed' [[ $(readlink -f "$root/opt/ci-fleet-tester/current") == "$root/opt/ci-fleet-tester/releases/$ref" ]] || fail 'current release link is wrong' +[[ -x $root/opt/ci-fleet-tester/tester-runtime ]] || fail 'stable tester launcher was not installed' +for service in ci-fleet-tester-health.service ci-fleet-tester-cleanup.service; do grep -Fq 'ExecStart=/opt/ci-fleet-tester/tester-runtime' "$root/etc/systemd/system/$service" || fail "$service bypasses stable launcher"; done "$installer" --install --config /etc/ci-fleet-tester/tester.env --ref "$ref" >/dev/null check_output=$("$installer" --check --config /etc/ci-fleet-tester/tester.env) grep -Fq CHECK_OK <<<"$check_output" || fail 'installed check failed' @@ -188,7 +195,7 @@ cp -a "$root/opt/ci-fleet-tester/releases/$ref" "$root/opt/ci-fleet-tester/relea chmod 0755 "$root/opt/ci-fleet-tester/releases/$old" chmod 0644 "$root/opt/ci-fleet-tester/releases/$old/.ci-fleet-source-revision" "$root/opt/ci-fleet-tester/releases/$old/.ci-fleet-release.sha256" printf '%s\n' "$old" >"$root/opt/ci-fleet-tester/releases/$old/.ci-fleet-source-revision" -(cd "$root/opt/ci-fleet-tester/releases/$old" && sha256sum scripts/tester-runtime.sh .ci-fleet-source-revision host/systemd/* >.ci-fleet-release.sha256) +(cd "$root/opt/ci-fleet-tester/releases/$old" && sha256sum scripts/tester-runtime.sh scripts/tester-launcher.sh .ci-fleet-source-revision host/systemd/* >.ci-fleet-release.sha256) chmod 0444 "$root/opt/ci-fleet-tester/releases/$old/.ci-fleet-source-revision" "$root/opt/ci-fleet-tester/releases/$old/.ci-fleet-release.sha256" chmod 0555 "$root/opt/ci-fleet-tester/releases/$old" printf '%s\n' "$old" >"$root/var/lib/ci-fleet-tester/last-known-good"; chmod 600 "$root/var/lib/ci-fleet-tester/last-known-good" @@ -214,6 +221,11 @@ timeout 0.2 "$installer" --reset --environment absent >/dev/null 2>&1 reset_lock_rc=$? set -e [[ $reset_lock_rc == 124 ]] || fail 'reset resolved the active release before acquiring the lifecycle lock' +set +e +timeout 0.2 "$root/opt/ci-fleet-tester/tester-runtime" --inspect --environment absent >/dev/null 2>&1 +launcher_lock_rc=$? +set -e +[[ $launcher_lock_rc == 124 ]] || fail 'stable launcher resolved the active release before acquiring the lifecycle lock' wait "$lock_pid" if FAKE_TESTER_SYSTEMCTL_FAIL='disable --now' "$installer" --uninstall --config /etc/ci-fleet-tester/tester.env >/dev/null 2>&1; then fail 'uninstall ignored systemd teardown failure'; fi [[ -L $root/opt/ci-fleet-tester/current ]] || fail 'failed uninstall removed the active release' diff --git a/scripts/tester-launcher.sh b/scripts/tester-launcher.sh new file mode 100644 index 00000000..6c8fd13a --- /dev/null +++ b/scripts/tester-launcher.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +set -Eeuo pipefail +umask 077 + +root_prefix=${CI_FLEET_ROOT_PREFIX:-} +[[ -z $root_prefix || ${CI_FLEET_TESTING:-0} == 1 ]] || { printf 'ERROR: CI_FLEET_ROOT_PREFIX is test-only\n' >&2; exit 1; } +root_path() { printf '%s%s' "$root_prefix" "$1"; } +lock_dir=$(root_path /run/lock/ci-fleet-tester) +lock_file=$lock_dir/runtime.lock +release_dir=$(root_path /opt/ci-fleet-tester/releases) +current_link=$(root_path /opt/ci-fleet-tester/current) +expected_uid=0 +[[ ${CI_FLEET_TESTING:-0} != 1 ]] || expected_uid=$(id -u) +[[ -d $lock_dir && ! -L $lock_dir && $(stat -c %u "$lock_dir") == "$expected_uid" && $(stat -c %a "$lock_dir") == 755 ]] || { printf 'ERROR: tester lock directory is unsafe\n' >&2; exit 1; } +exec 8>"$lock_file" +flock -x 8 +target=$(readlink -f "$current_link") +revision=$(basename "$target") +[[ $target == "$release_dir/$revision" && $revision =~ ^[0-9a-f]{40}$ && -x $target/scripts/tester-runtime.sh ]] || { printf 'ERROR: active tester release is invalid\n' >&2; exit 1; } +export CI_FLEET_TESTER_LOCK_FD=8 +exec "$target/scripts/tester-runtime.sh" "$@" diff --git a/scripts/tester-runtime.sh b/scripts/tester-runtime.sh index e7187ee6..dc647adc 100755 --- a/scripts/tester-runtime.sh +++ b/scripts/tester-runtime.sh @@ -13,7 +13,7 @@ environment_dir=$(root_path /etc/ci-fleet-tester/environments) definition_dir=$(root_path /etc/ci-fleet-tester/definitions) secret_root=$(root_path /etc/ci-fleet-tester/secrets) state_dir=$(root_path /var/lib/ci-fleet-tester/environments) -lock_file=$(root_path /run/lock/ci-fleet-tester.lock) +lock_file=$(root_path /run/lock/ci-fleet-tester/runtime.lock) expected_uid=0 [[ ${CI_FLEET_TESTING:-0} != 1 ]] || expected_uid=$(id -u) @@ -110,6 +110,13 @@ load_spec() { validate_compose() { local rendered=$1 empty_env variable local -a clean_environment=(env -i "PATH=$PATH" "DOCKER_HOST=$DOCKER_HOST") + if ! python3 - "$compose_file" <<'PY' +import re,sys +text=open(sys.argv[1],encoding='utf-8').read() +key=r'(?:!!str[ \t]+)?(?:include|"include"|\x27include\x27)[ \t]*:' +if re.search(r'(?m)^[ \t]*'+key,text) or re.search(r'[,{][ \t]*'+key,text): raise SystemExit('Compose include is forbidden') +PY + then return 1; fi if [[ ${CI_FLEET_TESTING:-0} == 1 ]]; then for variable in ${!FAKE_@}; do clean_environment+=("$variable=${!variable}"); done; fi empty_env=$(mktemp); chmod 600 "$empty_env" if ! "${clean_environment[@]}" docker compose --env-file "$empty_env" -p "$compose_project" -f "$compose_file" config --format json >"$rendered"; then rm -f "$empty_env"; return 1; fi @@ -124,7 +131,7 @@ image=re.compile(r'^[a-z0-9.-]+(?::[0-9]+)?/[A-Za-z0-9_./-]+@sha256:[0-9a-f]{64} ports=[] for name,service in services.items(): if not image.fullmatch(str(service.get('image',''))): raise SystemExit(f'{name}: image must use an immutable sha256 digest') - if service.get('privileged') or service.get('network_mode') or service.get('pid') or service.get('ipc') or service.get('userns_mode') or service.get('cgroup') or service.get('external_links') or service.get('post_start') or service.get('pre_stop'): raise SystemExit(f'{name}: external namespace/link/privileged lifecycle access is forbidden') + if service.get('privileged') or service.get('network_mode') or service.get('pid') or service.get('ipc') or service.get('uts') or service.get('userns_mode') or service.get('cgroup') or service.get('external_links') or service.get('logging') or service.get('post_start') or service.get('pre_stop'): raise SystemExit(f'{name}: external namespace/link/logging/privileged lifecycle access is forbidden') deploy=service.get('deploy') or {}; reservations=(deploy.get('resources') or {}).get('reservations') or {} if service.get('build') or service.get('devices') or service.get('gpus') or reservations.get('devices') or service.get('cap_add') or service.get('container_name') or service.get('hostname') or service.get('use_api_socket') or service.get('volumes_from'): raise SystemExit(f'{name}: build/device/capability/external mount/global identity is forbidden') if deploy.get('replicas',1) != 1: raise SystemExit(f'{name}: exactly one replica is required') From 7db5f25fe8eed2208f3ed6444b28bf8c369cca98 Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Sun, 16 Aug 2026 03:20:42 -0500 Subject: [PATCH 13/42] test: follow dedicated tester lifecycle lock --- scripts/test-install-tester.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/test-install-tester.sh b/scripts/test-install-tester.sh index a196469d..d311dec1 100755 --- a/scripts/test-install-tester.sh +++ b/scripts/test-install-tester.sh @@ -207,7 +207,7 @@ rollback_state_hash=$(sha256sum "$root/var/lib/ci-fleet-tester/environments/roll [[ $(sha256sum "$root/var/lib/ci-fleet-tester/environments/rollback-env.state") == "$rollback_state_hash" ]] || fail 'rollback changed active environment state' FAKE_TESTER_ROUTE_PORT=18086 "$runtime" --remove --environment rollback-env >/dev/null -lock_file=$root/run/lock/ci-fleet-tester.lock +lock_file=$root/run/lock/ci-fleet-tester/runtime.lock lock_ready=$tmp/lock-ready flock "$lock_file" -c "touch '$lock_ready'; sleep 1" & lock_pid=$! while [[ ! -e $lock_ready ]]; do kill -0 "$lock_pid" 2>/dev/null || fail 'could not acquire fixture lifecycle lock'; done From 034bf0dfd8448115f7b236daf81cae191903e0cb Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Sun, 16 Aug 2026 21:13:18 -0500 Subject: [PATCH 14/42] fix: make tester uninstall idempotent --- scripts/install-tester.sh | 8 ++++++-- scripts/test-install-tester.sh | 2 ++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/scripts/install-tester.sh b/scripts/install-tester.sh index 23047e36..229e909b 100755 --- a/scripts/install-tester.sh +++ b/scripts/install-tester.sh @@ -133,8 +133,12 @@ enable_timers() { systemctl enable --now "${timers[@]}" >/dev/null; } install_launcher() { install -m 0555 "$1/scripts/tester-launcher.sh" "$stable_launcher.new" && mv -fT "$stable_launcher.new" "$stable_launcher"; } remove_units() { - systemctl disable --now "${timers[@]}" >/dev/null 2>&1 || return 1 - local unit; for unit in "${units[@]}"; do rm -f -- "$systemd_dir/$unit" || return 1; done + local unit present=0 + for unit in "${units[@]}"; do [[ ! -e $systemd_dir/$unit ]] || present=1; done + if ((present)); then systemctl disable --now "${timers[@]}" >/dev/null 2>&1 || return 1 + else systemctl disable --now "${timers[@]}" >/dev/null 2>&1 || true + fi + for unit in "${units[@]}"; do rm -f -- "$systemd_dir/$unit" || return 1; done systemctl daemon-reload } diff --git a/scripts/test-install-tester.sh b/scripts/test-install-tester.sh index d311dec1..1f395079 100755 --- a/scripts/test-install-tester.sh +++ b/scripts/test-install-tester.sh @@ -26,6 +26,7 @@ cat >"$fake_bin/systemctl" <<'EOF' #!/usr/bin/env bash printf '%s\n' "$*" >>"${FAKE_TESTER_SYSTEMCTL_LOG:?}" [[ -z ${FAKE_TESTER_EVENT_LOG:-} ]] || printf 'systemctl %s\n' "$*" >>"$FAKE_TESTER_EVENT_LOG" +if [[ ${FAKE_TESTER_SYSTEMCTL_FAIL_IF_UNITS_MISSING:-0} == 1 && $1 == disable && ! -e ${CI_FLEET_ROOT_PREFIX:?}/etc/systemd/system/ci-fleet-tester-health.timer ]]; then exit 5; fi [[ -z ${FAKE_TESTER_SYSTEMCTL_FAIL:-} || " $* " != *" $FAKE_TESTER_SYSTEMCTL_FAIL "* ]] EOF printf '#!/usr/bin/env bash\nexit 0\n' >"$fake_bin/curl" @@ -231,6 +232,7 @@ if FAKE_TESTER_SYSTEMCTL_FAIL='disable --now' "$installer" --uninstall --config [[ -L $root/opt/ci-fleet-tester/current ]] || fail 'failed uninstall removed the active release' "$installer" --uninstall --config /etc/ci-fleet-tester/tester.env | grep -Fq UNINSTALL_OK || fail 'uninstall failed' [[ -f $root/etc/ci-fleet-tester/tester.env && ! -L $root/opt/ci-fleet-tester/current ]] || fail 'uninstall did not preserve config/remove runtime' +FAKE_TESTER_SYSTEMCTL_FAIL_IF_UNITS_MISSING=1 "$installer" --uninstall --config /etc/ci-fleet-tester/tester.env | grep -Fq UNINSTALL_OK || fail 'repeated uninstall failed on absent units' git -C "$upgrade_repo" checkout --quiet "$bad_ref" if FAKE_TESTER_SYSTEMCTL_FAIL='disable --now' "$upgrade_repo/scripts/install-tester.sh" --install --config /etc/ci-fleet-tester/tester.env --ref "$bad_ref" >/dev/null 2>&1; then fail 'invalid fresh install succeeded'; fi [[ -L $root/opt/ci-fleet-tester/current ]] || fail 'failed fresh-install teardown removed the recovery link' From d003568e9f425b6242fcf3f4b2981f0c1e533a16 Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Sun, 16 Aug 2026 21:42:10 -0500 Subject: [PATCH 15/42] fix: close final tester lifecycle gaps --- host/systemd/ci-fleet-tester-cleanup.service | 2 +- host/systemd/ci-fleet-tester-health.service | 2 +- scripts/fixtures/fake-tester-docker.sh | 5 +++-- scripts/install-tester.sh | 7 +++++-- scripts/test-install-tester.sh | 10 ++++++++++ scripts/tester-launcher.sh | 1 + scripts/tester-runtime.sh | 12 ++++++++---- 7 files changed, 29 insertions(+), 10 deletions(-) diff --git a/host/systemd/ci-fleet-tester-cleanup.service b/host/systemd/ci-fleet-tester-cleanup.service index b98d87ba..83929a92 100644 --- a/host/systemd/ci-fleet-tester-cleanup.service +++ b/host/systemd/ci-fleet-tester-cleanup.service @@ -14,7 +14,7 @@ NoNewPrivileges=yes PrivateTmp=yes ProtectHome=yes ProtectSystem=strict -ReadWritePaths=/var/lib/ci-fleet-tester /run/lock/ci-fleet-tester +ReadWritePaths=/var/lib/ci-fleet-tester -/run/lock/ci-fleet-tester RestrictAddressFamilies=AF_UNIX LockPersonality=yes MemoryDenyWriteExecute=yes diff --git a/host/systemd/ci-fleet-tester-health.service b/host/systemd/ci-fleet-tester-health.service index 658f7123..be56d1af 100644 --- a/host/systemd/ci-fleet-tester-health.service +++ b/host/systemd/ci-fleet-tester-health.service @@ -14,7 +14,7 @@ NoNewPrivileges=yes PrivateTmp=yes ProtectHome=yes ProtectSystem=strict -ReadWritePaths=/var/lib/ci-fleet-tester /run/lock/ci-fleet-tester +ReadWritePaths=/var/lib/ci-fleet-tester -/run/lock/ci-fleet-tester RestrictAddressFamilies=AF_UNIX LockPersonality=yes MemoryDenyWriteExecute=yes diff --git a/scripts/fixtures/fake-tester-docker.sh b/scripts/fixtures/fake-tester-docker.sh index f0bf3949..0e9421ef 100755 --- a/scripts/fixtures/fake-tester-docker.sh +++ b/scripts/fixtures/fake-tester-docker.sh @@ -4,14 +4,15 @@ printf '%s\n' "$*" >>"${FAKE_TESTER_DOCKER_LOG:?}" [[ -z ${FAKE_TESTER_EVENT_LOG:-} ]] || printf 'docker %s\n' "$*" >>"$FAKE_TESTER_EVENT_LOG" if [[ $1 == context && $2 == show ]]; then printf 'default\n'; exit 0; fi if [[ $1 == info ]]; then printf '%s\n' "${FAKE_TESTER_DOCKER_ROOT:?}"; exit 0; fi -if [[ $1 == ps ]]; then printf 'fixture-container-id\n'; exit 0; fi +if [[ $1 == ps ]]; then [[ ${FAKE_TESTER_PS_FAIL:-0} != 1 ]] || exit 9; printf 'fixture-container-id\n'; exit 0; fi if [[ $1 == inspect ]]; then if [[ " $* " == *' --size '* ]]; then printf '1024\n'; else printf '%s\n' "${FAKE_TESTER_CONTAINER_STATE:-running healthy}"; fi exit 0 fi -if [[ $1 == volume && $2 == ls ]]; then printf 'fixture-volume\n'; exit 0; fi +if [[ $1 == volume && $2 == ls ]]; then [[ ${FAKE_TESTER_VOLUME_LS_FAIL:-0} != 1 ]] || exit 9; printf 'fixture-volume\n'; exit 0; fi if [[ $1 == volume && $2 == inspect ]]; then printf '%s\n' "${FAKE_TESTER_VOLUME_ROOT:?}"; exit 0; fi if [[ $1 == compose && $2 == version ]]; then printf 'Docker Compose version v2.fixture\n'; exit 0; fi +if [[ $1 == compose && $2 == up && ${3:-} == --help ]]; then [[ ${FAKE_TESTER_NO_WAIT_TIMEOUT:-0} != 1 ]] && printf '%s\n' ' --wait-timeout int'; exit 0; fi if [[ $1 != compose ]]; then exit 2; fi shift project= diff --git a/scripts/install-tester.sh b/scripts/install-tester.sh index 229e909b..69551c4e 100755 --- a/scripts/install-tester.sh +++ b/scripts/install-tester.sh @@ -63,7 +63,8 @@ reject_git_replacements() { } acquire_lifecycle_lock() { - install -d -m 0755 "$(dirname "$runtime_lock")" + mkdir -m 0755 "$(dirname "$runtime_lock")" 2>/dev/null || true + secure_dir "$(dirname "$runtime_lock")" 755 exec 8>"$runtime_lock" flock -x 8 export CI_FLEET_TESTER_LOCK_FD=8 @@ -83,6 +84,8 @@ host_preflight() { [[ -S $docker_socket || ( ${CI_FLEET_TESTING:-0} == 1 && -e $docker_socket ) ]] || die 'local Docker socket is unavailable' actual_root=$(docker info --format '{{.DockerRootDir}}'); [[ $actual_root == "$docker_root" ]] || die 'Docker root does not match the local managed root' docker compose version >/dev/null + printf 'services: {}\n' | docker compose -f - config --format json >/dev/null || die 'Compose JSON rendering is unavailable' + docker compose up --help | grep -q -- '--wait-timeout' || die 'Compose wait-timeout support is unavailable' used=$(df -P "$docker_root" | awk 'NR==2{gsub(/%/,"",$5);print $5}') [[ $used =~ ^[0-9]+$ && $used -private-repository 80 ]] || die 'Docker storage is at or above 80%' } @@ -152,6 +155,7 @@ activate_release() { local commit=$1 target=$release_dir/$1 previous= release_complete "$target" "$commit" || die 'candidate tester release is incomplete' [[ ! -L $current_link ]] || previous=$(basename "$(readlink -f "$current_link")") + if [[ $previous =~ ^[0-9a-f]{40}$ && $previous != "$commit" ]] && release_complete "$release_dir/$previous" "$previous"; then write_lkg "$previous"; fi if [[ $previous =~ ^[0-9a-f]{40}$ ]] && ! systemctl disable --now "${timers[@]}" >/dev/null; then enable_timers || die 'could not quiesce or restore tester maintenance timers' die 'could not quiesce tester maintenance timers; incumbent timers restored' @@ -167,7 +171,6 @@ activate_release() { fi die 'candidate tester activation failed; previous release restored when available' fi - if [[ $previous =~ ^[0-9a-f]{40}$ && $previous != "$commit" ]] && release_complete "$release_dir/$previous" "$previous"; then write_lkg "$previous"; fi report "INSTALL_OK source_revision=$commit previous_revision=${previous:-none} config=$config" } diff --git a/scripts/test-install-tester.sh b/scripts/test-install-tester.sh index 1f395079..bbdcca2d 100755 --- a/scripts/test-install-tester.sh +++ b/scripts/test-install-tester.sh @@ -48,6 +48,10 @@ write_environment() { "$runtime" --check | grep -Fq CHECK_OK || fail 'runtime preflight failed' [[ $(stat -c %a "$root/run/lock") == 1777 && $(stat -c %a "$root/run/lock/ci-fleet-tester") == 755 ]] || fail 'runtime changed shared lock-directory permissions' +rm -rf "$root/run/lock/ci-fleet-tester"; mkdir "$tmp/lock-target"; chmod 700 "$tmp/lock-target"; ln -s "$tmp/lock-target" "$root/run/lock/ci-fleet-tester" +if "$runtime" --check >/dev/null 2>&1; then fail 'symlinked lifecycle lock directory was accepted'; fi +[[ $(stat -c %a "$tmp/lock-target") == 700 ]] || fail 'symlinked lock target permissions changed' +rm "$root/run/lock/ci-fleet-tester" if FAKE_TESTER_DOCKER_ROOT=/remote/docker "$runtime" --check >/dev/null 2>&1; then fail 'remote Docker daemon was accepted'; fi write_environment preview-a 18080 FAKE_TESTER_ROUTE_PORT=18080 "$runtime" --converge --environment preview-a | grep -Fq CONVERGED || fail 'converge failed' @@ -55,6 +59,8 @@ state=$root/var/lib/ci-fleet-tester/environments/preview-a.state [[ -f $state && $(stat -c %a "$state") == 600 ]] || fail 'state was not protected' inspect_output=$(FAKE_TESTER_ROUTE_PORT=18080 "$runtime" --inspect --environment preview-a) grep -q 'IMAGE_DIGESTS=sha256:[a-f0-9]\{64\}.*STATUS=running DISK_BYTES=[1-9][0-9]*' <<<"$inspect_output" || fail 'inspect did not report health and disk use' +if FAKE_TESTER_PS_FAIL=1 "$runtime" --inspect --environment preview-a >/dev/null 2>&1; then fail 'container inventory failure was hidden'; fi +if FAKE_TESTER_VOLUME_LS_FAIL=1 "$runtime" --inspect --environment preview-a >/dev/null 2>&1; then fail 'volume inventory failure was hidden'; fi deployed_inode=$(stat -c %i "$root/var/lib/ci-fleet-tester/environments/preview-a.compose.json") FAKE_TESTER_ROUTE_PORT=18080 "$runtime" --converge --environment preview-a >/dev/null [[ $(find "$root/var/lib/ci-fleet-tester/environments" -name '*.state' | wc -l) == 1 ]] || fail 'idempotent converge duplicated state' @@ -73,6 +79,8 @@ done write_environment bad-include 18091 printf 'include:\n - path: /etc/passwd\nservices: {}\n' >"$root/etc/ci-fleet-tester/definitions/bad-include.yaml" if FAKE_TESTER_ROUTE_PORT=18091 "$runtime" --converge --environment bad-include >/dev/null 2>&1; then fail 'Compose include was accepted before rendering'; fi +printf '"incl\\u0075de": [{path: /etc/passwd}]\nservices: {}\n' >"$root/etc/ci-fleet-tester/definitions/bad-include.yaml" +if FAKE_TESTER_ROUTE_PORT=18091 "$runtime" --converge --environment bad-include >/dev/null 2>&1; then fail 'escaped Compose include was accepted'; fi write_environment interpolation 18090 TOKEN=must-not-render FAKE_TESTER_ROUTE_PORT=18090 FAKE_TESTER_POLICY=interpolation "$runtime" --converge --environment interpolation >/dev/null if grep -Fq must-not-render "$root/var/lib/ci-fleet-tester/environments/interpolation.compose.json"; then fail 'caller environment was interpolated into the Compose model'; fi @@ -131,6 +139,8 @@ done ref=$(git -C "$repo_root" rev-parse HEAD) if DOCKER_HOST=tcp://example.invalid:2375 "$installer" --check --config /etc/ci-fleet-tester/tester.env >/dev/null 2>&1; then fail 'installer accepted a remote Docker selector'; fi "$installer" --install --config /etc/ci-fleet-tester/tester.env --ref "$ref" | grep -Fq INSTALL_OK || fail 'fresh install failed' +rm -rf "$root/run/lock/ci-fleet-tester" +"$root/opt/ci-fleet-tester/tester-runtime" --health >/dev/null || fail 'stable launcher did not recreate the volatile lock directory' [[ $(readlink -f "$root/opt/ci-fleet-tester/current") == "$root/opt/ci-fleet-tester/releases/$ref" ]] || fail 'current release link is wrong' [[ -x $root/opt/ci-fleet-tester/tester-runtime ]] || fail 'stable tester launcher was not installed' for service in ci-fleet-tester-health.service ci-fleet-tester-cleanup.service; do grep -Fq 'ExecStart=/opt/ci-fleet-tester/tester-runtime' "$root/etc/systemd/system/$service" || fail "$service bypasses stable launcher"; done diff --git a/scripts/tester-launcher.sh b/scripts/tester-launcher.sh index 6c8fd13a..e5c2987b 100644 --- a/scripts/tester-launcher.sh +++ b/scripts/tester-launcher.sh @@ -11,6 +11,7 @@ release_dir=$(root_path /opt/ci-fleet-tester/releases) current_link=$(root_path /opt/ci-fleet-tester/current) expected_uid=0 [[ ${CI_FLEET_TESTING:-0} != 1 ]] || expected_uid=$(id -u) +mkdir -m 0755 "$lock_dir" 2>/dev/null || true [[ -d $lock_dir && ! -L $lock_dir && $(stat -c %u "$lock_dir") == "$expected_uid" && $(stat -c %a "$lock_dir") == 755 ]] || { printf 'ERROR: tester lock directory is unsafe\n' >&2; exit 1; } exec 8>"$lock_file" flock -x 8 diff --git a/scripts/tester-runtime.sh b/scripts/tester-runtime.sh index dc647adc..b386d62a 100755 --- a/scripts/tester-runtime.sh +++ b/scripts/tester-runtime.sh @@ -114,7 +114,8 @@ validate_compose() { import re,sys text=open(sys.argv[1],encoding='utf-8').read() key=r'(?:!!str[ \t]+)?(?:include|"include"|\x27include\x27)[ \t]*:' -if re.search(r'(?m)^[ \t]*'+key,text) or re.search(r'[,{][ \t]*'+key,text): raise SystemExit('Compose include is forbidden') +escaped_key=r'"[^"\n]*\\[^"\n]*"[ \t]*:' +if re.search(r'(?m)^[ \t]*'+key,text) or re.search(r'[,{][ \t]*'+key,text) or re.search(escaped_key,text): raise SystemExit('Compose include or escaped mapping key is forbidden') PY then return 1; fi if [[ ${CI_FLEET_TESTING:-0} == 1 ]]; then for variable in ${!FAKE_@}; do clean_environment+=("$variable=${!variable}"); done; fi @@ -239,14 +240,16 @@ inspect_environment() { running_state=$(docker inspect --format '{{.State.Status}} {{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$resource") [[ $running_state == 'running healthy' || $running_state == 'running none' ]] || status=unhealthy done + inventory=$(docker ps -aq --filter "label=com.docker.compose.project=$(project_name "$id")") || die 'container inventory failed' while IFS= read -r resource; do [[ -n $resource ]] || continue value=$(docker inspect --size --format '{{.SizeRw}}' "$resource"); [[ $value =~ ^[0-9]+$ ]] || die 'container disk size is invalid'; bytes=$((bytes + value)) - done < <(docker ps -aq --filter "label=com.docker.compose.project=$(project_name "$id")") + done <<<"$inventory" + inventory=$(docker volume ls -q --filter "label=com.docker.compose.project=$(project_name "$id")") || die 'volume inventory failed' while IFS= read -r resource; do [[ -n $resource ]] || continue mount=$(docker volume inspect --format '{{.Mountpoint}}' "$resource"); value=$(du -sb "$mount" | awk '{print $1}'); [[ $value =~ ^[0-9]+$ ]] || die 'volume disk size is invalid'; bytes=$((bytes + value)) - done < <(docker volume ls -q --filter "label=com.docker.compose.project=$(project_name "$id")") + done <<<"$inventory" printf 'STATUS=%s DISK_BYTES=%s\n' "$status" "$bytes" } @@ -263,7 +266,8 @@ export DOCKER_HOST="unix://$docker_socket" if [[ ${CI_FLEET_TESTER_LOCK_FD:-} == 8 && -e /proc/$$/fd/8 && $(readlink -f /proc/$$/fd/8) == "$lock_file" ]] && flock -n 8; then unset CI_FLEET_TESTER_LOCK_FD else - install -d -m 0755 "$(dirname "$lock_file")" + mkdir -m 0755 "$(dirname "$lock_file")" 2>/dev/null || true + secure_directory "$(dirname "$lock_file")" 755 exec 9>"$lock_file" flock -x 9 fi From 4be273eb0e38d159ed5c7feca01bfee2e1fa3458 Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Sun, 16 Aug 2026 21:44:17 -0500 Subject: [PATCH 16/42] fix: restore tester when rollback record fails --- scripts/install-tester.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/install-tester.sh b/scripts/install-tester.sh index 69551c4e..1c4463e7 100755 --- a/scripts/install-tester.sh +++ b/scripts/install-tester.sh @@ -155,7 +155,6 @@ activate_release() { local commit=$1 target=$release_dir/$1 previous= release_complete "$target" "$commit" || die 'candidate tester release is incomplete' [[ ! -L $current_link ]] || previous=$(basename "$(readlink -f "$current_link")") - if [[ $previous =~ ^[0-9a-f]{40}$ && $previous != "$commit" ]] && release_complete "$release_dir/$previous" "$previous"; then write_lkg "$previous"; fi if [[ $previous =~ ^[0-9a-f]{40}$ ]] && ! systemctl disable --now "${timers[@]}" >/dev/null; then enable_timers || die 'could not quiesce or restore tester maintenance timers' die 'could not quiesce tester maintenance timers; incumbent timers restored' @@ -171,6 +170,11 @@ activate_release() { fi die 'candidate tester activation failed; previous release restored when available' fi + if [[ $previous =~ ^[0-9a-f]{40}$ && $previous != "$commit" ]] && release_complete "$release_dir/$previous" "$previous" && ! write_lkg "$previous"; then + ln -sfn "$release_dir/$previous" "$current_link.new"; mv -Tf "$current_link.new" "$current_link" + if ! install_launcher "$release_dir/$previous" || ! install_units "$release_dir/$previous" || ! enable_timers; then die 'last-known-good recording failed and incumbent restore failed; incumbent link retained for recovery'; fi + die 'last-known-good recording failed; previous release restored' + fi report "INSTALL_OK source_revision=$commit previous_revision=${previous:-none} config=$config" } From 25b4306cf1d346aa9bc373a9c8547f6fc20f5e4e Mon Sep 17 00:00:00 2001 From: Nickfost <1572453+Nickfost@users.noreply.github.com> Date: Thu, 27 Aug 2026 22:14:43 -0500 Subject: [PATCH 17/42] fix: repair PR #76 rewrite corruption and resolve 8 Codex findings - restore 4 numeric comparisons corrupted to -private-repository (bash syntax errors that broke exact-head CI) back to -lt, with a focused regression - reject explicit-key Compose includes (? "include":) alongside inline/escaped - reject Compose profiles, label_file, and custom network IPAM in the validator - ship tmpfiles.d drop-in so the volatile tester lock dir is reboot-creatable - make remove_units tolerate a partial unit set (interrupted install) - reject symlinked/non-regular release archive members before chmod dereferences - bound maintenance oneshot execution with TimeoutStartSec=300 --- host/systemd/ci-fleet-tester-cleanup.service | 1 + host/systemd/ci-fleet-tester-health.service | 1 + host/systemd/ci-fleet-tester-lock.conf | 4 ++ scripts/fixtures/fake-tester-docker.sh | 3 ++ scripts/install-tester.sh | 41 +++++++++++---- scripts/test-install-tester.sh | 41 ++++++++++++++- scripts/test-numeric-regressions.sh | 53 ++++++++++++++++++++ scripts/tester-runtime.sh | 10 ++-- 8 files changed, 140 insertions(+), 14 deletions(-) create mode 100644 host/systemd/ci-fleet-tester-lock.conf create mode 100644 scripts/test-numeric-regressions.sh diff --git a/host/systemd/ci-fleet-tester-cleanup.service b/host/systemd/ci-fleet-tester-cleanup.service index 83929a92..9f970274 100644 --- a/host/systemd/ci-fleet-tester-cleanup.service +++ b/host/systemd/ci-fleet-tester-cleanup.service @@ -15,6 +15,7 @@ PrivateTmp=yes ProtectHome=yes ProtectSystem=strict ReadWritePaths=/var/lib/ci-fleet-tester -/run/lock/ci-fleet-tester +TimeoutStartSec=300 RestrictAddressFamilies=AF_UNIX LockPersonality=yes MemoryDenyWriteExecute=yes diff --git a/host/systemd/ci-fleet-tester-health.service b/host/systemd/ci-fleet-tester-health.service index be56d1af..7440d67c 100644 --- a/host/systemd/ci-fleet-tester-health.service +++ b/host/systemd/ci-fleet-tester-health.service @@ -15,6 +15,7 @@ PrivateTmp=yes ProtectHome=yes ProtectSystem=strict ReadWritePaths=/var/lib/ci-fleet-tester -/run/lock/ci-fleet-tester +TimeoutStartSec=300 RestrictAddressFamilies=AF_UNIX LockPersonality=yes MemoryDenyWriteExecute=yes diff --git a/host/systemd/ci-fleet-tester-lock.conf b/host/systemd/ci-fleet-tester-lock.conf new file mode 100644 index 00000000..c8a4be4f --- /dev/null +++ b/host/systemd/ci-fleet-tester-lock.conf @@ -0,0 +1,4 @@ +# Recreate the volatile tester lifecycle lock directory at boot and on +# tmpfiles --create, so the maintenance units and stable launcher can open it +# after a reboot even though /run is cleared. +d /run/lock/ci-fleet-tester 0755 root root - diff --git a/scripts/fixtures/fake-tester-docker.sh b/scripts/fixtures/fake-tester-docker.sh index 0e9421ef..4ed79958 100755 --- a/scripts/fixtures/fake-tester-docker.sh +++ b/scripts/fixtures/fake-tester-docker.sh @@ -42,6 +42,7 @@ case ${operation:-} in custom-volume) volume_extra=',"driver":"local"' ;; volumes-from) service_extra=',"volumes_from":["container:other:rw"]' ;; custom-network) network_extra=',"driver":"macvlan","driver_opts":{"parent":"eth0"}' ;; + ipam) network_extra=',"ipam":{"config":[{"subnet":"172.30.0.0/24"}]}' ;; replicas) service_extra=',"deploy":{"replicas":2}' ;; lifecycle-hook) service_extra=',"post_start":[{"command":"true","privileged":true}]' ;; gpu) service_extra=',"gpus":"all"' ;; @@ -55,6 +56,8 @@ case ${operation:-} in uts-host) service_extra=',"uts":"host"' ;; remote-logging) service_extra=',"logging":{"driver":"syslog","options":{"syslog-address":"tcp://example.invalid:514"}}' ;; interpolation) [[ -z ${TOKEN:-} ]] || service_extra=$(printf ',"command":["app","--token=%s"]' "$TOKEN") ;; + profiles) service_extra=',"profiles":["extra"]' ;; + label-file) service_extra=',"label_file":"/root/credential.env"' ;; valid-secret|outside-secret) secrets=$(printf '{"credential":{"file":"%s"}}' "${FAKE_TESTER_SECRET_FILE:?}") ;; esac volume=${volume:-'{"type":"volume","source":"data","target":"/data"}'} diff --git a/scripts/install-tester.sh b/scripts/install-tester.sh index 1c4463e7..fe3ccc55 100755 --- a/scripts/install-tester.sh +++ b/scripts/install-tester.sh @@ -51,6 +51,8 @@ docker_socket=$(root_path /var/run/docker.sock) runtime_lock=$(root_path /run/lock/ci-fleet-tester/runtime.lock) units=(ci-fleet-tester-health.service ci-fleet-tester-health.timer ci-fleet-tester-cleanup.service ci-fleet-tester-cleanup.timer) timers=(ci-fleet-tester-health.timer ci-fleet-tester-cleanup.timer) +tmpfiles_conf=ci-fleet-tester-lock.conf +tmpfiles_dir=$(root_path /usr/lib/tmpfiles.d) secure_file() { [[ -f $1 && ! -L $1 && $(stat -c %u "$1") == "$expected_uid" && $(stat -c %a "$1") == "$2" ]] || die "protected file is unsafe: $1"; } secure_dir() { [[ -d $1 && ! -L $1 && $(stat -c %u "$1") == "$expected_uid" && $(stat -c %a "$1") == "$2" ]] || die "protected directory is unsafe: $1"; } @@ -87,7 +89,7 @@ host_preflight() { printf 'services: {}\n' | docker compose -f - config --format json >/dev/null || die 'Compose JSON rendering is unavailable' docker compose up --help | grep -q -- '--wait-timeout' || die 'Compose wait-timeout support is unavailable' used=$(df -P "$docker_root" | awk 'NR==2{gsub(/%/,"",$5);print $5}') - [[ $used =~ ^[0-9]+$ && $used -private-repository 80 ]] || die 'Docker storage is at or above 80%' + [[ $used =~ ^[0-9]+$ && $used -lt 80 ]] || die 'Docker storage is at or above 80%' } ensure_directories() { @@ -114,7 +116,12 @@ stage_release() { if [[ -e $target ]] && release_complete "$target" "$commit"; then return; fi remove_release_tree "$staging"; remove_release_tree "$replaced"; install -d -m 0755 "$staging" GIT_NO_REPLACE_OBJECTS=1 git -C "$repo_root" cat-file -e "$commit^{commit}" 2>/dev/null || die 'requested source commit is unavailable locally' - GIT_NO_REPLACE_OBJECTS=1 git -C "$repo_root" archive "$commit" scripts/tester-runtime.sh scripts/tester-launcher.sh host/systemd/ci-fleet-tester-health.service host/systemd/ci-fleet-tester-health.timer host/systemd/ci-fleet-tester-cleanup.service host/systemd/ci-fleet-tester-cleanup.timer | tar -x -C "$staging" + GIT_NO_REPLACE_OBJECTS=1 git -C "$repo_root" archive "$commit" scripts/tester-runtime.sh scripts/tester-launcher.sh host/systemd/ci-fleet-tester-health.service host/systemd/ci-fleet-tester-health.timer host/systemd/ci-fleet-tester-cleanup.service host/systemd/ci-fleet-tester-cleanup.timer host/systemd/ci-fleet-tester-lock.conf | tar -x -C "$staging" + # Reject any non-regular or symlinked member before chmod can dereference it + # (chmod follows symlinks and would change the host target's permissions). + while IFS= read -r -d '' entry; do + if [[ -L $entry || ! -f $entry ]]; then remove_release_tree "$staging"; die "release archive contains a non-regular or symlinked member: $entry"; fi + done < <(find "$staging" -type f -print0) printf '%s\n' "$commit" >"$staging/.ci-fleet-source-revision"; chmod 0644 "$staging/.ci-fleet-source-revision" chmod 0755 "$staging/scripts/tester-runtime.sh" "$staging/scripts/tester-launcher.sh"; shellcheck "$staging/scripts/tester-runtime.sh" "$staging/scripts/tester-launcher.sh"; bash -n "$staging/scripts/tester-runtime.sh" "$staging/scripts/tester-launcher.sh" (cd "$staging" && sha256sum scripts/tester-runtime.sh scripts/tester-launcher.sh .ci-fleet-source-revision host/systemd/* >.ci-fleet-release.sha256) @@ -129,19 +136,35 @@ install_units() { local source=$1 unit for unit in "${units[@]}"; do install -m 0644 "$source/host/systemd/$unit" "$systemd_dir/$unit.new" || return 1; done for unit in "${units[@]}"; do mv -fT "$systemd_dir/$unit.new" "$systemd_dir/$unit" || return 1; done + install -d -m 0755 "$tmpfiles_dir" + install -m 0644 "$source/host/systemd/$tmpfiles_conf" "$tmpfiles_dir/$tmpfiles_conf.new" || return 1 + mv -fT "$tmpfiles_dir/$tmpfiles_conf.new" "$tmpfiles_dir/$tmpfiles_conf" || return 1 systemctl daemon-reload || return 1 } +create_tmpfiles() { + # Recreate the volatile tester lock directory at install time (and at boot via + # the installed tmpfiles.d drop-in) so maintenance units work after a reboot. + mkdir -m 0755 "$(dirname "$runtime_lock")" 2>/dev/null || true + secure_dir "$(dirname "$runtime_lock")" 755 + if command -v systemd-tmpfiles >/dev/null 2>&1; then systemd-tmpfiles --create "$tmpfiles_dir/$tmpfiles_conf" 2>/dev/null || true; fi +} + enable_timers() { systemctl enable --now "${timers[@]}" >/dev/null; } install_launcher() { install -m 0555 "$1/scripts/tester-launcher.sh" "$stable_launcher.new" && mv -fT "$stable_launcher.new" "$stable_launcher"; } remove_units() { - local unit present=0 - for unit in "${units[@]}"; do [[ ! -e $systemd_dir/$unit ]] || present=1; done - if ((present)); then systemctl disable --now "${timers[@]}" >/dev/null 2>&1 || return 1 - else systemctl disable --now "${timers[@]}" >/dev/null 2>&1 || true + local unit present_units=() present_timers=() + for unit in "${units[@]}"; do + [[ ! -e $systemd_dir/$unit ]] || present_units+=("$unit") + done + for unit in "${timers[@]}"; do + [[ ! -e $systemd_dir/$unit ]] || present_timers+=("$unit") + done + if (( ${#present_timers[@]} )); then + systemctl disable --now "${present_timers[@]}" >/dev/null 2>&1 || return 1 fi - for unit in "${units[@]}"; do rm -f -- "$systemd_dir/$unit" || return 1; done + for unit in "${present_units[@]}"; do rm -f -- "$systemd_dir/$unit" || return 1; done systemctl daemon-reload } @@ -160,10 +183,10 @@ activate_release() { die 'could not quiesce tester maintenance timers; incumbent timers restored' fi ln -sfn "$target" "$current_link.new"; mv -Tf "$current_link.new" "$current_link" - if ! install_launcher "$target" || ! install_units "$target" || ! "$target/scripts/tester-runtime.sh" --check || ! "$target/scripts/tester-runtime.sh" --health || ! enable_timers; then + if ! install_launcher "$target" || ! install_units "$target" || ! create_tmpfiles || ! "$target/scripts/tester-runtime.sh" --check || ! "$target/scripts/tester-runtime.sh" --health || ! enable_timers; then if [[ $previous =~ ^[0-9a-f]{40}$ ]] && release_complete "$release_dir/$previous" "$previous"; then ln -sfn "$release_dir/$previous" "$current_link.new"; mv -Tf "$current_link.new" "$current_link" - if ! install_launcher "$release_dir/$previous" || ! install_units "$release_dir/$previous" || ! enable_timers; then die 'candidate activation failed and incumbent unit restore failed; launcher and incumbent link retained for recovery'; fi + if ! install_launcher "$release_dir/$previous" || ! install_units "$release_dir/$previous" || ! create_tmpfiles || ! enable_timers; then die 'candidate activation failed and incumbent unit restore failed; launcher and incumbent link retained for recovery'; fi else remove_units || die 'candidate activation failed and fresh-install unit teardown also failed; candidate retained for recovery' rm -f -- "$current_link" "$stable_launcher" diff --git a/scripts/test-install-tester.sh b/scripts/test-install-tester.sh index bbdcca2d..b7a963e8 100755 --- a/scripts/test-install-tester.sh +++ b/scripts/test-install-tester.sh @@ -72,7 +72,7 @@ FAKE_TESTER_ROUTE_PORT=18080 "$runtime" --converge --environment preview-a >/dev [[ $(awk -F= '$1=="EXPIRES_AT"{print $2}' "$state") == "$original_expiry" ]] || fail 'idempotent converge extended expiration' write_environment preview-b 18080 if FAKE_TESTER_ROUTE_PORT=18080 "$runtime" --converge --environment preview-b >/dev/null 2>&1; then fail 'duplicate route port was accepted'; fi -for policy in mutable privileged bind broad-port external-network environment configs use-api-socket namespace-share false-nnp unconfined custom-volume volumes-from external-links userns-host cgroup-host uts-host remote-logging custom-network replicas lifecycle-hook gpu deploy-device build; do +for policy in mutable privileged bind broad-port external-network environment configs use-api-socket namespace-share false-nnp unconfined custom-volume volumes-from external-links userns-host cgroup-host uts-host remote-logging custom-network ipam replicas lifecycle-hook gpu deploy-device build profiles label-file; do write_environment "bad-$policy" 18081 if FAKE_TESTER_ROUTE_PORT=18081 FAKE_TESTER_POLICY=$policy "$runtime" --converge --environment "bad-$policy" >/dev/null 2>&1; then fail "unsafe compose policy was accepted: $policy"; fi done @@ -81,6 +81,10 @@ printf 'include:\n - path: /etc/passwd\nservices: {}\n' >"$root/etc/ci-fleet-te if FAKE_TESTER_ROUTE_PORT=18091 "$runtime" --converge --environment bad-include >/dev/null 2>&1; then fail 'Compose include was accepted before rendering'; fi printf '"incl\\u0075de": [{path: /etc/passwd}]\nservices: {}\n' >"$root/etc/ci-fleet-tester/definitions/bad-include.yaml" if FAKE_TESTER_ROUTE_PORT=18091 "$runtime" --converge --environment bad-include >/dev/null 2>&1; then fail 'escaped Compose include was accepted'; fi +printf '? "include":\n - path: /etc/passwd\nservices: {}\n' >"$root/etc/ci-fleet-tester/definitions/bad-include.yaml" +if FAKE_TESTER_ROUTE_PORT=18091 "$runtime" --converge --environment bad-include >/dev/null 2>&1; then fail 'explicit-key Compose include was accepted'; fi +printf "? 'include':\n - path: /etc/passwd\nservices: {}\n" >"$root/etc/ci-fleet-tester/definitions/bad-include.yaml" +if FAKE_TESTER_ROUTE_PORT=18091 "$runtime" --converge --environment bad-include >/dev/null 2>&1; then fail 'single-quoted explicit-key Compose include was accepted'; fi write_environment interpolation 18090 TOKEN=must-not-render FAKE_TESTER_ROUTE_PORT=18090 FAKE_TESTER_POLICY=interpolation "$runtime" --converge --environment interpolation >/dev/null if grep -Fq must-not-render "$root/var/lib/ci-fleet-tester/environments/interpolation.compose.json"; then fail 'caller environment was interpolated into the Compose model'; fi @@ -137,6 +141,23 @@ done # Commit-backed installer tests run after the implementation commit exists. ref=$(git -C "$repo_root" rev-parse HEAD) +# A release archive containing a symlinked or non-regular member must be rejected +# before chmod can dereference it and alter a host-side target's permissions. +symlink_repo=$tmp/symlink-repo +git init --quiet "$symlink_repo" +git -C "$symlink_repo" config user.name Example +git -C "$symlink_repo" config user.email example@invalid.example +mkdir -p "$symlink_repo/scripts" "$symlink_repo/host/systemd" +cp "$repo_root/scripts/tester-runtime.sh" "$symlink_repo/scripts/tester-runtime.sh" +cp "$repo_root/scripts/tester-launcher.sh" "$symlink_repo/scripts/tester-launcher.sh" +cp "$repo_root"/host/systemd/ci-fleet-tester-*.service "$repo_root"/host/systemd/ci-fleet-tester-*.timer "$symlink_repo/host/systemd/" +printf '#!/usr/bin/env bash\nexit 0\n' >"$symlink_repo/scripts/host-evil"; chmod 0755 "$symlink_repo/scripts/host-evil" +rm -f "$symlink_repo/scripts/tester-runtime.sh" +ln -s host-evil "$symlink_repo/scripts/tester-runtime.sh" +git -C "$symlink_repo" add -A +git -C "$symlink_repo" commit --quiet -m 'fixture: symlinked release member' +symlink_ref=$(git -C "$symlink_repo" rev-parse HEAD) +if "$installer" --install --config /etc/ci-fleet-tester/tester.env --ref "$symlink_ref" >/dev/null 2>&1; then fail 'install accepted a release archive with a symlinked member'; fi if DOCKER_HOST=tcp://example.invalid:2375 "$installer" --check --config /etc/ci-fleet-tester/tester.env >/dev/null 2>&1; then fail 'installer accepted a remote Docker selector'; fi "$installer" --install --config /etc/ci-fleet-tester/tester.env --ref "$ref" | grep -Fq INSTALL_OK || fail 'fresh install failed' rm -rf "$root/run/lock/ci-fleet-tester" @@ -148,7 +169,17 @@ for service in ci-fleet-tester-health.service ci-fleet-tester-cleanup.service; d check_output=$("$installer" --check --config /etc/ci-fleet-tester/tester.env) grep -Fq CHECK_OK <<<"$check_output" || fail 'installed check failed' for unit in ci-fleet-tester-health.service ci-fleet-tester-health.timer ci-fleet-tester-cleanup.service ci-fleet-tester-cleanup.timer; do [[ -f $root/etc/systemd/system/$unit ]] || fail "unit missing: $unit"; done +for service in ci-fleet-tester-health.service ci-fleet-tester-cleanup.service; do + grep -Fq 'TimeoutStartSec=300' "$root/etc/systemd/system/$service" || fail "$service does not bound oneshot start time" +done release=$root/opt/ci-fleet-tester/releases/$ref +# Reboot-creatable lock path: the tmpfiles.d drop-in ships in the release and +# recreates the volatile lock directory, so maintenance units survive a reboot. +[[ -f $release/host/systemd/ci-fleet-tester-lock.conf ]] || fail 'tmpfiles.d lock drop-in is missing from the release' +grep -Eq '^d /run/lock/ci-fleet-tester 0755' "$release/host/systemd/ci-fleet-tester-lock.conf" || fail 'tmpfiles.d drop-in does not recreate the tester lock directory' +rm -rf "$root/run/lock/ci-fleet-tester" +if "$release/scripts/tester-runtime.sh" --health >/dev/null 2>&1; then :; fi +[[ -d $root/run/lock/ci-fleet-tester && ! -L $root/run/lock/ci-fleet-tester && $(stat -c %a "$root/run/lock/ci-fleet-tester") == 755 ]] || fail 'stable launcher did not recreate the volatile lock directory' chmod u+w "$release/scripts/tester-runtime.sh"; printf '# tamper\n' >>"$release/scripts/tester-runtime.sh"; chmod 0555 "$release/scripts/tester-runtime.sh" if "$installer" --check --config /etc/ci-fleet-tester/tester.env >/dev/null 2>&1; then fail 'tampered installed release passed check'; fi "$installer" --install --config /etc/ci-fleet-tester/tester.env --ref "$ref" >/dev/null || fail 'same-ref install did not repair an incomplete release' @@ -198,7 +229,7 @@ grep -Fq 'enable --now ci-fleet-tester-health.timer ci-fleet-tester-cleanup.time disable_line=$(grep -n '^systemctl disable --now ' "$tmp/events.log" | tail -1 | cut -d: -f1) check_line=$(grep -n '^docker info ' "$tmp/events.log" | tail -1 | cut -d: -f1) enable_line=$(grep -n '^systemctl enable --now ' "$tmp/events.log" | tail -1 | cut -d: -f1) -[[ -n $disable_line && -n $check_line && -n $enable_line && $disable_line -private-repository $check_line && $check_line -private-repository $enable_line ]] || fail 'timers were not quiesced until candidate validation completed' +[[ -n $disable_line && -n $check_line && -n $enable_line && $disable_line -lt $check_line && $check_line -lt $enable_line ]] || fail 'timers were not quiesced until candidate validation completed' # Rollback switches only to a complete recorded release and keeps environments intact. old=0000000000000000000000000000000000000000 @@ -247,4 +278,10 @@ git -C "$upgrade_repo" checkout --quiet "$bad_ref" if FAKE_TESTER_SYSTEMCTL_FAIL='disable --now' "$upgrade_repo/scripts/install-tester.sh" --install --config /etc/ci-fleet-tester/tester.env --ref "$bad_ref" >/dev/null 2>&1; then fail 'invalid fresh install succeeded'; fi [[ -L $root/opt/ci-fleet-tester/current ]] || fail 'failed fresh-install teardown removed the recovery link' "$upgrade_repo/scripts/install-tester.sh" --uninstall --config /etc/ci-fleet-tester/tester.env >/dev/null +# Partial unit set (interrupted install) must still uninstall cleanly rather +# than failing because a timer unit file is missing. +"$installer" --install --config /etc/ci-fleet-tester/tester.env --ref "$ref" >/dev/null || fail 'fresh install failed before partial-unit test' +rm -f "$root/etc/systemd/system/ci-fleet-tester-cleanup.timer" +"$installer" --uninstall --config /etc/ci-fleet-tester/tester.env | grep -Fq UNINSTALL_OK || fail 'uninstall failed on a partial unit set' +[[ -L $root/opt/ci-fleet-tester/current ]] && fail 'partial uninstall left the active release link' printf 'TESTER_INSTALLER_TESTS_OK\n' diff --git a/scripts/test-numeric-regressions.sh b/scripts/test-numeric-regressions.sh new file mode 100644 index 00000000..2d0e8ce5 --- /dev/null +++ b/scripts/test-numeric-regressions.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +# shellcheck disable=SC2016 # backslash-$ in grep -E patterns is intentional here +# Regression for PR #76 rewrite corruption: four numeric comparisons in the +# tester installer/runtime were rewritten to the invalid token `-private-repository`, +# which is a bash syntax error (SC1073/SC1035/SC1072) and broke exact-head CI. +# Each site must use a real numeric comparison (`-lt`). This test reproduces all +# four (fails on corruption) and passes once `-lt` is restored. +set -Eeuo pipefail + +repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +installer=$repo_root/scripts/install-tester.sh +runtime=$repo_root/scripts/tester-runtime.sh +test_install=$repo_root/scripts/test-install-tester.sh + +fail() { printf 'FAIL %s\n' "$*" >&2; exit 1; } + +# Execution-level reproduction: a corrupted comparison makes the whole script +# unparseable, so `bash -n` must succeed on every affected file. +for f in "$installer" "$runtime" "$test_install"; do + bash -n "$f" || fail "syntax error in $f (corrupted numeric comparison?)" +done + +# Per-site assertions: the corrupted token must be gone, and the correct `-lt` +# numeric comparison must be present. +check_site() { + local file=$1 label=$2 pattern=$3 + if grep -Eq -- '-private-repository' "$file"; then + fail "$label: corrupted '-private-repository' token still present in $file" + fi + if ! grep -Eq "$pattern" "$file"; then + fail "$label: expected numeric '-lt' comparison not found in $file" + fi +} + +# 1. install-tester.sh host_preflight: disk usage below 80% warn gate. +check_site "$installer" "install-tester.sh:90 disk<80" 'used -lt 80' +# 2. tester-runtime.sh prepare_converge: tracked count below max_environments. +check_site "$runtime" "tester-runtime.sh:192 count/dev/null || die 'test-host DNS probe failed' curl --fail --silent --show-error --head --max-time 10 --output /dev/null "$probe_url" || die 'test-host HTTPS/proxy probe failed' used=$(df -P "$(root_path /var/lib/docker)" | awk 'NR==2{gsub(/%/,"",$5);print $5}') - [[ $used =~ ^[0-9]+$ && $used -private-repository $disk_warn ]] || die 'Docker storage exceeds configured warning threshold' + [[ $used =~ ^[0-9]+$ && $used -lt $disk_warn ]] || die 'Docker storage exceeds configured warning threshold' report "CHECK_OK max_environments=$max_environments disk_used_percent=$used" ;; --converge) converge ;; From c661e729ca5f61ef5a5dff5fcbb3a2fd3fc60d7c Mon Sep 17 00:00:00 2001 From: Nick's Hermes <1572453+Nickfost@users.noreply.github.com> Date: Sat, 29 Aug 2026 23:07:12 -0500 Subject: [PATCH 18/42] test: reproduce archive symlink dereference --- scripts/test-install-tester.sh | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/scripts/test-install-tester.sh b/scripts/test-install-tester.sh index b7a963e8..10ff76d9 100755 --- a/scripts/test-install-tester.sh +++ b/scripts/test-install-tester.sh @@ -144,20 +144,17 @@ ref=$(git -C "$repo_root" rev-parse HEAD) # A release archive containing a symlinked or non-regular member must be rejected # before chmod can dereference it and alter a host-side target's permissions. symlink_repo=$tmp/symlink-repo -git init --quiet "$symlink_repo" -git -C "$symlink_repo" config user.name Example -git -C "$symlink_repo" config user.email example@invalid.example -mkdir -p "$symlink_repo/scripts" "$symlink_repo/host/systemd" -cp "$repo_root/scripts/tester-runtime.sh" "$symlink_repo/scripts/tester-runtime.sh" -cp "$repo_root/scripts/tester-launcher.sh" "$symlink_repo/scripts/tester-launcher.sh" -cp "$repo_root"/host/systemd/ci-fleet-tester-*.service "$repo_root"/host/systemd/ci-fleet-tester-*.timer "$symlink_repo/host/systemd/" -printf '#!/usr/bin/env bash\nexit 0\n' >"$symlink_repo/scripts/host-evil"; chmod 0755 "$symlink_repo/scripts/host-evil" +git clone --quiet --shared "$repo_root" "$symlink_repo" +cp "$repo_root/scripts/install-tester.sh" "$symlink_repo/scripts/install-tester.sh" +host_target=$tmp/host-target +printf '#!/usr/bin/env bash\nexit 0\n' >"$host_target"; chmod 0640 "$host_target" rm -f "$symlink_repo/scripts/tester-runtime.sh" -ln -s host-evil "$symlink_repo/scripts/tester-runtime.sh" +ln -s "$host_target" "$symlink_repo/scripts/tester-runtime.sh" git -C "$symlink_repo" add -A -git -C "$symlink_repo" commit --quiet -m 'fixture: symlinked release member' +git -C "$symlink_repo" -c user.name=Example -c user.email=example@invalid.example commit --quiet -m 'fixture: symlinked release member' symlink_ref=$(git -C "$symlink_repo" rev-parse HEAD) -if "$installer" --install --config /etc/ci-fleet-tester/tester.env --ref "$symlink_ref" >/dev/null 2>&1; then fail 'install accepted a release archive with a symlinked member'; fi +if "$symlink_repo/scripts/install-tester.sh" --install --config /etc/ci-fleet-tester/tester.env --ref "$symlink_ref" >/dev/null 2>&1; then fail 'install accepted a release archive with a symlinked member'; fi +[[ $(stat -c %a "$host_target") == 640 ]] || fail 'release validation dereferenced an archive symlink' if DOCKER_HOST=tcp://example.invalid:2375 "$installer" --check --config /etc/ci-fleet-tester/tester.env >/dev/null 2>&1; then fail 'installer accepted a remote Docker selector'; fi "$installer" --install --config /etc/ci-fleet-tester/tester.env --ref "$ref" | grep -Fq INSTALL_OK || fail 'fresh install failed' rm -rf "$root/run/lock/ci-fleet-tester" From 3ef9fbf09cc3227ab66aa3be86dd0136bc75e64a Mon Sep 17 00:00:00 2001 From: Nickfost <1572453+Nickfost@users.noreply.github.com> Date: Sat, 29 Aug 2026 23:08:15 -0500 Subject: [PATCH 19/42] fix: reject symlinked tester release members --- scripts/install-tester.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/install-tester.sh b/scripts/install-tester.sh index fe3ccc55..cba6fa4a 100755 --- a/scripts/install-tester.sh +++ b/scripts/install-tester.sh @@ -121,7 +121,7 @@ stage_release() { # (chmod follows symlinks and would change the host target's permissions). while IFS= read -r -d '' entry; do if [[ -L $entry || ! -f $entry ]]; then remove_release_tree "$staging"; die "release archive contains a non-regular or symlinked member: $entry"; fi - done < <(find "$staging" -type f -print0) + done < <(find "$staging" ! -type d -print0) printf '%s\n' "$commit" >"$staging/.ci-fleet-source-revision"; chmod 0644 "$staging/.ci-fleet-source-revision" chmod 0755 "$staging/scripts/tester-runtime.sh" "$staging/scripts/tester-launcher.sh"; shellcheck "$staging/scripts/tester-runtime.sh" "$staging/scripts/tester-launcher.sh"; bash -n "$staging/scripts/tester-runtime.sh" "$staging/scripts/tester-launcher.sh" (cd "$staging" && sha256sum scripts/tester-runtime.sh scripts/tester-launcher.sh .ci-fleet-source-revision host/systemd/* >.ci-fleet-release.sha256) From 6fcdb6760c8b5d12c49bc529b978e16633da17f4 Mon Sep 17 00:00:00 2001 From: Nickfost <1572453+Nickfost@users.noreply.github.com> Date: Sat, 29 Aug 2026 23:09:49 -0500 Subject: [PATCH 20/42] test: reproduce escaped explicit include key --- scripts/test-install-tester.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/test-install-tester.sh b/scripts/test-install-tester.sh index 10ff76d9..62e333c1 100755 --- a/scripts/test-install-tester.sh +++ b/scripts/test-install-tester.sh @@ -85,6 +85,8 @@ printf '? "include":\n - path: /etc/passwd\nservices: {}\n' >"$root/etc/ci-flee if FAKE_TESTER_ROUTE_PORT=18091 "$runtime" --converge --environment bad-include >/dev/null 2>&1; then fail 'explicit-key Compose include was accepted'; fi printf "? 'include':\n - path: /etc/passwd\nservices: {}\n" >"$root/etc/ci-fleet-tester/definitions/bad-include.yaml" if FAKE_TESTER_ROUTE_PORT=18091 "$runtime" --converge --environment bad-include >/dev/null 2>&1; then fail 'single-quoted explicit-key Compose include was accepted'; fi +printf '%s\n' '? "incl\u0075de"' ': [{path: /etc/passwd, env_file: /etc/passwd}]' 'services: {}' >"$root/etc/ci-fleet-tester/definitions/bad-include.yaml" +if FAKE_TESTER_ROUTE_PORT=18091 "$runtime" --converge --environment bad-include >/dev/null 2>&1; then fail 'escaped explicit-key Compose include was accepted'; fi write_environment interpolation 18090 TOKEN=must-not-render FAKE_TESTER_ROUTE_PORT=18090 FAKE_TESTER_POLICY=interpolation "$runtime" --converge --environment interpolation >/dev/null if grep -Fq must-not-render "$root/var/lib/ci-fleet-tester/environments/interpolation.compose.json"; then fail 'caller environment was interpolated into the Compose model'; fi From c1650ac0ca77af1226f74941e0d73e8ff7d575ee Mon Sep 17 00:00:00 2001 From: Nickfost <1572453+Nickfost@users.noreply.github.com> Date: Sat, 29 Aug 2026 23:10:37 -0500 Subject: [PATCH 21/42] fix: reject explicit Compose mapping keys --- scripts/tester-runtime.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/tester-runtime.sh b/scripts/tester-runtime.sh index cf23d6de..200b9286 100755 --- a/scripts/tester-runtime.sh +++ b/scripts/tester-runtime.sh @@ -114,7 +114,7 @@ validate_compose() { import re,sys text=open(sys.argv[1],encoding='utf-8').read() key=r'(?:!!str[ \t]+)?(?:include|"include"|\x27include\x27)[ \t]*:' -explicit_key=r'(?m)^[ \t]*\?[ \t]*(?:include|"include"|\x27include\x27)[ \t]*:?$' +explicit_key=r'(?m)^[ \t]*\?' escaped_key=r'"[^"\n]*\\[^"\n]*"[ \t]*:' if re.search(r'(?m)^[ \t]*'+key,text) or re.search(r'[,{][ \t]*'+key,text) or re.search(escaped_key,text) or re.search(explicit_key,text): raise SystemExit('Compose include or escaped mapping key is forbidden') PY From 054d15935740733180ece6341e442ff02e082d1a Mon Sep 17 00:00:00 2001 From: Nickfost <1572453+Nickfost@users.noreply.github.com> Date: Sat, 29 Aug 2026 23:12:15 -0500 Subject: [PATCH 22/42] test: reproduce source label file loading --- scripts/test-install-tester.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/test-install-tester.sh b/scripts/test-install-tester.sh index 62e333c1..5002305b 100755 --- a/scripts/test-install-tester.sh +++ b/scripts/test-install-tester.sh @@ -87,6 +87,9 @@ printf "? 'include':\n - path: /etc/passwd\nservices: {}\n" >"$root/etc/ci-flee if FAKE_TESTER_ROUTE_PORT=18091 "$runtime" --converge --environment bad-include >/dev/null 2>&1; then fail 'single-quoted explicit-key Compose include was accepted'; fi printf '%s\n' '? "incl\u0075de"' ': [{path: /etc/passwd, env_file: /etc/passwd}]' 'services: {}' >"$root/etc/ci-fleet-tester/definitions/bad-include.yaml" if FAKE_TESTER_ROUTE_PORT=18091 "$runtime" --converge --environment bad-include >/dev/null 2>&1; then fail 'escaped explicit-key Compose include was accepted'; fi +write_environment bad-source-label 18092 +printf 'services:\n web:\n label_file: /root/credential.env\n' >"$root/etc/ci-fleet-tester/definitions/bad-source-label.yaml" +if FAKE_TESTER_ROUTE_PORT=18092 "$runtime" --converge --environment bad-source-label >/dev/null 2>&1; then fail 'source Compose label_file was accepted before rendering'; fi write_environment interpolation 18090 TOKEN=must-not-render FAKE_TESTER_ROUTE_PORT=18090 FAKE_TESTER_POLICY=interpolation "$runtime" --converge --environment interpolation >/dev/null if grep -Fq must-not-render "$root/var/lib/ci-fleet-tester/environments/interpolation.compose.json"; then fail 'caller environment was interpolated into the Compose model'; fi From 1269edd175ba019f809a281f013146218a6f0f20 Mon Sep 17 00:00:00 2001 From: Nickfost <1572453+Nickfost@users.noreply.github.com> Date: Sat, 29 Aug 2026 23:12:59 -0500 Subject: [PATCH 23/42] fix: reject Compose label files before rendering --- scripts/tester-runtime.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/tester-runtime.sh b/scripts/tester-runtime.sh index 200b9286..372e0add 100755 --- a/scripts/tester-runtime.sh +++ b/scripts/tester-runtime.sh @@ -113,10 +113,10 @@ validate_compose() { if ! python3 - "$compose_file" <<'PY' import re,sys text=open(sys.argv[1],encoding='utf-8').read() -key=r'(?:!!str[ \t]+)?(?:include|"include"|\x27include\x27)[ \t]*:' +key=r'(?:!!str[ \t]+)?(?:include|label_file|"(?:include|label_file)"|\x27(?:include|label_file)\x27)[ \t]*:' explicit_key=r'(?m)^[ \t]*\?' escaped_key=r'"[^"\n]*\\[^"\n]*"[ \t]*:' -if re.search(r'(?m)^[ \t]*'+key,text) or re.search(r'[,{][ \t]*'+key,text) or re.search(escaped_key,text) or re.search(explicit_key,text): raise SystemExit('Compose include or escaped mapping key is forbidden') +if re.search(r'(?m)^[ \t]*'+key,text) or re.search(r'[,{][ \t]*'+key,text) or re.search(escaped_key,text) or re.search(explicit_key,text): raise SystemExit('Compose include, label_file, or escaped mapping key is forbidden') PY then return 1; fi if [[ ${CI_FLEET_TESTING:-0} == 1 ]]; then for variable in ${!FAKE_@}; do clean_environment+=("$variable=${!variable}"); done; fi From 511398212fa2fd549ea693ab80dcebb46b38490b Mon Sep 17 00:00:00 2001 From: Nickfost <1572453+Nickfost@users.noreply.github.com> Date: Sat, 29 Aug 2026 23:14:55 -0500 Subject: [PATCH 24/42] test: reproduce broken deployed tester launcher --- scripts/test-install-tester.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/scripts/test-install-tester.sh b/scripts/test-install-tester.sh index 5002305b..af79b5ae 100755 --- a/scripts/test-install-tester.sh +++ b/scripts/test-install-tester.sh @@ -200,6 +200,16 @@ git -C "$upgrade_repo" -c user.name=Example -c user.email=example@invalid.exampl bad_ref=$(git -C "$upgrade_repo" rev-parse HEAD) if "$upgrade_repo/scripts/install-tester.sh" --upgrade --config /etc/ci-fleet-tester/tester.env --ref "$bad_ref" >/dev/null 2>&1; then fail 'failed candidate activation succeeded'; fi [[ $(readlink -f "$root/opt/ci-fleet-tester/current") == "$root/opt/ci-fleet-tester/releases/$ref" ]] || fail 'failed upgrade did not restore incumbent release' + +# Post-switch checks must use the installed launcher rather than bypassing it. +launcher_repo=$tmp/launcher-repo +git clone --quiet --shared "$repo_root" "$launcher_repo" +printf '#!/usr/bin/env bash\nexit 1\n' >"$launcher_repo/scripts/tester-launcher.sh"; chmod 0755 "$launcher_repo/scripts/tester-launcher.sh" +git -C "$launcher_repo" add scripts/tester-launcher.sh +git -C "$launcher_repo" -c user.name=Example -c user.email=example@invalid.example commit --quiet -m 'fixture: fail tester launcher' +launcher_fail_ref=$(git -C "$launcher_repo" rev-parse HEAD) +if "$launcher_repo/scripts/install-tester.sh" --upgrade --config /etc/ci-fleet-tester/tester.env --ref "$launcher_fail_ref" >/dev/null 2>&1; then fail 'candidate with a broken deployed launcher was accepted'; fi +[[ $(readlink -f "$root/opt/ci-fleet-tester/current") == "$root/opt/ci-fleet-tester/releases/$ref" ]] || fail 'launcher failure did not restore incumbent release' git -C "$upgrade_repo" checkout --quiet "$ref" git -C "$upgrade_repo" replace "$ref" "$bad_ref" if "$upgrade_repo/scripts/install-tester.sh" --install --config /etc/ci-fleet-tester/tester.env --ref "$ref" >/dev/null 2>&1; then fail 'Git replacement metadata was accepted'; fi From e3f425be87635a8bb70b11b07f74afb7faf828aa Mon Sep 17 00:00:00 2001 From: Nickfost <1572453+Nickfost@users.noreply.github.com> Date: Sat, 29 Aug 2026 23:16:05 -0500 Subject: [PATCH 25/42] fix: validate activation through deployed launcher --- scripts/install-tester.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/install-tester.sh b/scripts/install-tester.sh index cba6fa4a..06f17b93 100755 --- a/scripts/install-tester.sh +++ b/scripts/install-tester.sh @@ -183,7 +183,7 @@ activate_release() { die 'could not quiesce tester maintenance timers; incumbent timers restored' fi ln -sfn "$target" "$current_link.new"; mv -Tf "$current_link.new" "$current_link" - if ! install_launcher "$target" || ! install_units "$target" || ! create_tmpfiles || ! "$target/scripts/tester-runtime.sh" --check || ! "$target/scripts/tester-runtime.sh" --health || ! enable_timers; then + if ! install_launcher "$target" || ! install_units "$target" || ! create_tmpfiles || ! "$stable_launcher" --check || ! "$stable_launcher" --health || ! enable_timers; then if [[ $previous =~ ^[0-9a-f]{40}$ ]] && release_complete "$release_dir/$previous" "$previous"; then ln -sfn "$release_dir/$previous" "$current_link.new"; mv -Tf "$current_link.new" "$current_link" if ! install_launcher "$release_dir/$previous" || ! install_units "$release_dir/$previous" || ! create_tmpfiles || ! enable_timers; then die 'candidate activation failed and incumbent unit restore failed; launcher and incumbent link retained for recovery'; fi From bfa705b4dc7f850fcd633146760937e9e0b8e69a Mon Sep 17 00:00:00 2001 From: Nickfost <1572453+Nickfost@users.noreply.github.com> Date: Sat, 29 Aug 2026 23:20:31 -0500 Subject: [PATCH 26/42] fix: preserve inherited tester lifecycle lock --- scripts/tester-launcher.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/tester-launcher.sh b/scripts/tester-launcher.sh index e5c2987b..0063ebfe 100644 --- a/scripts/tester-launcher.sh +++ b/scripts/tester-launcher.sh @@ -13,8 +13,10 @@ expected_uid=0 [[ ${CI_FLEET_TESTING:-0} != 1 ]] || expected_uid=$(id -u) mkdir -m 0755 "$lock_dir" 2>/dev/null || true [[ -d $lock_dir && ! -L $lock_dir && $(stat -c %u "$lock_dir") == "$expected_uid" && $(stat -c %a "$lock_dir") == 755 ]] || { printf 'ERROR: tester lock directory is unsafe\n' >&2; exit 1; } -exec 8>"$lock_file" -flock -x 8 +if [[ ${CI_FLEET_TESTER_LOCK_FD:-} != 8 || ! -e /proc/$$/fd/8 || $(readlink -f /proc/$$/fd/8) != "$lock_file" ]]; then + exec 8>"$lock_file" + flock -x 8 +fi target=$(readlink -f "$current_link") revision=$(basename "$target") [[ $target == "$release_dir/$revision" && $revision =~ ^[0-9a-f]{40}$ && -x $target/scripts/tester-runtime.sh ]] || { printf 'ERROR: active tester release is invalid\n' >&2; exit 1; } From df14fa59fac842eb29c464234a2d27c8f5aabf77 Mon Sep 17 00:00:00 2001 From: Nickfost <1572453+Nickfost@users.noreply.github.com> Date: Sat, 29 Aug 2026 23:22:07 -0500 Subject: [PATCH 27/42] test: require activation service execution --- scripts/test-install-tester.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/test-install-tester.sh b/scripts/test-install-tester.sh index af79b5ae..09280b5c 100755 --- a/scripts/test-install-tester.sh +++ b/scripts/test-install-tester.sh @@ -225,6 +225,8 @@ unit_hash_before=$(sha256sum "$root/etc/systemd/system/ci-fleet-tester-health.se if FAKE_TESTER_SYSTEMCTL_FAIL=daemon-reload "$upgrade_repo/scripts/install-tester.sh" --upgrade --config /etc/ci-fleet-tester/tester.env --ref "$unit_fail_ref" >"$tmp/unit-restore.log" 2>&1; then fail 'unit activation failure succeeded'; fi grep -Fq 'incumbent unit restore failed' "$tmp/unit-restore.log" || fail 'secondary incumbent restore failure was not surfaced' [[ $(readlink -f "$root/opt/ci-fleet-tester/current") == "$root/opt/ci-fleet-tester/releases/$ref" && $(sha256sum "$root/etc/systemd/system/ci-fleet-tester-health.service") == "$unit_hash_before" ]] || fail 'unit activation failure did not restore incumbent release and units' +if FAKE_TESTER_SYSTEMCTL_FAIL='start ci-fleet-tester-health.service ci-fleet-tester-cleanup.service' "$upgrade_repo/scripts/install-tester.sh" --upgrade --config /etc/ci-fleet-tester/tester.env --ref "$unit_fail_ref" >/dev/null 2>&1; then fail 'candidate activation did not exercise maintenance services'; fi +[[ $(readlink -f "$root/opt/ci-fleet-tester/current") == "$root/opt/ci-fleet-tester/releases/$ref" ]] || fail 'maintenance service failure did not restore incumbent release' # A corrupt incumbent is never recorded as the rollback target. chmod u+w "$release/scripts/tester-runtime.sh"; printf '# corrupt incumbent\n' >>"$release/scripts/tester-runtime.sh"; chmod 0555 "$release/scripts/tester-runtime.sh" From d3cc48dc636244358cc2716c7d4aa809977cfe4d Mon Sep 17 00:00:00 2001 From: Nickfost <1572453+Nickfost@users.noreply.github.com> Date: Sat, 29 Aug 2026 23:23:24 -0500 Subject: [PATCH 28/42] fix: exercise tester maintenance services on activation --- scripts/install-tester.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/install-tester.sh b/scripts/install-tester.sh index 06f17b93..c26cb7e9 100755 --- a/scripts/install-tester.sh +++ b/scripts/install-tester.sh @@ -151,6 +151,7 @@ create_tmpfiles() { } enable_timers() { systemctl enable --now "${timers[@]}" >/dev/null; } +start_maintenance() { systemctl start ci-fleet-tester-health.service ci-fleet-tester-cleanup.service >/dev/null; } install_launcher() { install -m 0555 "$1/scripts/tester-launcher.sh" "$stable_launcher.new" && mv -fT "$stable_launcher.new" "$stable_launcher"; } remove_units() { @@ -183,7 +184,7 @@ activate_release() { die 'could not quiesce tester maintenance timers; incumbent timers restored' fi ln -sfn "$target" "$current_link.new"; mv -Tf "$current_link.new" "$current_link" - if ! install_launcher "$target" || ! install_units "$target" || ! create_tmpfiles || ! "$stable_launcher" --check || ! "$stable_launcher" --health || ! enable_timers; then + if ! install_launcher "$target" || ! install_units "$target" || ! create_tmpfiles || ! "$stable_launcher" --check || ! "$stable_launcher" --health || ! start_maintenance || ! enable_timers; then if [[ $previous =~ ^[0-9a-f]{40}$ ]] && release_complete "$release_dir/$previous" "$previous"; then ln -sfn "$release_dir/$previous" "$current_link.new"; mv -Tf "$current_link.new" "$current_link" if ! install_launcher "$release_dir/$previous" || ! install_units "$release_dir/$previous" || ! create_tmpfiles || ! enable_timers; then die 'candidate activation failed and incumbent unit restore failed; launcher and incumbent link retained for recovery'; fi From 7516fa50d6cabb970f8ad3f281a229c2d0764088 Mon Sep 17 00:00:00 2001 From: Nickfost <1572453+Nickfost@users.noreply.github.com> Date: Sat, 29 Aug 2026 23:44:20 -0500 Subject: [PATCH 29/42] fix: close tester maintenance review findings --- scripts/install-tester.sh | 2 ++ scripts/test-install-tester.sh | 7 +++++++ scripts/test-numeric-regressions.sh | 4 ++++ scripts/tester-runtime.sh | 12 ++++++++++-- scripts/validate.sh | 1 + 5 files changed, 24 insertions(+), 2 deletions(-) mode change 100644 => 100755 scripts/test-numeric-regressions.sh diff --git a/scripts/install-tester.sh b/scripts/install-tester.sh index c26cb7e9..58fa4bd9 100755 --- a/scripts/install-tester.sh +++ b/scripts/install-tester.sh @@ -166,6 +166,7 @@ remove_units() { systemctl disable --now "${present_timers[@]}" >/dev/null 2>&1 || return 1 fi for unit in "${present_units[@]}"; do rm -f -- "$systemd_dir/$unit" || return 1; done + rm -f -- "$tmpfiles_dir/$tmpfiles_conf" || return 1 systemctl daemon-reload } @@ -230,6 +231,7 @@ case $action in release_complete "$release_dir/$current" "$current" || die 'installed release is incomplete' cmp -s "$release_dir/$current/scripts/tester-launcher.sh" "$stable_launcher" || die 'installed launcher differs from active release' for unit in "${units[@]}"; do cmp -s "$release_dir/$current/host/systemd/$unit" "$systemd_dir/$unit" || die "installed unit differs from active release: $unit"; done + cmp -s "$release_dir/$current/host/systemd/$tmpfiles_conf" "$tmpfiles_dir/$tmpfiles_conf" || die 'installed tmpfiles rule differs from active release' for timer in "${timers[@]}"; do if ! systemctl is-enabled --quiet "$timer" || ! systemctl is-active --quiet "$timer"; then die "timer is inactive: $timer"; fi done diff --git a/scripts/test-install-tester.sh b/scripts/test-install-tester.sh index 09280b5c..f60683a0 100755 --- a/scripts/test-install-tester.sh +++ b/scripts/test-install-tester.sh @@ -100,6 +100,8 @@ if FAKE_TESTER_ROUTE_PORT=18080 FAKE_TESTER_POLICY=changed-model "$runtime" --co if FAKE_TESTER_ROUTE_PORT=18080 FAKE_TESTER_POLICY=mutable "$runtime" --reset --environment preview-a >/dev/null 2>&1; then fail 'reset accepted invalid replacement'; fi [[ -f $state ]] || fail 'reset deleted the incumbent before validation' if FAKE_TESTER_ROUTE_PORT=18080 FAKE_TESTER_CONTAINER_STATE='exited unhealthy' "$runtime" --health >/dev/null 2>&1; then fail 'health accepted a stopped managed service'; fi +if FAKE_TESTER_ROUTE_PORT=18080 FAKE_TESTER_CONTAINER_STATE='running none' "$runtime" --converge --environment preview-a >/dev/null 2>&1; then fail 'converge accepted a route without health evidence'; fi +if FAKE_TESTER_ROUTE_PORT=18080 FAKE_TESTER_CONTAINER_STATE='running none' "$runtime" --health >/dev/null 2>&1; then fail 'health accepted a route without health evidence'; fi write_environment secret-preview 18082 secret_file=$root/etc/ci-fleet-tester/secrets/secret-preview/credential printf 'example-test-scope-value\n' >"$secret_file"; chmod 600 "$secret_file" @@ -190,6 +192,10 @@ installed_unit=$root/etc/systemd/system/ci-fleet-tester-health.service printf '# drift\n' >>"$installed_unit" if "$installer" --check --config /etc/ci-fleet-tester/tester.env >/dev/null 2>&1; then fail 'installed unit drift passed check'; fi cp "$release/host/systemd/ci-fleet-tester-health.service" "$installed_unit" +installed_tmpfiles=$root/usr/lib/tmpfiles.d/ci-fleet-tester-lock.conf +rm "$installed_tmpfiles" +if "$installer" --check --config /etc/ci-fleet-tester/tester.env >/dev/null 2>&1; then fail 'missing installed tmpfiles rule passed check'; fi +cp "$release/host/systemd/ci-fleet-tester-lock.conf" "$installed_tmpfiles" # A syntactically valid candidate that fails its post-switch check restores the incumbent. upgrade_repo=$tmp/upgrade-repo @@ -287,6 +293,7 @@ if FAKE_TESTER_SYSTEMCTL_FAIL='disable --now' "$installer" --uninstall --config [[ -L $root/opt/ci-fleet-tester/current ]] || fail 'failed uninstall removed the active release' "$installer" --uninstall --config /etc/ci-fleet-tester/tester.env | grep -Fq UNINSTALL_OK || fail 'uninstall failed' [[ -f $root/etc/ci-fleet-tester/tester.env && ! -L $root/opt/ci-fleet-tester/current ]] || fail 'uninstall did not preserve config/remove runtime' +[[ ! -e $root/usr/lib/tmpfiles.d/ci-fleet-tester-lock.conf ]] || fail 'uninstall left the tester tmpfiles rule installed' FAKE_TESTER_SYSTEMCTL_FAIL_IF_UNITS_MISSING=1 "$installer" --uninstall --config /etc/ci-fleet-tester/tester.env | grep -Fq UNINSTALL_OK || fail 'repeated uninstall failed on absent units' git -C "$upgrade_repo" checkout --quiet "$bad_ref" if FAKE_TESTER_SYSTEMCTL_FAIL='disable --now' "$upgrade_repo/scripts/install-tester.sh" --install --config /etc/ci-fleet-tester/tester.env --ref "$bad_ref" >/dev/null 2>&1; then fail 'invalid fresh install succeeded'; fi diff --git a/scripts/test-numeric-regressions.sh b/scripts/test-numeric-regressions.sh old mode 100644 new mode 100755 index 2d0e8ce5..9690fc21 --- a/scripts/test-numeric-regressions.sh +++ b/scripts/test-numeric-regressions.sh @@ -11,9 +11,13 @@ repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) installer=$repo_root/scripts/install-tester.sh runtime=$repo_root/scripts/tester-runtime.sh test_install=$repo_root/scripts/test-install-tester.sh +validate=$repo_root/scripts/validate.sh fail() { printf 'FAIL %s\n' "$*" >&2; exit 1; } +[[ -x $0 ]] || fail 'numeric regression test is not executable' +grep -Fq 'scripts/test-numeric-regressions.sh' "$validate" || fail 'validation does not run numeric regressions' + # Execution-level reproduction: a corrupted comparison makes the whole script # unparseable, so `bash -n` must succeed on every affected file. for f in "$installer" "$runtime" "$test_install"; do diff --git a/scripts/tester-runtime.sh b/scripts/tester-runtime.sh index 372e0add..84a2bc83 100755 --- a/scripts/tester-runtime.sh +++ b/scripts/tester-runtime.sh @@ -210,6 +210,7 @@ apply_converge() { die 'environment activation failed; tracked state retained for cleanup' fi rm -f "$prepared_rendered" + inspect_environment "$(state_path "$environment")" | grep -q 'STATUS=running' || die 'environment route is unhealthy; tracked state retained for cleanup' report "CONVERGED environment=$environment project=$project owner=$owner route=loopback:$route_port expires_at=$expires_at" } @@ -230,19 +231,26 @@ remove_environment() { } inspect_environment() { - local target=$1 id compose status=running resource value bytes=0 mount expected running_state + local target=$1 id compose route_service route_container status=running resource value bytes=0 mount expected running_state local -a containers=() id=$(basename "$target" .state); secure_file "$target" 600 while IFS='=' read -r key value; do case $key in ENVIRONMENT|PROJECT|OWNER|ROUTE_PORT|EXPIRES_AT|SOURCE_REVISION|IMAGE_DIGESTS|UPDATED_AT) printf '%s=%s ' "$key" "$value" ;; esac done <"$target" compose=$(awk -F= '$1=="COMPOSE_FILE"{print substr($0,index($0,"=")+1)}' "$target") + route_service=$(awk -F= '$1=="ROUTE_SERVICE"{print $2}' "$target") expected=$(python3 -c 'import json,sys; print(len(json.load(open(sys.argv[1]))["services"]))' "$compose") mapfile -t containers < <(docker compose -p "$(project_name "$id")" -f "$compose" ps -q) + route_container=$(docker compose -p "$(project_name "$id")" -f "$compose" ps -q "$route_service") || die 'route container inventory failed' [[ ${#containers[@]} == "$expected" ]] || status=unhealthy + [[ -n $route_container ]] || status=unhealthy for resource in "${containers[@]}"; do running_state=$(docker inspect --format '{{.State.Status}} {{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$resource") - [[ $running_state == 'running healthy' || $running_state == 'running none' ]] || status=unhealthy + if [[ $resource == "$route_container" ]]; then + [[ $running_state == 'running healthy' ]] || status=unhealthy + else + [[ $running_state == 'running healthy' || $running_state == 'running none' ]] || status=unhealthy + fi done inventory=$(docker ps -aq --filter "label=com.docker.compose.project=$(project_name "$id")") || die 'container inventory failed' while IFS= read -r resource; do diff --git a/scripts/validate.sh b/scripts/validate.sh index d45bc9da..37a2f646 100755 --- a/scripts/validate.sh +++ b/scripts/validate.sh @@ -27,6 +27,7 @@ python3 scripts/desired_state.py validate-engine-capabilities --manifest engine- python3 .github/actions/plan/plan.py --plan examples/project/scripts/ci/plan.json --group fast >/dev/null python3 .github/actions/plan/plan.py --plan examples/project/scripts/ci/plan.json --group full >/dev/null scripts/test-capacity-preflight.sh +scripts/test-numeric-regressions.sh scripts/test-install-tester.sh scripts/test-install-worker-controller.sh scripts/test-install-status-receiver.sh From 08be87de5034a953093d9c70a72e28c96c71b637 Mon Sep 17 00:00:00 2001 From: Nickfost <1572453+Nickfost@users.noreply.github.com> Date: Sat, 29 Aug 2026 23:51:24 -0500 Subject: [PATCH 30/42] fix: avoid tester maintenance lock deadlock --- scripts/install-tester.sh | 2 +- scripts/test-install-tester.sh | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/scripts/install-tester.sh b/scripts/install-tester.sh index 58fa4bd9..92bc04b3 100755 --- a/scripts/install-tester.sh +++ b/scripts/install-tester.sh @@ -151,7 +151,7 @@ create_tmpfiles() { } enable_timers() { systemctl enable --now "${timers[@]}" >/dev/null; } -start_maintenance() { systemctl start ci-fleet-tester-health.service ci-fleet-tester-cleanup.service >/dev/null; } +start_maintenance() { systemctl start --no-block ci-fleet-tester-health.service ci-fleet-tester-cleanup.service >/dev/null; } install_launcher() { install -m 0555 "$1/scripts/tester-launcher.sh" "$stable_launcher.new" && mv -fT "$stable_launcher.new" "$stable_launcher"; } remove_units() { diff --git a/scripts/test-install-tester.sh b/scripts/test-install-tester.sh index f60683a0..a9071ce8 100755 --- a/scripts/test-install-tester.sh +++ b/scripts/test-install-tester.sh @@ -27,6 +27,7 @@ cat >"$fake_bin/systemctl" <<'EOF' printf '%s\n' "$*" >>"${FAKE_TESTER_SYSTEMCTL_LOG:?}" [[ -z ${FAKE_TESTER_EVENT_LOG:-} ]] || printf 'systemctl %s\n' "$*" >>"$FAKE_TESTER_EVENT_LOG" if [[ ${FAKE_TESTER_SYSTEMCTL_FAIL_IF_UNITS_MISSING:-0} == 1 && $1 == disable && ! -e ${CI_FLEET_ROOT_PREFIX:?}/etc/systemd/system/ci-fleet-tester-health.timer ]]; then exit 5; fi +if [[ ${FAKE_TESTER_SYSTEMCTL_REQUIRE_NO_BLOCK:-0} == 1 && $1 == start && " $* " == *' ci-fleet-tester-health.service '* && " $* " != *' --no-block '* ]]; then exit 6; fi [[ -z ${FAKE_TESTER_SYSTEMCTL_FAIL:-} || " $* " != *" $FAKE_TESTER_SYSTEMCTL_FAIL "* ]] EOF printf '#!/usr/bin/env bash\nexit 0\n' >"$fake_bin/curl" @@ -163,7 +164,7 @@ symlink_ref=$(git -C "$symlink_repo" rev-parse HEAD) if "$symlink_repo/scripts/install-tester.sh" --install --config /etc/ci-fleet-tester/tester.env --ref "$symlink_ref" >/dev/null 2>&1; then fail 'install accepted a release archive with a symlinked member'; fi [[ $(stat -c %a "$host_target") == 640 ]] || fail 'release validation dereferenced an archive symlink' if DOCKER_HOST=tcp://example.invalid:2375 "$installer" --check --config /etc/ci-fleet-tester/tester.env >/dev/null 2>&1; then fail 'installer accepted a remote Docker selector'; fi -"$installer" --install --config /etc/ci-fleet-tester/tester.env --ref "$ref" | grep -Fq INSTALL_OK || fail 'fresh install failed' +FAKE_TESTER_SYSTEMCTL_REQUIRE_NO_BLOCK=1 "$installer" --install --config /etc/ci-fleet-tester/tester.env --ref "$ref" | grep -Fq INSTALL_OK || fail 'activation waited for lock-blocked maintenance services' rm -rf "$root/run/lock/ci-fleet-tester" "$root/opt/ci-fleet-tester/tester-runtime" --health >/dev/null || fail 'stable launcher did not recreate the volatile lock directory' [[ $(readlink -f "$root/opt/ci-fleet-tester/current") == "$root/opt/ci-fleet-tester/releases/$ref" ]] || fail 'current release link is wrong' @@ -231,7 +232,7 @@ unit_hash_before=$(sha256sum "$root/etc/systemd/system/ci-fleet-tester-health.se if FAKE_TESTER_SYSTEMCTL_FAIL=daemon-reload "$upgrade_repo/scripts/install-tester.sh" --upgrade --config /etc/ci-fleet-tester/tester.env --ref "$unit_fail_ref" >"$tmp/unit-restore.log" 2>&1; then fail 'unit activation failure succeeded'; fi grep -Fq 'incumbent unit restore failed' "$tmp/unit-restore.log" || fail 'secondary incumbent restore failure was not surfaced' [[ $(readlink -f "$root/opt/ci-fleet-tester/current") == "$root/opt/ci-fleet-tester/releases/$ref" && $(sha256sum "$root/etc/systemd/system/ci-fleet-tester-health.service") == "$unit_hash_before" ]] || fail 'unit activation failure did not restore incumbent release and units' -if FAKE_TESTER_SYSTEMCTL_FAIL='start ci-fleet-tester-health.service ci-fleet-tester-cleanup.service' "$upgrade_repo/scripts/install-tester.sh" --upgrade --config /etc/ci-fleet-tester/tester.env --ref "$unit_fail_ref" >/dev/null 2>&1; then fail 'candidate activation did not exercise maintenance services'; fi +if FAKE_TESTER_SYSTEMCTL_FAIL='start --no-block ci-fleet-tester-health.service ci-fleet-tester-cleanup.service' "$upgrade_repo/scripts/install-tester.sh" --upgrade --config /etc/ci-fleet-tester/tester.env --ref "$unit_fail_ref" >/dev/null 2>&1; then fail 'candidate activation did not exercise maintenance services'; fi [[ $(readlink -f "$root/opt/ci-fleet-tester/current") == "$root/opt/ci-fleet-tester/releases/$ref" ]] || fail 'maintenance service failure did not restore incumbent release' # A corrupt incumbent is never recorded as the rollback target. From 9654ace274752027a1c9307d32eb735149287066 Mon Sep 17 00:00:00 2001 From: Nickfost <1572453+Nickfost@users.noreply.github.com> Date: Sun, 30 Aug 2026 00:11:05 -0500 Subject: [PATCH 31/42] fix: harden tester Compose policy validation --- scripts/fixtures/fake-tester-docker.sh | 1 + scripts/test-install-tester.sh | 5 ++++- scripts/tester-runtime.sh | 4 ++-- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/scripts/fixtures/fake-tester-docker.sh b/scripts/fixtures/fake-tester-docker.sh index 4ed79958..10f4f5a4 100755 --- a/scripts/fixtures/fake-tester-docker.sh +++ b/scripts/fixtures/fake-tester-docker.sh @@ -58,6 +58,7 @@ case ${operation:-} in interpolation) [[ -z ${TOKEN:-} ]] || service_extra=$(printf ',"command":["app","--token=%s"]' "$TOKEN") ;; profiles) service_extra=',"profiles":["extra"]' ;; label-file) service_extra=',"label_file":"/root/credential.env"' ;; + runtime) service_extra=',"runtime":"alternative"' ;; valid-secret|outside-secret) secrets=$(printf '{"credential":{"file":"%s"}}' "${FAKE_TESTER_SECRET_FILE:?}") ;; esac volume=${volume:-'{"type":"volume","source":"data","target":"/data"}'} diff --git a/scripts/test-install-tester.sh b/scripts/test-install-tester.sh index a9071ce8..3d4c0cf6 100755 --- a/scripts/test-install-tester.sh +++ b/scripts/test-install-tester.sh @@ -73,7 +73,7 @@ FAKE_TESTER_ROUTE_PORT=18080 "$runtime" --converge --environment preview-a >/dev [[ $(awk -F= '$1=="EXPIRES_AT"{print $2}' "$state") == "$original_expiry" ]] || fail 'idempotent converge extended expiration' write_environment preview-b 18080 if FAKE_TESTER_ROUTE_PORT=18080 "$runtime" --converge --environment preview-b >/dev/null 2>&1; then fail 'duplicate route port was accepted'; fi -for policy in mutable privileged bind broad-port external-network environment configs use-api-socket namespace-share false-nnp unconfined custom-volume volumes-from external-links userns-host cgroup-host uts-host remote-logging custom-network ipam replicas lifecycle-hook gpu deploy-device build profiles label-file; do +for policy in mutable privileged bind broad-port external-network environment configs use-api-socket namespace-share false-nnp unconfined custom-volume volumes-from external-links userns-host cgroup-host uts-host remote-logging custom-network ipam replicas lifecycle-hook gpu deploy-device build profiles label-file runtime; do write_environment "bad-$policy" 18081 if FAKE_TESTER_ROUTE_PORT=18081 FAKE_TESTER_POLICY=$policy "$runtime" --converge --environment "bad-$policy" >/dev/null 2>&1; then fail "unsafe compose policy was accepted: $policy"; fi done @@ -91,6 +91,9 @@ if FAKE_TESTER_ROUTE_PORT=18091 "$runtime" --converge --environment bad-include write_environment bad-source-label 18092 printf 'services:\n web:\n label_file: /root/credential.env\n' >"$root/etc/ci-fleet-tester/definitions/bad-source-label.yaml" if FAKE_TESTER_ROUTE_PORT=18092 "$runtime" --converge --environment bad-source-label >/dev/null 2>&1; then fail 'source Compose label_file was accepted before rendering'; fi +write_environment bad-tagged-label 18093 +printf 'services:\n web:\n ! label_file: /root/credential.env\n' >"$root/etc/ci-fleet-tester/definitions/bad-tagged-label.yaml" +if FAKE_TESTER_ROUTE_PORT=18093 "$runtime" --converge --environment bad-tagged-label >/dev/null 2>&1; then fail 'verbatim-tagged Compose label_file was accepted before rendering'; fi write_environment interpolation 18090 TOKEN=must-not-render FAKE_TESTER_ROUTE_PORT=18090 FAKE_TESTER_POLICY=interpolation "$runtime" --converge --environment interpolation >/dev/null if grep -Fq must-not-render "$root/var/lib/ci-fleet-tester/environments/interpolation.compose.json"; then fail 'caller environment was interpolated into the Compose model'; fi diff --git a/scripts/tester-runtime.sh b/scripts/tester-runtime.sh index 84a2bc83..878441e2 100755 --- a/scripts/tester-runtime.sh +++ b/scripts/tester-runtime.sh @@ -113,7 +113,7 @@ validate_compose() { if ! python3 - "$compose_file" <<'PY' import re,sys text=open(sys.argv[1],encoding='utf-8').read() -key=r'(?:!!str[ \t]+)?(?:include|label_file|"(?:include|label_file)"|\x27(?:include|label_file)\x27)[ \t]*:' +key=r'(?:(?:!!str|!<[^>\n]+>)[ \t]+)?(?:include|label_file|"(?:include|label_file)"|\x27(?:include|label_file)\x27)[ \t]*:' explicit_key=r'(?m)^[ \t]*\?' escaped_key=r'"[^"\n]*\\[^"\n]*"[ \t]*:' if re.search(r'(?m)^[ \t]*'+key,text) or re.search(r'[,{][ \t]*'+key,text) or re.search(escaped_key,text) or re.search(explicit_key,text): raise SystemExit('Compose include, label_file, or escaped mapping key is forbidden') @@ -133,7 +133,7 @@ image=re.compile(r'^[a-z0-9.-]+(?::[0-9]+)?/[A-Za-z0-9_./-]+@sha256:[0-9a-f]{64} ports=[] for name,service in services.items(): if not image.fullmatch(str(service.get('image',''))): raise SystemExit(f'{name}: image must use an immutable sha256 digest') - if service.get('privileged') or service.get('network_mode') or service.get('pid') or service.get('ipc') or service.get('uts') or service.get('userns_mode') or service.get('cgroup') or service.get('external_links') or service.get('logging') or service.get('post_start') or service.get('pre_stop'): raise SystemExit(f'{name}: external namespace/link/logging/privileged lifecycle access is forbidden') + if service.get('privileged') or service.get('runtime') or service.get('network_mode') or service.get('pid') or service.get('ipc') or service.get('uts') or service.get('userns_mode') or service.get('cgroup') or service.get('external_links') or service.get('logging') or service.get('post_start') or service.get('pre_stop'): raise SystemExit(f'{name}: external runtime/namespace/link/logging/privileged lifecycle access is forbidden') deploy=service.get('deploy') or {}; reservations=(deploy.get('resources') or {}).get('reservations') or {} if service.get('build') or service.get('devices') or service.get('gpus') or reservations.get('devices') or service.get('cap_add') or service.get('container_name') or service.get('hostname') or service.get('use_api_socket') or service.get('volumes_from'): raise SystemExit(f'{name}: build/device/capability/external mount/global identity is forbidden') if deploy.get('replicas',1) != 1: raise SystemExit(f'{name}: exactly one replica is required') From 55db13afac1a73a89bb9d79fbb91c045aeee9096 Mon Sep 17 00:00:00 2001 From: Nickfost <1572453+Nickfost@users.noreply.github.com> Date: Sun, 30 Aug 2026 00:40:27 -0500 Subject: [PATCH 32/42] fix: close tester activation policy gaps --- scripts/fixtures/fake-tester-docker.sh | 1 + scripts/install-tester.sh | 13 +++++++++++-- scripts/test-install-tester.sh | 10 +++++++--- scripts/tester-runtime.sh | 2 +- 4 files changed, 20 insertions(+), 6 deletions(-) diff --git a/scripts/fixtures/fake-tester-docker.sh b/scripts/fixtures/fake-tester-docker.sh index 10f4f5a4..5383dc8a 100755 --- a/scripts/fixtures/fake-tester-docker.sh +++ b/scripts/fixtures/fake-tester-docker.sh @@ -44,6 +44,7 @@ case ${operation:-} in custom-network) network_extra=',"driver":"macvlan","driver_opts":{"parent":"eth0"}' ;; ipam) network_extra=',"ipam":{"config":[{"subnet":"172.30.0.0/24"}]}' ;; replicas) service_extra=',"deploy":{"replicas":2}' ;; + scale) service_extra=',"scale":2' ;; lifecycle-hook) service_extra=',"post_start":[{"command":"true","privileged":true}]' ;; gpu) service_extra=',"gpus":"all"' ;; deploy-device) service_extra=',"deploy":{"resources":{"reservations":{"devices":[{"capabilities":["gpu"]}]}}}' ;; diff --git a/scripts/install-tester.sh b/scripts/install-tester.sh index 92bc04b3..1aaf735d 100755 --- a/scripts/install-tester.sh +++ b/scripts/install-tester.sh @@ -49,8 +49,9 @@ runtime_state=$state_root/environments docker_root=$(root_path /var/lib/docker) docker_socket=$(root_path /var/run/docker.sock) runtime_lock=$(root_path /run/lock/ci-fleet-tester/runtime.lock) -units=(ci-fleet-tester-health.service ci-fleet-tester-health.timer ci-fleet-tester-cleanup.service ci-fleet-tester-cleanup.timer) +services=(ci-fleet-tester-health.service ci-fleet-tester-cleanup.service) timers=(ci-fleet-tester-health.timer ci-fleet-tester-cleanup.timer) +units=("${services[@]}" "${timers[@]}") tmpfiles_conf=ci-fleet-tester-lock.conf tmpfiles_dir=$(root_path /usr/lib/tmpfiles.d) @@ -151,7 +152,15 @@ create_tmpfiles() { } enable_timers() { systemctl enable --now "${timers[@]}" >/dev/null; } -start_maintenance() { systemctl start --no-block ci-fleet-tester-health.service ci-fleet-tester-cleanup.service >/dev/null; } +start_maintenance() { + local status=0 + systemctl start --no-block "${services[@]}" >/dev/null || return 1 + flock -u 8; unset CI_FLEET_TESTER_LOCK_FD + systemctl start "${services[@]}" >/dev/null || status=$? + flock -x 8 || return 1 + export CI_FLEET_TESTER_LOCK_FD=8 + return "$status" +} install_launcher() { install -m 0555 "$1/scripts/tester-launcher.sh" "$stable_launcher.new" && mv -fT "$stable_launcher.new" "$stable_launcher"; } remove_units() { diff --git a/scripts/test-install-tester.sh b/scripts/test-install-tester.sh index 3d4c0cf6..5c531c03 100755 --- a/scripts/test-install-tester.sh +++ b/scripts/test-install-tester.sh @@ -27,7 +27,7 @@ cat >"$fake_bin/systemctl" <<'EOF' printf '%s\n' "$*" >>"${FAKE_TESTER_SYSTEMCTL_LOG:?}" [[ -z ${FAKE_TESTER_EVENT_LOG:-} ]] || printf 'systemctl %s\n' "$*" >>"$FAKE_TESTER_EVENT_LOG" if [[ ${FAKE_TESTER_SYSTEMCTL_FAIL_IF_UNITS_MISSING:-0} == 1 && $1 == disable && ! -e ${CI_FLEET_ROOT_PREFIX:?}/etc/systemd/system/ci-fleet-tester-health.timer ]]; then exit 5; fi -if [[ ${FAKE_TESTER_SYSTEMCTL_REQUIRE_NO_BLOCK:-0} == 1 && $1 == start && " $* " == *' ci-fleet-tester-health.service '* && " $* " != *' --no-block '* ]]; then exit 6; fi +if [[ ${FAKE_TESTER_SYSTEMCTL_FAIL_WAIT:-0} == 1 && $1 == start && " $* " == *' ci-fleet-tester-health.service '* && " $* " != *' --no-block '* ]]; then exit 7; fi [[ -z ${FAKE_TESTER_SYSTEMCTL_FAIL:-} || " $* " != *" $FAKE_TESTER_SYSTEMCTL_FAIL "* ]] EOF printf '#!/usr/bin/env bash\nexit 0\n' >"$fake_bin/curl" @@ -73,7 +73,7 @@ FAKE_TESTER_ROUTE_PORT=18080 "$runtime" --converge --environment preview-a >/dev [[ $(awk -F= '$1=="EXPIRES_AT"{print $2}' "$state") == "$original_expiry" ]] || fail 'idempotent converge extended expiration' write_environment preview-b 18080 if FAKE_TESTER_ROUTE_PORT=18080 "$runtime" --converge --environment preview-b >/dev/null 2>&1; then fail 'duplicate route port was accepted'; fi -for policy in mutable privileged bind broad-port external-network environment configs use-api-socket namespace-share false-nnp unconfined custom-volume volumes-from external-links userns-host cgroup-host uts-host remote-logging custom-network ipam replicas lifecycle-hook gpu deploy-device build profiles label-file runtime; do +for policy in mutable privileged bind broad-port external-network environment configs use-api-socket namespace-share false-nnp unconfined custom-volume volumes-from external-links userns-host cgroup-host uts-host remote-logging custom-network ipam replicas scale lifecycle-hook gpu deploy-device build profiles label-file runtime; do write_environment "bad-$policy" 18081 if FAKE_TESTER_ROUTE_PORT=18081 FAKE_TESTER_POLICY=$policy "$runtime" --converge --environment "bad-$policy" >/dev/null 2>&1; then fail "unsafe compose policy was accepted: $policy"; fi done @@ -167,7 +167,8 @@ symlink_ref=$(git -C "$symlink_repo" rev-parse HEAD) if "$symlink_repo/scripts/install-tester.sh" --install --config /etc/ci-fleet-tester/tester.env --ref "$symlink_ref" >/dev/null 2>&1; then fail 'install accepted a release archive with a symlinked member'; fi [[ $(stat -c %a "$host_target") == 640 ]] || fail 'release validation dereferenced an archive symlink' if DOCKER_HOST=tcp://example.invalid:2375 "$installer" --check --config /etc/ci-fleet-tester/tester.env >/dev/null 2>&1; then fail 'installer accepted a remote Docker selector'; fi -FAKE_TESTER_SYSTEMCTL_REQUIRE_NO_BLOCK=1 "$installer" --install --config /etc/ci-fleet-tester/tester.env --ref "$ref" | grep -Fq INSTALL_OK || fail 'activation waited for lock-blocked maintenance services' +"$installer" --install --config /etc/ci-fleet-tester/tester.env --ref "$ref" | grep -Fq INSTALL_OK || fail 'activation failed' +grep -Fq 'start --no-block ci-fleet-tester-health.service ci-fleet-tester-cleanup.service' "$tmp/systemctl.log" || fail 'activation did not queue lock-blocked maintenance services' rm -rf "$root/run/lock/ci-fleet-tester" "$root/opt/ci-fleet-tester/tester-runtime" --health >/dev/null || fail 'stable launcher did not recreate the volatile lock directory' [[ $(readlink -f "$root/opt/ci-fleet-tester/current") == "$root/opt/ci-fleet-tester/releases/$ref" ]] || fail 'current release link is wrong' @@ -204,6 +205,7 @@ cp "$release/host/systemd/ci-fleet-tester-lock.conf" "$installed_tmpfiles" # A syntactically valid candidate that fails its post-switch check restores the incumbent. upgrade_repo=$tmp/upgrade-repo git clone --quiet --shared "$repo_root" "$upgrade_repo" +cp "$repo_root/scripts/install-tester.sh" "$upgrade_repo/scripts/install-tester.sh" printf '#!/usr/bin/env bash\nexit 1\n' >"$upgrade_repo/scripts/tester-runtime.sh"; chmod 0755 "$upgrade_repo/scripts/tester-runtime.sh" git -C "$upgrade_repo" add scripts/tester-runtime.sh git -C "$upgrade_repo" -c user.name=Example -c user.email=example@invalid.example commit --quiet -m 'fixture: fail tester activation' @@ -237,6 +239,8 @@ grep -Fq 'incumbent unit restore failed' "$tmp/unit-restore.log" || fail 'second [[ $(readlink -f "$root/opt/ci-fleet-tester/current") == "$root/opt/ci-fleet-tester/releases/$ref" && $(sha256sum "$root/etc/systemd/system/ci-fleet-tester-health.service") == "$unit_hash_before" ]] || fail 'unit activation failure did not restore incumbent release and units' if FAKE_TESTER_SYSTEMCTL_FAIL='start --no-block ci-fleet-tester-health.service ci-fleet-tester-cleanup.service' "$upgrade_repo/scripts/install-tester.sh" --upgrade --config /etc/ci-fleet-tester/tester.env --ref "$unit_fail_ref" >/dev/null 2>&1; then fail 'candidate activation did not exercise maintenance services'; fi [[ $(readlink -f "$root/opt/ci-fleet-tester/current") == "$root/opt/ci-fleet-tester/releases/$ref" ]] || fail 'maintenance service failure did not restore incumbent release' +if FAKE_TESTER_SYSTEMCTL_FAIL_WAIT=1 "$upgrade_repo/scripts/install-tester.sh" --upgrade --config /etc/ci-fleet-tester/tester.env --ref "$unit_fail_ref" >/dev/null 2>&1; then fail 'candidate activation did not wait for maintenance services'; fi +[[ $(readlink -f "$root/opt/ci-fleet-tester/current") == "$root/opt/ci-fleet-tester/releases/$ref" ]] || fail 'asynchronous maintenance failure did not restore incumbent release' # A corrupt incumbent is never recorded as the rollback target. chmod u+w "$release/scripts/tester-runtime.sh"; printf '# corrupt incumbent\n' >>"$release/scripts/tester-runtime.sh"; chmod 0555 "$release/scripts/tester-runtime.sh" diff --git a/scripts/tester-runtime.sh b/scripts/tester-runtime.sh index 878441e2..ff78c5d6 100755 --- a/scripts/tester-runtime.sh +++ b/scripts/tester-runtime.sh @@ -136,7 +136,7 @@ for name,service in services.items(): if service.get('privileged') or service.get('runtime') or service.get('network_mode') or service.get('pid') or service.get('ipc') or service.get('uts') or service.get('userns_mode') or service.get('cgroup') or service.get('external_links') or service.get('logging') or service.get('post_start') or service.get('pre_stop'): raise SystemExit(f'{name}: external runtime/namespace/link/logging/privileged lifecycle access is forbidden') deploy=service.get('deploy') or {}; reservations=(deploy.get('resources') or {}).get('reservations') or {} if service.get('build') or service.get('devices') or service.get('gpus') or reservations.get('devices') or service.get('cap_add') or service.get('container_name') or service.get('hostname') or service.get('use_api_socket') or service.get('volumes_from'): raise SystemExit(f'{name}: build/device/capability/external mount/global identity is forbidden') - if deploy.get('replicas',1) != 1: raise SystemExit(f'{name}: exactly one replica is required') + if service.get('scale',1) != 1 or deploy.get('replicas',1) != 1: raise SystemExit(f'{name}: exactly one replica is required') if service.get('environment') or service.get('env_file') or service.get('configs'): raise SystemExit(f'{name}: alternate credential channels are forbidden') if service.get('profiles'): raise SystemExit(f'{name}: Compose profiles are forbidden') if service.get('label_file'): raise SystemExit(f'{name}: external label files are forbidden') From e8a517afeb92772eb44970c9c35fba71ef8a65b9 Mon Sep 17 00:00:00 2001 From: Nickfost <1572453+Nickfost@users.noreply.github.com> Date: Sun, 30 Aug 2026 01:26:05 -0500 Subject: [PATCH 33/42] fix: close tester verification gaps --- scripts/install-tester.sh | 5 ++++- scripts/test-install-tester.sh | 12 ++++++++++++ scripts/tester-runtime.sh | 3 ++- 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/scripts/install-tester.sh b/scripts/install-tester.sh index 1aaf735d..e5287715 100755 --- a/scripts/install-tester.sh +++ b/scripts/install-tester.sh @@ -234,10 +234,13 @@ case $action in stage_release "$ref"; activate_release "$ref" ;; --check) - host_preflight; ensure_directories + host_preflight + for directory in "$opt_dir" "$release_dir" "$systemd_dir"; do secure_dir "$directory" 755; done + for directory in "$config_root" "$environment_dir" "$definition_dir" "$secret_root" "$state_root" "$runtime_state"; do secure_dir "$directory" 700; done secure_file "$(root_path "$config")" 600 current=$(installed_revision) || die 'tester is not installed' release_complete "$release_dir/$current" "$current" || die 'installed release is incomplete' + secure_file "$stable_launcher" 555 cmp -s "$release_dir/$current/scripts/tester-launcher.sh" "$stable_launcher" || die 'installed launcher differs from active release' for unit in "${units[@]}"; do cmp -s "$release_dir/$current/host/systemd/$unit" "$systemd_dir/$unit" || die "installed unit differs from active release: $unit"; done cmp -s "$release_dir/$current/host/systemd/$tmpfiles_conf" "$tmpfiles_dir/$tmpfiles_conf" || die 'installed tmpfiles rule differs from active release' diff --git a/scripts/test-install-tester.sh b/scripts/test-install-tester.sh index 5c531c03..70520e13 100755 --- a/scripts/test-install-tester.sh +++ b/scripts/test-install-tester.sh @@ -94,6 +94,9 @@ if FAKE_TESTER_ROUTE_PORT=18092 "$runtime" --converge --environment bad-source-l write_environment bad-tagged-label 18093 printf 'services:\n web:\n ! label_file: /root/credential.env\n' >"$root/etc/ci-fleet-tester/definitions/bad-tagged-label.yaml" if FAKE_TESTER_ROUTE_PORT=18093 "$runtime" --converge --environment bad-tagged-label >/dev/null 2>&1; then fail 'verbatim-tagged Compose label_file was accepted before rendering'; fi +write_environment bad-aliased-label 18094 +printf 'x-key: &external-key label_file\nservices:\n web:\n *external-key: /root/credential.env\n' >"$root/etc/ci-fleet-tester/definitions/bad-aliased-label.yaml" +if FAKE_TESTER_ROUTE_PORT=18094 "$runtime" --converge --environment bad-aliased-label >/dev/null 2>&1; then fail 'aliased Compose label_file was accepted before rendering'; fi write_environment interpolation 18090 TOKEN=must-not-render FAKE_TESTER_ROUTE_PORT=18090 FAKE_TESTER_POLICY=interpolation "$runtime" --converge --environment interpolation >/dev/null if grep -Fq must-not-render "$root/var/lib/ci-fleet-tester/environments/interpolation.compose.json"; then fail 'caller environment was interpolated into the Compose model'; fi @@ -177,6 +180,10 @@ for service in ci-fleet-tester-health.service ci-fleet-tester-cleanup.service; d "$installer" --install --config /etc/ci-fleet-tester/tester.env --ref "$ref" >/dev/null check_output=$("$installer" --check --config /etc/ci-fleet-tester/tester.env) grep -Fq CHECK_OK <<<"$check_output" || fail 'installed check failed' +stable_launcher=$root/opt/ci-fleet-tester/tester-runtime +chmod 0777 "$stable_launcher" +if "$installer" --check --config /etc/ci-fleet-tester/tester.env >/dev/null 2>&1; then fail 'writable privileged launcher passed check'; fi +chmod 0555 "$stable_launcher" for unit in ci-fleet-tester-health.service ci-fleet-tester-health.timer ci-fleet-tester-cleanup.service ci-fleet-tester-cleanup.timer; do [[ -f $root/etc/systemd/system/$unit ]] || fail "unit missing: $unit"; done for service in ci-fleet-tester-health.service ci-fleet-tester-cleanup.service; do grep -Fq 'TimeoutStartSec=300' "$root/etc/systemd/system/$service" || fail "$service does not bound oneshot start time" @@ -201,6 +208,11 @@ installed_tmpfiles=$root/usr/lib/tmpfiles.d/ci-fleet-tester-lock.conf rm "$installed_tmpfiles" if "$installer" --check --config /etc/ci-fleet-tester/tester.env >/dev/null 2>&1; then fail 'missing installed tmpfiles rule passed check'; fi cp "$release/host/systemd/ci-fleet-tester-lock.conf" "$installed_tmpfiles" +runtime_state=$root/var/lib/ci-fleet-tester/environments +rmdir "$runtime_state" +if "$installer" --check --config /etc/ci-fleet-tester/tester.env >/dev/null 2>&1; then fail 'missing runtime state directory passed check'; fi +[[ ! -e $runtime_state ]] || fail 'check recreated the missing runtime state directory' +install -d -m 0700 "$runtime_state" # A syntactically valid candidate that fails its post-switch check restores the incumbent. upgrade_repo=$tmp/upgrade-repo diff --git a/scripts/tester-runtime.sh b/scripts/tester-runtime.sh index ff78c5d6..56ba3dee 100755 --- a/scripts/tester-runtime.sh +++ b/scripts/tester-runtime.sh @@ -116,7 +116,8 @@ text=open(sys.argv[1],encoding='utf-8').read() key=r'(?:(?:!!str|!<[^>\n]+>)[ \t]+)?(?:include|label_file|"(?:include|label_file)"|\x27(?:include|label_file)\x27)[ \t]*:' explicit_key=r'(?m)^[ \t]*\?' escaped_key=r'"[^"\n]*\\[^"\n]*"[ \t]*:' -if re.search(r'(?m)^[ \t]*'+key,text) or re.search(r'[,{][ \t]*'+key,text) or re.search(escaped_key,text) or re.search(explicit_key,text): raise SystemExit('Compose include, label_file, or escaped mapping key is forbidden') +alias_key=r'(?m)(?:^[ \t]*|[,{][ \t]*)\*[A-Za-z0-9_-]+[ \t]*:' +if re.search(r'(?m)^[ \t]*'+key,text) or re.search(r'[,{][ \t]*'+key,text) or re.search(escaped_key,text) or re.search(explicit_key,text) or re.search(alias_key,text): raise SystemExit('Compose include, label_file, or indirect mapping key is forbidden') PY then return 1; fi if [[ ${CI_FLEET_TESTING:-0} == 1 ]]; then for variable in ${!FAKE_@}; do clean_environment+=("$variable=${!variable}"); done; fi From 1892638f52cfe958442a04b254663c81828b3867 Mon Sep 17 00:00:00 2001 From: Nickfost <1572453+Nickfost@users.noreply.github.com> Date: Sun, 30 Aug 2026 02:07:50 -0500 Subject: [PATCH 34/42] fix: close tester lifecycle verification gaps --- scripts/install-tester.sh | 13 ++++++++---- scripts/test-install-tester.sh | 36 +++++++++++++++++++++++++++++++++- 2 files changed, 44 insertions(+), 5 deletions(-) diff --git a/scripts/install-tester.sh b/scripts/install-tester.sh index e5287715..0e19635b 100755 --- a/scripts/install-tester.sh +++ b/scripts/install-tester.sh @@ -49,6 +49,7 @@ runtime_state=$state_root/environments docker_root=$(root_path /var/lib/docker) docker_socket=$(root_path /var/run/docker.sock) runtime_lock=$(root_path /run/lock/ci-fleet-tester/runtime.lock) +lifecycle_lock=$(root_path /run/lock/ci-fleet-tester/lifecycle.lock) services=(ci-fleet-tester-health.service ci-fleet-tester-cleanup.service) timers=(ci-fleet-tester-health.timer ci-fleet-tester-cleanup.timer) units=("${services[@]}" "${timers[@]}") @@ -68,6 +69,8 @@ reject_git_replacements() { acquire_lifecycle_lock() { mkdir -m 0755 "$(dirname "$runtime_lock")" 2>/dev/null || true secure_dir "$(dirname "$runtime_lock")" 755 + exec 9>"$lifecycle_lock" + flock -x 9 exec 8>"$runtime_lock" flock -x 8 export CI_FLEET_TESTER_LOCK_FD=8 @@ -148,7 +151,7 @@ create_tmpfiles() { # the installed tmpfiles.d drop-in) so maintenance units work after a reboot. mkdir -m 0755 "$(dirname "$runtime_lock")" 2>/dev/null || true secure_dir "$(dirname "$runtime_lock")" 755 - if command -v systemd-tmpfiles >/dev/null 2>&1; then systemd-tmpfiles --create "$tmpfiles_dir/$tmpfiles_conf" 2>/dev/null || true; fi + if command -v systemd-tmpfiles >/dev/null 2>&1; then systemd-tmpfiles --create "$tmpfiles_dir/$tmpfiles_conf" 2>/dev/null; fi } enable_timers() { systemctl enable --now "${timers[@]}" >/dev/null; } @@ -242,7 +245,11 @@ case $action in release_complete "$release_dir/$current" "$current" || die 'installed release is incomplete' secure_file "$stable_launcher" 555 cmp -s "$release_dir/$current/scripts/tester-launcher.sh" "$stable_launcher" || die 'installed launcher differs from active release' - for unit in "${units[@]}"; do cmp -s "$release_dir/$current/host/systemd/$unit" "$systemd_dir/$unit" || die "installed unit differs from active release: $unit"; done + for unit in "${units[@]}"; do + secure_file "$systemd_dir/$unit" 644 + cmp -s "$release_dir/$current/host/systemd/$unit" "$systemd_dir/$unit" || die "installed unit differs from active release: $unit" + done + secure_file "$tmpfiles_dir/$tmpfiles_conf" 644 cmp -s "$release_dir/$current/host/systemd/$tmpfiles_conf" "$tmpfiles_dir/$tmpfiles_conf" || die 'installed tmpfiles rule differs from active release' for timer in "${timers[@]}"; do if ! systemctl is-enabled --quiet "$timer" || ! systemctl is-active --quiet "$timer"; then die "timer is inactive: $timer"; fi @@ -258,9 +265,7 @@ case $action in --rollback) ensure_directories; secure_file "$lkg_file" 600 target=$(<"$lkg_file"); [[ $target =~ ^[0-9a-f]{40}$ ]] || die 'last-known-good revision is invalid' - current=$(installed_revision || true) activate_release "$target" - [[ ! $current =~ ^[0-9a-f]{40}$ || $current == "$target" ]] || write_lkg "$current" report "ROLLBACK_OK source_revision=$target" ;; --uninstall) diff --git a/scripts/test-install-tester.sh b/scripts/test-install-tester.sh index 70520e13..f0d5c48b 100755 --- a/scripts/test-install-tester.sh +++ b/scripts/test-install-tester.sh @@ -28,11 +28,17 @@ printf '%s\n' "$*" >>"${FAKE_TESTER_SYSTEMCTL_LOG:?}" [[ -z ${FAKE_TESTER_EVENT_LOG:-} ]] || printf 'systemctl %s\n' "$*" >>"$FAKE_TESTER_EVENT_LOG" if [[ ${FAKE_TESTER_SYSTEMCTL_FAIL_IF_UNITS_MISSING:-0} == 1 && $1 == disable && ! -e ${CI_FLEET_ROOT_PREFIX:?}/etc/systemd/system/ci-fleet-tester-health.timer ]]; then exit 5; fi if [[ ${FAKE_TESTER_SYSTEMCTL_FAIL_WAIT:-0} == 1 && $1 == start && " $* " == *' ci-fleet-tester-health.service '* && " $* " != *' --no-block '* ]]; then exit 7; fi +if [[ -n ${FAKE_TESTER_SYSTEMCTL_WAIT_READY:-} && $1 == start && " $* " != *' --no-block '* ]]; then + touch "$FAKE_TESTER_SYSTEMCTL_WAIT_READY" + while [[ ! -e ${FAKE_TESTER_SYSTEMCTL_WAIT_RELEASE:?} ]]; do sleep 0.05; done +fi [[ -z ${FAKE_TESTER_SYSTEMCTL_FAIL:-} || " $* " != *" $FAKE_TESTER_SYSTEMCTL_FAIL "* ]] EOF printf '#!/usr/bin/env bash\nexit 0\n' >"$fake_bin/curl" printf '#!/usr/bin/env bash\nexit 0\n' >"$fake_bin/getent" -chmod 0755 "$fake_bin/df" "$fake_bin/systemctl" "$fake_bin/curl" "$fake_bin/getent" +# shellcheck disable=SC2016 # Write the expansion for the fake to evaluate. +printf '%s\n' '#!/usr/bin/env bash' '[[ ${FAKE_TESTER_TMPFILES_FAIL:-0} != 1 ]]' >"$fake_bin/systemd-tmpfiles" +chmod 0755 "$fake_bin/df" "$fake_bin/systemctl" "$fake_bin/curl" "$fake_bin/getent" "$fake_bin/systemd-tmpfiles" export PATH="$fake_bin:$PATH" CI_FLEET_TESTING=1 CI_FLEET_ROOT_PREFIX=$root export FAKE_TESTER_DOCKER_ROOT=$root/var/lib/docker FAKE_TESTER_VOLUME_ROOT=$root/var/lib/fake-tester-volume FAKE_TESTER_DOCKER_LOG=$tmp/docker.log FAKE_TESTER_SYSTEMCTL_LOG=$tmp/systemctl.log export FAKE_TESTER_EVENT_LOG=$tmp/events.log @@ -208,6 +214,13 @@ installed_tmpfiles=$root/usr/lib/tmpfiles.d/ci-fleet-tester-lock.conf rm "$installed_tmpfiles" if "$installer" --check --config /etc/ci-fleet-tester/tester.env >/dev/null 2>&1; then fail 'missing installed tmpfiles rule passed check'; fi cp "$release/host/systemd/ci-fleet-tester-lock.conf" "$installed_tmpfiles" +unsafe_policy_accepted=0 +for policy_file in "$installed_unit" "$installed_tmpfiles"; do + chmod 0666 "$policy_file" + if "$installer" --check --config /etc/ci-fleet-tester/tester.env >/dev/null 2>&1; then unsafe_policy_accepted=1; fi + chmod 0644 "$policy_file" +done +((unsafe_policy_accepted == 0)) || fail 'writable installed systemd policy passed check' runtime_state=$root/var/lib/ci-fleet-tester/environments rmdir "$runtime_state" if "$installer" --check --config /etc/ci-fleet-tester/tester.env >/dev/null 2>&1; then fail 'missing runtime state directory passed check'; fi @@ -245,6 +258,8 @@ git -C "$upgrade_repo" show "$ref:scripts/tester-runtime.sh" >"$upgrade_repo/scr git -C "$upgrade_repo" add scripts/tester-runtime.sh git -C "$upgrade_repo" -c user.name=Example -c user.email=example@invalid.example commit --quiet -m 'fixture: valid tester candidate' unit_fail_ref=$(git -C "$upgrade_repo" rev-parse HEAD) +if FAKE_TESTER_TMPFILES_FAIL=1 "$upgrade_repo/scripts/install-tester.sh" --upgrade --config /etc/ci-fleet-tester/tester.env --ref "$unit_fail_ref" >/dev/null 2>&1; then fail 'tmpfiles creation failure succeeded'; fi +[[ $(readlink -f "$root/opt/ci-fleet-tester/current") == "$root/opt/ci-fleet-tester/releases/$ref" ]] || fail 'tmpfiles creation failure did not restore incumbent release' unit_hash_before=$(sha256sum "$root/etc/systemd/system/ci-fleet-tester-health.service") if FAKE_TESTER_SYSTEMCTL_FAIL=daemon-reload "$upgrade_repo/scripts/install-tester.sh" --upgrade --config /etc/ci-fleet-tester/tester.env --ref "$unit_fail_ref" >"$tmp/unit-restore.log" 2>&1; then fail 'unit activation failure succeeded'; fi grep -Fq 'incumbent unit restore failed' "$tmp/unit-restore.log" || fail 'secondary incumbent restore failure was not surfaced' @@ -271,6 +286,20 @@ check_line=$(grep -n '^docker info ' "$tmp/events.log" | tail -1 | cut -d: -f1) enable_line=$(grep -n '^systemctl enable --now ' "$tmp/events.log" | tail -1 | cut -d: -f1) [[ -n $disable_line && -n $check_line && -n $enable_line && $disable_line -lt $check_line && $check_line -lt $enable_line ]] || fail 'timers were not quiesced until candidate validation completed' +# Maintenance probes release the runtime lock but retain lifecycle serialization. +probe_ready=$tmp/probe-ready +probe_release=$tmp/probe-release +FAKE_TESTER_SYSTEMCTL_WAIT_READY=$probe_ready FAKE_TESTER_SYSTEMCTL_WAIT_RELEASE=$probe_release \ + "$upgrade_repo/scripts/install-tester.sh" --upgrade --config /etc/ci-fleet-tester/tester.env --ref "$valid_ref" >/dev/null & probe_pid=$! +while [[ ! -e $probe_ready ]]; do kill -0 "$probe_pid" 2>/dev/null || fail 'maintenance probe fixture exited early'; done +set +e +timeout 2 "$installer" --check --config /etc/ci-fleet-tester/tester.env >/dev/null 2>&1 +probe_lock_rc=$? +set -e +touch "$probe_release" +wait "$probe_pid" +[[ $probe_lock_rc == 124 ]] || fail 'maintenance probe allowed another lifecycle action' + # Rollback switches only to a complete recorded release and keeps environments intact. old=0000000000000000000000000000000000000000 cp -a "$root/opt/ci-fleet-tester/releases/$ref" "$root/opt/ci-fleet-tester/releases/$old" @@ -284,8 +313,13 @@ printf '%s\n' "$old" >"$root/var/lib/ci-fleet-tester/last-known-good"; chmod 600 write_environment rollback-env 18086 FAKE_TESTER_ROUTE_PORT=18086 "$runtime" --converge --environment rollback-env >/dev/null rollback_state_hash=$(sha256sum "$root/var/lib/ci-fleet-tester/environments/rollback-env.state") +pre_rollback_release=$(readlink -f "$root/opt/ci-fleet-tester/current") +chmod u+w "$pre_rollback_release/scripts/tester-runtime.sh" +printf '# corrupt before rollback\n' >>"$pre_rollback_release/scripts/tester-runtime.sh" +chmod 0555 "$pre_rollback_release/scripts/tester-runtime.sh" "$installer" --rollback --config /etc/ci-fleet-tester/tester.env | grep -Fq ROLLBACK_OK || fail 'rollback failed' [[ $(readlink -f "$root/opt/ci-fleet-tester/current") == "$root/opt/ci-fleet-tester/releases/$old" ]] || fail 'rollback selected the wrong release' +[[ $(<"$root/var/lib/ci-fleet-tester/last-known-good") == "$old" ]] || fail 'rollback recorded a corrupt prior release' [[ $(sha256sum "$root/var/lib/ci-fleet-tester/environments/rollback-env.state") == "$rollback_state_hash" ]] || fail 'rollback changed active environment state' FAKE_TESTER_ROUTE_PORT=18086 "$runtime" --remove --environment rollback-env >/dev/null From 818b78024fe1239c1da2e24b37275883dbdcb28f Mon Sep 17 00:00:00 2001 From: Nickfost <1572453+Nickfost@users.noreply.github.com> Date: Sun, 30 Aug 2026 02:29:12 -0500 Subject: [PATCH 35/42] fix: close remaining tester policy gaps --- docs/TESTER-HOST.md | 4 ++-- scripts/fixtures/fake-tester-docker.sh | 4 +++- scripts/install-tester.sh | 13 +++++++++++-- scripts/test-install-tester.sh | 18 ++++++++++++++++-- scripts/tester-runtime.sh | 12 +++++++++--- 5 files changed, 41 insertions(+), 10 deletions(-) diff --git a/docs/TESTER-HOST.md b/docs/TESTER-HOST.md index 670caa74..6f78caf9 100644 --- a/docs/TESTER-HOST.md +++ b/docs/TESTER-HOST.md @@ -13,7 +13,7 @@ A tester host accepts only: - application images addressed by an immutable `sha256` digest; - environment secrets stored below that environment's fixed host-local secret directory. -It rejects mutable images, public port binds, host bind mounts, external/unscoped Docker resources, custom volume drivers/options, privileged containers, added capabilities, host or shared namespaces, Docker API access, global container names, and credentials outside the environment secret boundary. Compose environment variables, env files, and configs are forbidden credential channels; use only fixed mode-`0600` Compose secrets. Every service must be read-only, drop all capabilities, and set `no-new-privileges=true`. The validated rendered Compose model is copied into protected runtime state before activation, so partial starts remain tracked and later cleanup does not depend on a mutable or deleted source definition. Test identity, networks, storage, routes, domains, data, and credentials must have no production authority. Host/network isolation is an external acceptance gate, not something this repository-only change can prove. +It rejects mutable images, public port binds, host bind mounts, external/unscoped Docker resources, custom volume drivers/options, privileged containers, added capabilities, host or shared namespaces, Docker API access, global container names, and credentials outside the environment secret boundary. Compose environment variables, env files, and configs are forbidden credential channels; use only fixed mode-`0600` Compose secrets. Every service must define positive CPU, memory, and PID limits, remain OOM-killable, be read-only, drop all capabilities, and set `no-new-privileges=true`. The validated rendered Compose model is copied into protected runtime state before activation, so partial starts remain tracked and later cleanup does not depend on a mutable or deleted source definition. Test identity, networks, storage, routes, domains, data, and credentials must have no production authority. Host/network isolation is an external acceptance gate, not something this repository-only change can prove. ## Prepare host-local configuration @@ -52,7 +52,7 @@ CI_FLEET_TESTER_ROUTE_SERVICE=web CI_FLEET_TESTER_ROUTE_PORT=18080 ``` -The Compose file is root-owned mode `0644` and may contain no credential value. Each image must use `registry/path@sha256:REVIEWED_64_HEX_DIGEST`. Exactly one route is published, on loopback only, at the declared port. Compose-generated network and volume names must remain below `ci-fleet-test-_...`; explicit external names are rejected. +The Compose file is root-owned mode `0644` and may contain no credential value. Each image must use `registry/path@sha256:REVIEWED_64_HEX_DIGEST`, and each service must set positive `cpus`, `mem_limit`, and `pids_limit` values. Exactly one route is published, on loopback only, at the declared port. Compose-generated network and volume names must remain below `ci-fleet-test-_...`; explicit external names are rejected. If credentials are required, create `/etc/ci-fleet-tester/secrets/example-preview` as root-owned mode `0700`, put only test-scope regular files there as root-owned mode `0600`, and reference them through Compose `secrets.file`. Symlinks, external secrets, production credentials, environment-variable secret transport, and files outside that exact directory are unsupported. diff --git a/scripts/fixtures/fake-tester-docker.sh b/scripts/fixtures/fake-tester-docker.sh index 5383dc8a..f8c00e0e 100755 --- a/scripts/fixtures/fake-tester-docker.sh +++ b/scripts/fixtures/fake-tester-docker.sh @@ -27,7 +27,7 @@ done case ${operation:-} in config) digest=$(printf 'a%.0s' {1..64}) - privileged=false; read_only=true; host_ip=127.0.0.1; image="registry.example/example/app@sha256:$digest"; network_name="${project}_default"; secrets='{}'; service_extra=; top_extra=; volume_extra=; network_extra=; security='no-new-privileges:true' + privileged=false; read_only=true; host_ip=127.0.0.1; image="registry.example/example/app@sha256:$digest"; network_name="${project}_default"; secrets='{}'; service_extra=',"cpus":0.5,"mem_limit":134217728,"pids_limit":128'; top_extra=; volume_extra=; network_extra=; security='no-new-privileges:true' case ${FAKE_TESTER_POLICY:-valid} in mutable) image=registry.example/example/app:latest ;; privileged) privileged=true ;; @@ -60,6 +60,8 @@ case ${operation:-} in profiles) service_extra=',"profiles":["extra"]' ;; label-file) service_extra=',"label_file":"/root/credential.env"' ;; runtime) service_extra=',"runtime":"alternative"' ;; + unbounded) service_extra=',"cpus":0,"mem_limit":0,"pids_limit":-1' ;; + oom-priority) service_extra=',"cpus":0.5,"mem_limit":134217728,"pids_limit":128,"oom_kill_disable":true,"oom_score_adj":-1000' ;; valid-secret|outside-secret) secrets=$(printf '{"credential":{"file":"%s"}}' "${FAKE_TESTER_SECRET_FILE:?}") ;; esac volume=${volume:-'{"type":"volume","source":"data","target":"/data"}'} diff --git a/scripts/install-tester.sh b/scripts/install-tester.sh index 0e19635b..10639a11 100755 --- a/scripts/install-tester.sh +++ b/scripts/install-tester.sh @@ -31,9 +31,18 @@ root_path() { printf '%s%s' "$root_prefix" "$1"; } expected_uid=0 [[ ${CI_FLEET_TESTING:-0} != 1 ]] || expected_uid=$(id -u) if [[ ${CI_FLEET_TESTING:-0} != 1 && ${EUID:-$(id -u)} -ne 0 ]]; then die 'run installer as root'; fi -for command in awk bash chmod cmp curl date df dirname docker du find flock getent git grep install ln mktemp mv python3 readlink rm sha256sum shellcheck stat systemctl tar wc; do command -v "$command" >/dev/null || die "required command is unavailable: $command"; done +case $action in + --install|--upgrade) required_commands='awk bash chmod cmp curl date df dirname docker du find flock getent git grep install ln mktemp mv python3 readlink rm sha256sum shellcheck stat systemctl tar wc' ;; + --check) required_commands='awk bash basename cmp df dirname docker env flock grep mkdir readlink sha256sum stat systemctl' ;; + --reset) required_commands='bash basename dirname env flock mkdir readlink stat' ;; + --rollback) required_commands='bash basename chmod dirname env flock install ln mkdir mv readlink rm sha256sum stat systemctl' ;; + --uninstall) required_commands='bash chmod dirname env find flock grep install mkdir rm stat systemctl' ;; +esac +for command in $required_commands; do command -v "$command" >/dev/null || die "required command is unavailable: $command"; done -repo_root=$(git -C "$(dirname "${BASH_SOURCE[0]}")/.." rev-parse --show-toplevel 2>/dev/null) || die 'installer must run from a Git checkout' +if [[ $action == --install || $action == --upgrade ]]; then + repo_root=$(git -C "$(dirname "${BASH_SOURCE[0]}")/.." rev-parse --show-toplevel 2>/dev/null) || die 'installer must run from a Git checkout' +fi opt_dir=$(root_path /opt/ci-fleet-tester) release_dir=$opt_dir/releases current_link=$opt_dir/current diff --git a/scripts/test-install-tester.sh b/scripts/test-install-tester.sh index f0d5c48b..bacd2f4d 100755 --- a/scripts/test-install-tester.sh +++ b/scripts/test-install-tester.sh @@ -43,6 +43,14 @@ export PATH="$fake_bin:$PATH" CI_FLEET_TESTING=1 CI_FLEET_ROOT_PREFIX=$root export FAKE_TESTER_DOCKER_ROOT=$root/var/lib/docker FAKE_TESTER_VOLUME_ROOT=$root/var/lib/fake-tester-volume FAKE_TESTER_DOCKER_LOG=$tmp/docker.log FAKE_TESTER_SYSTEMCTL_LOG=$tmp/systemctl.log export FAKE_TESTER_EVENT_LOG=$tmp/events.log +standalone_installer=$tmp/install-tester.sh +cp "$installer" "$standalone_installer" +uninstall_bin=$tmp/uninstall-bin +mkdir "$uninstall_bin" +for command in bash chmod dirname env find flock grep id install mkdir rm stat; do ln -s "$(command -v "$command")" "$uninstall_bin/$command"; done +ln -s "$fake_bin/systemctl" "$uninstall_bin/systemctl" +PATH=$uninstall_bin "$standalone_installer" --uninstall --config /etc/ci-fleet-tester/tester.env >/dev/null || fail 'standalone uninstall required installation-only tools or a Git checkout' + write_environment() { local id=$1 port=$2 mkdir -p "$root/etc/ci-fleet-tester/secrets/$id"; chmod 700 "$root/etc/ci-fleet-tester/secrets/$id" @@ -60,6 +68,10 @@ if "$runtime" --check >/dev/null 2>&1; then fail 'symlinked lifecycle lock direc [[ $(stat -c %a "$tmp/lock-target") == 700 ]] || fail 'symlinked lock target permissions changed' rm "$root/run/lock/ci-fleet-tester" if FAKE_TESTER_DOCKER_ROOT=/remote/docker "$runtime" --check >/dev/null 2>&1; then fail 'remote Docker daemon was accepted'; fi +write_environment secretless 18079 +rmdir "$root/etc/ci-fleet-tester/secrets/secretless" +FAKE_TESTER_ROUTE_PORT=18079 "$runtime" --converge --environment secretless >/dev/null || fail 'secretless environment required an empty secret directory' +FAKE_TESTER_ROUTE_PORT=18079 "$runtime" --remove --environment secretless >/dev/null write_environment preview-a 18080 FAKE_TESTER_ROUTE_PORT=18080 "$runtime" --converge --environment preview-a | grep -Fq CONVERGED || fail 'converge failed' state=$root/var/lib/ci-fleet-tester/environments/preview-a.state @@ -79,7 +91,7 @@ FAKE_TESTER_ROUTE_PORT=18080 "$runtime" --converge --environment preview-a >/dev [[ $(awk -F= '$1=="EXPIRES_AT"{print $2}' "$state") == "$original_expiry" ]] || fail 'idempotent converge extended expiration' write_environment preview-b 18080 if FAKE_TESTER_ROUTE_PORT=18080 "$runtime" --converge --environment preview-b >/dev/null 2>&1; then fail 'duplicate route port was accepted'; fi -for policy in mutable privileged bind broad-port external-network environment configs use-api-socket namespace-share false-nnp unconfined custom-volume volumes-from external-links userns-host cgroup-host uts-host remote-logging custom-network ipam replicas scale lifecycle-hook gpu deploy-device build profiles label-file runtime; do +for policy in mutable privileged bind broad-port external-network environment configs use-api-socket namespace-share false-nnp unconfined custom-volume volumes-from external-links userns-host cgroup-host uts-host remote-logging custom-network ipam replicas scale lifecycle-hook gpu deploy-device build profiles label-file runtime unbounded oom-priority; do write_environment "bad-$policy" 18081 if FAKE_TESTER_ROUTE_PORT=18081 FAKE_TESTER_POLICY=$policy "$runtime" --converge --environment "bad-$policy" >/dev/null 2>&1; then fail "unsafe compose policy was accepted: $policy"; fi done @@ -159,6 +171,8 @@ done "$runtime" --cleanup >/dev/null [[ ! -e $state && ! -e $root/var/lib/ci-fleet-tester/environments/expired-a.state && ! -e $root/var/lib/ci-fleet-tester/environments/expired-b.state ]] || fail 'expired environment survived cleanup' +[[ ${CI_FLEET_TESTER_RUNTIME_ONLY:-0} != 1 ]] || { printf 'TESTER_RUNTIME_TESTS_OK\n'; exit 0; } + # Commit-backed installer tests run after the implementation commit exists. ref=$(git -C "$repo_root" rev-parse HEAD) # A release archive containing a symlinked or non-regular member must be rejected @@ -357,6 +371,6 @@ if FAKE_TESTER_SYSTEMCTL_FAIL='disable --now' "$upgrade_repo/scripts/install-tes # than failing because a timer unit file is missing. "$installer" --install --config /etc/ci-fleet-tester/tester.env --ref "$ref" >/dev/null || fail 'fresh install failed before partial-unit test' rm -f "$root/etc/systemd/system/ci-fleet-tester-cleanup.timer" -"$installer" --uninstall --config /etc/ci-fleet-tester/tester.env | grep -Fq UNINSTALL_OK || fail 'uninstall failed on a partial unit set' +PATH=$uninstall_bin "$standalone_installer" --uninstall --config /etc/ci-fleet-tester/tester.env | grep -Fq UNINSTALL_OK || fail 'standalone uninstall required installation-only tools or a Git checkout' [[ -L $root/opt/ci-fleet-tester/current ]] && fail 'partial uninstall left the active release link' printf 'TESTER_INSTALLER_TESTS_OK\n' diff --git a/scripts/tester-runtime.sh b/scripts/tester-runtime.sh index 56ba3dee..97c81e10 100755 --- a/scripts/tester-runtime.sh +++ b/scripts/tester-runtime.sh @@ -104,7 +104,6 @@ load_spec() { secure_file "$compose_file" 644 compose_project=$(project_name "$id") secret_dir=$secret_root/$id - secure_directory "$secret_dir" 700 } validate_compose() { @@ -126,7 +125,7 @@ PY rm -f "$empty_env" chmod 600 "$rendered" python3 - "$rendered" "$route_service" "$route_port" "$compose_project" "$secret_dir" "$expected_uid" <<'PY' || return 1 -import json,os,re,stat,sys +import json,math,os,re,stat,sys value=json.load(open(sys.argv[1])); route_service=sys.argv[2]; route_port=int(sys.argv[3]); project=sys.argv[4]; secret_dir=sys.argv[5]; expected_uid=int(sys.argv[6]) services=value.get('services') if not isinstance(services,dict) or route_service not in services: raise SystemExit('route service is missing') @@ -138,6 +137,8 @@ for name,service in services.items(): deploy=service.get('deploy') or {}; reservations=(deploy.get('resources') or {}).get('reservations') or {} if service.get('build') or service.get('devices') or service.get('gpus') or reservations.get('devices') or service.get('cap_add') or service.get('container_name') or service.get('hostname') or service.get('use_api_socket') or service.get('volumes_from'): raise SystemExit(f'{name}: build/device/capability/external mount/global identity is forbidden') if service.get('scale',1) != 1 or deploy.get('replicas',1) != 1: raise SystemExit(f'{name}: exactly one replica is required') + if not all(isinstance(service.get(key),(int,float)) and not isinstance(service[key],bool) and math.isfinite(service[key]) and service[key] > 0 for key in ('cpus','mem_limit','pids_limit')): raise SystemExit(f'{name}: positive CPU, memory, and PID limits are required') + if service.get('oom_kill_disable') or service.get('oom_score_adj',0) < 0: raise SystemExit(f'{name}: OOM priority overrides are forbidden') if service.get('environment') or service.get('env_file') or service.get('configs'): raise SystemExit(f'{name}: alternate credential channels are forbidden') if service.get('profiles'): raise SystemExit(f'{name}: Compose profiles are forbidden') if service.get('label_file'): raise SystemExit(f'{name}: external label files are forbidden') @@ -157,7 +158,12 @@ for section in ('networks','volumes'): if section == 'networks' and (item.get('driver') not in (None,'bridge') or item.get('driver_opts')): raise SystemExit(f'{section}.{name}: custom network drivers are forbidden') if section == 'networks' and item.get('ipam'): raise SystemExit(f'{section}.{name}: custom network IPAM configuration is forbidden') if section == 'volumes' and (item.get('driver') or item.get('driver_opts')): raise SystemExit(f'{section}.{name}: custom volume drivers are forbidden') -for name,item in value.get('secrets',{}).items(): +secrets=value.get('secrets',{}) +if secrets: + try: metadata=os.lstat(secret_dir) + except FileNotFoundError: raise SystemExit('environment secret directory is missing') + if not stat.S_ISDIR(metadata.st_mode) or stat.S_ISLNK(metadata.st_mode) or metadata.st_uid != expected_uid or stat.S_IMODE(metadata.st_mode) != 0o700: raise SystemExit('environment secret directory must be owner-controlled mode 0700') +for name,item in secrets.items(): path=item.get('file') if item.get('external') or not isinstance(path,str) or os.path.realpath(path).rsplit('/',1)[0] != secret_dir: raise SystemExit(f'secrets.{name}: secret must be a host-local file in the environment secret directory') metadata=os.lstat(path) From 8087c375ab3840aa9606e3e2e1548bb8e84b2255 Mon Sep 17 00:00:00 2001 From: Nickfost <1572453+Nickfost@users.noreply.github.com> Date: Sun, 30 Aug 2026 04:04:59 -0500 Subject: [PATCH 36/42] fix: close tester lifecycle verification gaps --- scripts/fixtures/fake-tester-docker.sh | 52 ++++++++++++++++++++++---- scripts/install-tester.sh | 33 +++++++++++++--- scripts/test-install-tester.sh | 17 ++++++++- scripts/tester-runtime.sh | 15 +++++--- 4 files changed, 98 insertions(+), 19 deletions(-) diff --git a/scripts/fixtures/fake-tester-docker.sh b/scripts/fixtures/fake-tester-docker.sh index f8c00e0e..8f0d599a 100755 --- a/scripts/fixtures/fake-tester-docker.sh +++ b/scripts/fixtures/fake-tester-docker.sh @@ -4,13 +4,42 @@ printf '%s\n' "$*" >>"${FAKE_TESTER_DOCKER_LOG:?}" [[ -z ${FAKE_TESTER_EVENT_LOG:-} ]] || printf 'docker %s\n' "$*" >>"$FAKE_TESTER_EVENT_LOG" if [[ $1 == context && $2 == show ]]; then printf 'default\n'; exit 0; fi if [[ $1 == info ]]; then printf '%s\n' "${FAKE_TESTER_DOCKER_ROOT:?}"; exit 0; fi -if [[ $1 == ps ]]; then [[ ${FAKE_TESTER_PS_FAIL:-0} != 1 ]] || exit 9; printf 'fixture-container-id\n'; exit 0; fi +if [[ $1 == ps ]]; then + [[ ${FAKE_TESTER_PS_FAIL:-0} != 1 ]] || exit 9 + if [[ " $* " == *' label=com.docker.compose.project '* ]]; then + [[ -z ${FAKE_TESTER_ORPHAN_PROJECT:-} ]] || printf 'fixture-orphan-container-id\n' + else + printf 'fixture-container-id\n' + fi + exit 0 +fi if [[ $1 == inspect ]]; then - if [[ " $* " == *' --size '* ]]; then printf '1024\n'; else printf '%s\n' "${FAKE_TESTER_CONTAINER_STATE:-running healthy}"; fi + if [[ " $* " == *' --size '* ]]; then + printf '1024\n' + elif [[ " $* " == *'com.docker.compose.project'* ]]; then + printf '%s\n' "${FAKE_TESTER_ORPHAN_PROJECT:-}" + elif [[ " $* " == *'com.docker.compose.service'* && " $* " == *'.Config.Image'* ]]; then + service=web + [[ ${*: -1} != fixture-worker-id || ${FAKE_TESTER_DUPLICATE_SERVICE:-0} == 1 ]] || service=worker + printf '%s %s\n' "$service" "${FAKE_TESTER_CONTAINER_IMAGE:-registry.example/example/app@sha256:$(printf 'a%.0s' {1..64})}" + else + printf '%s\n' "${FAKE_TESTER_CONTAINER_STATE:-running healthy}" + fi + exit 0 +fi +if [[ $1 == volume && $2 == ls ]]; then + [[ ${FAKE_TESTER_VOLUME_LS_FAIL:-0} != 1 ]] || exit 9 + if [[ " $* " == *' label=com.docker.compose.project=ci-fleet-test-'* ]]; then printf 'fixture-volume\n'; + elif [[ -n ${FAKE_TESTER_ORPHAN_VOLUME_PROJECT:-} ]]; then printf 'fixture-orphan-volume-id\n'; fi exit 0 fi -if [[ $1 == volume && $2 == ls ]]; then [[ ${FAKE_TESTER_VOLUME_LS_FAIL:-0} != 1 ]] || exit 9; printf 'fixture-volume\n'; exit 0; fi -if [[ $1 == volume && $2 == inspect ]]; then printf '%s\n' "${FAKE_TESTER_VOLUME_ROOT:?}"; exit 0; fi +if [[ $1 == volume && $2 == inspect ]]; then + if [[ " $* " == *'com.docker.compose.project'* ]]; then printf '%s\n' "${FAKE_TESTER_ORPHAN_VOLUME_PROJECT:-}"; + else printf '%s\n' "${FAKE_TESTER_VOLUME_ROOT:?}"; fi + exit 0 +fi +if [[ $1 == network && $2 == ls ]]; then [[ -z ${FAKE_TESTER_ORPHAN_NETWORK_PROJECT:-} ]] || printf 'fixture-orphan-network-id\n'; exit 0; fi +if [[ $1 == network && $2 == inspect ]]; then printf '%s\n' "${FAKE_TESTER_ORPHAN_NETWORK_PROJECT:-}"; exit 0; fi if [[ $1 == compose && $2 == version ]]; then printf 'Docker Compose version v2.fixture\n'; exit 0; fi if [[ $1 == compose && $2 == up && ${3:-} == --help ]]; then [[ ${FAKE_TESTER_NO_WAIT_TIMEOUT:-0} != 1 ]] && printf '%s\n' ' --wait-timeout int'; exit 0; fi if [[ $1 != compose ]]; then exit 2; fi @@ -27,12 +56,13 @@ done case ${operation:-} in config) digest=$(printf 'a%.0s' {1..64}) - privileged=false; read_only=true; host_ip=127.0.0.1; image="registry.example/example/app@sha256:$digest"; network_name="${project}_default"; secrets='{}'; service_extra=',"cpus":0.5,"mem_limit":134217728,"pids_limit":128'; top_extra=; volume_extra=; network_extra=; security='no-new-privileges:true' + privileged=false; read_only=true; host_ip=127.0.0.1; protocol=tcp; image="registry.example/example/app@sha256:$digest"; network_name="${project}_default"; secrets='{}'; service_extra=',"cpus":0.5,"mem_limit":134217728,"pids_limit":128'; extra_services=; top_extra=; volume_extra=; network_extra=; security='no-new-privileges:true' case ${FAKE_TESTER_POLICY:-valid} in mutable) image=registry.example/example/app:latest ;; privileged) privileged=true ;; bind) volume='{"type":"bind","source":"/","target":"/host"}' ;; broad-port) host_ip=0.0.0.0 ;; + udp-route) protocol=udp ;; external-network) network_name=shared ;; environment) service_extra=',"environment":{"TOKEN":"example"}' ;; configs) service_extra=',"configs":[{"source":"credential"}]'; top_extra=',"configs":{"credential":{"file":"/tmp/example"}}' ;; @@ -62,14 +92,20 @@ case ${operation:-} in runtime) service_extra=',"runtime":"alternative"' ;; unbounded) service_extra=',"cpus":0,"mem_limit":0,"pids_limit":-1' ;; oom-priority) service_extra=',"cpus":0.5,"mem_limit":134217728,"pids_limit":128,"oom_kill_disable":true,"oom_score_adj":-1000' ;; + two-services) extra_services=$(printf ',"worker":{"image":"%s","privileged":false,"read_only":true,"cap_drop":["ALL"],"security_opt":["no-new-privileges:true"],"volumes":[],"cpus":0.5,"mem_limit":134217728,"pids_limit":128}' "$image") ;; valid-secret|outside-secret) secrets=$(printf '{"credential":{"file":"%s"}}' "${FAKE_TESTER_SECRET_FILE:?}") ;; esac volume=${volume:-'{"type":"volume","source":"data","target":"/data"}'} - printf '{"services":{"web":{"image":"%s","privileged":%s,"read_only":%s,"cap_drop":["ALL"],"security_opt":["%s"],"volumes":[%s],"ports":[{"host_ip":"%s","published":%s,"target":8080,"protocol":"tcp"}]%s}},"networks":{"default":{"name":"%s"%s}},"volumes":{"data":{"name":"%s_data"%s}},"secrets":%s%s}\n' \ - "$image" "$privileged" "$read_only" "$security" "$volume" "$host_ip" "${FAKE_TESTER_ROUTE_PORT:-18080}" "$service_extra" "$network_name" "$network_extra" "$project" "$volume_extra" "$secrets" "$top_extra" + printf '{"services":{"web":{"image":"%s","privileged":%s,"read_only":%s,"cap_drop":["ALL"],"security_opt":["%s"],"volumes":[%s],"ports":[{"host_ip":"%s","published":%s,"target":8080,"protocol":"%s"}]%s}%s},"networks":{"default":{"name":"%s"%s}},"volumes":{"data":{"name":"%s_data"%s}},"secrets":%s%s}\n' \ + "$image" "$privileged" "$read_only" "$security" "$volume" "$host_ip" "${FAKE_TESTER_ROUTE_PORT:-18080}" "$protocol" "$service_extra" "$extra_services" "$network_name" "$network_extra" "$project" "$volume_extra" "$secrets" "$top_extra" ;; up) [[ ${FAKE_TESTER_UP_FAIL:-0} != 1 ]] ;; down) [[ ${FAKE_TESTER_DOWN_FAIL:-0} != 1 ]] ;; - ps) [[ ${FAKE_TESTER_UNHEALTHY:-0} == 1 ]] || printf 'fixture-container-id\n' ;; + ps) + [[ ${FAKE_TESTER_UNHEALTHY:-0} == 1 ]] || { + printf 'fixture-container-id\n' + [[ ${FAKE_TESTER_POLICY:-valid} != two-services || $* != -q ]] || printf 'fixture-worker-id\n' + } + ;; *) exit 2 ;; esac diff --git a/scripts/install-tester.sh b/scripts/install-tester.sh index 10639a11..7ee91fd1 100755 --- a/scripts/install-tester.sh +++ b/scripts/install-tester.sh @@ -36,7 +36,7 @@ case $action in --check) required_commands='awk bash basename cmp df dirname docker env flock grep mkdir readlink sha256sum stat systemctl' ;; --reset) required_commands='bash basename dirname env flock mkdir readlink stat' ;; --rollback) required_commands='bash basename chmod dirname env flock install ln mkdir mv readlink rm sha256sum stat systemctl' ;; - --uninstall) required_commands='bash chmod dirname env find flock grep install mkdir rm stat systemctl' ;; + --uninstall) required_commands='bash chmod dirname docker env find flock grep install mkdir rm stat systemctl' ;; esac for command in $required_commands; do command -v "$command" >/dev/null || die "required command is unavailable: $command"; done @@ -85,6 +85,28 @@ acquire_lifecycle_lock() { export CI_FLEET_TESTER_LOCK_FD=8 } +pin_local_docker() { + [[ -z ${DOCKER_HOST:-} && -z ${DOCKER_CONTEXT:-} ]] || die 'Docker environment selectors are forbidden' + unset DOCKER_CONTEXT + export DOCKER_HOST="unix://$docker_socket" + [[ -S $docker_socket || ( ${CI_FLEET_TESTING:-0} == 1 && -e $docker_socket ) ]] || die 'local Docker socket is unavailable' +} + +reject_managed_compose_resources() { + local kind=$1 inventory resource project format + local -a inspect_command + case $kind in + container) inventory=$(docker ps -aq --filter label=com.docker.compose.project) || die "Docker Compose $kind inventory could not be inspected"; inspect_command=(docker inspect); format='{{index .Config.Labels "com.docker.compose.project"}}' ;; + volume) inventory=$(docker volume ls -q --filter label=com.docker.compose.project) || die "Docker Compose $kind inventory could not be inspected"; inspect_command=(docker volume inspect); format='{{index .Labels "com.docker.compose.project"}}' ;; + network) inventory=$(docker network ls -q --filter label=com.docker.compose.project) || die "Docker Compose $kind inventory could not be inspected"; inspect_command=(docker network inspect); format='{{index .Labels "com.docker.compose.project"}}' ;; + esac + while IFS= read -r resource; do + [[ -n $resource ]] || continue + project=$("${inspect_command[@]}" --format "$format" "$resource") || die "Docker Compose $kind identity could not be inspected" + [[ $project != ci-fleet-test-* ]] || die 'remove every managed Docker Compose project before uninstalling the tester service' + done <<<"$inventory" +} + host_preflight() { local os_release docker_context actual_root used os_release=$(root_path /etc/os-release) @@ -92,11 +114,8 @@ host_preflight() { # shellcheck disable=SC1090 . "$os_release" [[ ${ID:-} == debian && ${VERSION_ID:-} =~ ^[0-9]+$ && ${VERSION_ID%%.*} -ge 12 ]] || die 'tester hosts require Debian 12 or newer' - [[ -z ${DOCKER_HOST:-} && -z ${DOCKER_CONTEXT:-} ]] || die 'Docker environment selectors are forbidden' - unset DOCKER_CONTEXT - export DOCKER_HOST="unix://$docker_socket" + pin_local_docker docker_context=$(docker context show); [[ $docker_context == default ]] || die 'tester requires the local default Docker context' - [[ -S $docker_socket || ( ${CI_FLEET_TESTING:-0} == 1 && -e $docker_socket ) ]] || die 'local Docker socket is unavailable' actual_root=$(docker info --format '{{.DockerRootDir}}'); [[ $actual_root == "$docker_root" ]] || die 'Docker root does not match the local managed root' docker compose version >/dev/null printf 'services: {}\n' | docker compose -f - config --format json >/dev/null || die 'Compose JSON rendering is unavailable' @@ -280,6 +299,10 @@ case $action in --uninstall) ensure_directories if find "$runtime_state" -maxdepth 1 -type f -name '*.state' | grep -q .; then die 'remove every test environment before uninstalling the tester service'; fi + pin_local_docker + reject_managed_compose_resources container + reject_managed_compose_resources volume + reject_managed_compose_resources network remove_units || die 'could not stop and disable tester maintenance units' rm -f -- "$current_link" "$stable_launcher" "$lkg_file" [[ ${CI_FLEET_TESTING:-0} != 1 ]] || chmod -R u+w "$release_dir" diff --git a/scripts/test-install-tester.sh b/scripts/test-install-tester.sh index bacd2f4d..a914294e 100755 --- a/scripts/test-install-tester.sh +++ b/scripts/test-install-tester.sh @@ -48,6 +48,7 @@ cp "$installer" "$standalone_installer" uninstall_bin=$tmp/uninstall-bin mkdir "$uninstall_bin" for command in bash chmod dirname env find flock grep id install mkdir rm stat; do ln -s "$(command -v "$command")" "$uninstall_bin/$command"; done +ln -s "$fake_bin/docker" "$uninstall_bin/docker" ln -s "$fake_bin/systemctl" "$uninstall_bin/systemctl" PATH=$uninstall_bin "$standalone_installer" --uninstall --config /etc/ci-fleet-tester/tester.env >/dev/null || fail 'standalone uninstall required installation-only tools or a Git checkout' @@ -91,7 +92,7 @@ FAKE_TESTER_ROUTE_PORT=18080 "$runtime" --converge --environment preview-a >/dev [[ $(awk -F= '$1=="EXPIRES_AT"{print $2}' "$state") == "$original_expiry" ]] || fail 'idempotent converge extended expiration' write_environment preview-b 18080 if FAKE_TESTER_ROUTE_PORT=18080 "$runtime" --converge --environment preview-b >/dev/null 2>&1; then fail 'duplicate route port was accepted'; fi -for policy in mutable privileged bind broad-port external-network environment configs use-api-socket namespace-share false-nnp unconfined custom-volume volumes-from external-links userns-host cgroup-host uts-host remote-logging custom-network ipam replicas scale lifecycle-hook gpu deploy-device build profiles label-file runtime unbounded oom-priority; do +for policy in mutable privileged bind broad-port udp-route external-network environment configs use-api-socket namespace-share false-nnp unconfined custom-volume volumes-from external-links userns-host cgroup-host uts-host remote-logging custom-network ipam replicas scale lifecycle-hook gpu deploy-device build profiles label-file runtime unbounded oom-priority; do write_environment "bad-$policy" 18081 if FAKE_TESTER_ROUTE_PORT=18081 FAKE_TESTER_POLICY=$policy "$runtime" --converge --environment "bad-$policy" >/dev/null 2>&1; then fail "unsafe compose policy was accepted: $policy"; fi done @@ -127,6 +128,11 @@ if FAKE_TESTER_ROUTE_PORT=18080 FAKE_TESTER_POLICY=mutable "$runtime" --reset -- if FAKE_TESTER_ROUTE_PORT=18080 FAKE_TESTER_CONTAINER_STATE='exited unhealthy' "$runtime" --health >/dev/null 2>&1; then fail 'health accepted a stopped managed service'; fi if FAKE_TESTER_ROUTE_PORT=18080 FAKE_TESTER_CONTAINER_STATE='running none' "$runtime" --converge --environment preview-a >/dev/null 2>&1; then fail 'converge accepted a route without health evidence'; fi if FAKE_TESTER_ROUTE_PORT=18080 FAKE_TESTER_CONTAINER_STATE='running none' "$runtime" --health >/dev/null 2>&1; then fail 'health accepted a route without health evidence'; fi +if FAKE_TESTER_ROUTE_PORT=18080 FAKE_TESTER_CONTAINER_IMAGE="registry.example/example/app@sha256:$(printf 'b%.0s' {1..64})" "$runtime" --health >/dev/null 2>&1; then fail 'health accepted a container using an unapproved image digest'; fi +write_environment service-map 18090 +FAKE_TESTER_ROUTE_PORT=18090 FAKE_TESTER_POLICY=two-services "$runtime" --converge --environment service-map >/dev/null +if FAKE_TESTER_ROUTE_PORT=18090 FAKE_TESTER_POLICY=two-services FAKE_TESTER_DUPLICATE_SERVICE=1 "$runtime" --inspect --environment service-map | grep -q 'STATUS=running'; then fail 'health accepted duplicate and missing service identities'; fi +FAKE_TESTER_ROUTE_PORT=18090 FAKE_TESTER_POLICY=two-services "$runtime" --remove --environment service-map >/dev/null write_environment secret-preview 18082 secret_file=$root/etc/ci-fleet-tester/secrets/secret-preview/credential printf 'example-test-scope-value\n' >"$secret_file"; chmod 600 "$secret_file" @@ -357,6 +363,15 @@ launcher_lock_rc=$? set -e [[ $launcher_lock_rc == 124 ]] || fail 'stable launcher resolved the active release before acquiring the lifecycle lock' wait "$lock_pid" +: >"$tmp/docker.log" +if DOCKER_HOST=tcp://example.invalid:2375 "$installer" --uninstall --config /etc/ci-fleet-tester/tester.env >/dev/null 2>&1; then fail 'uninstall accepted a remote Docker selector'; fi +[[ -L $root/opt/ci-fleet-tester/current ]] || fail 'remote-selector uninstall removed the active release' +[[ ! -s $tmp/docker.log ]] || fail 'remote-selector uninstall contacted Docker' +if DOCKER_CONTEXT=remote "$installer" --uninstall --config /etc/ci-fleet-tester/tester.env >/dev/null 2>&1; then fail 'uninstall accepted a remote Docker context'; fi +if FAKE_TESTER_ORPHAN_PROJECT=ci-fleet-test-missing-state "$installer" --uninstall --config /etc/ci-fleet-tester/tester.env >/dev/null 2>&1; then fail 'uninstall ignored a managed Compose project with missing state'; fi +[[ -L $root/opt/ci-fleet-tester/current ]] || fail 'orphan-project uninstall removed the active release' +if FAKE_TESTER_ORPHAN_VOLUME_PROJECT=ci-fleet-test-missing-state "$installer" --uninstall --config /etc/ci-fleet-tester/tester.env >/dev/null 2>&1; then fail 'uninstall ignored a managed Compose volume with missing state'; fi +if FAKE_TESTER_ORPHAN_NETWORK_PROJECT=ci-fleet-test-missing-state "$installer" --uninstall --config /etc/ci-fleet-tester/tester.env >/dev/null 2>&1; then fail 'uninstall ignored a managed Compose network with missing state'; fi if FAKE_TESTER_SYSTEMCTL_FAIL='disable --now' "$installer" --uninstall --config /etc/ci-fleet-tester/tester.env >/dev/null 2>&1; then fail 'uninstall ignored systemd teardown failure'; fi [[ -L $root/opt/ci-fleet-tester/current ]] || fail 'failed uninstall removed the active release' "$installer" --uninstall --config /etc/ci-fleet-tester/tester.env | grep -Fq UNINSTALL_OK || fail 'uninstall failed' diff --git a/scripts/tester-runtime.sh b/scripts/tester-runtime.sh index 97c81e10..15ca86e3 100755 --- a/scripts/tester-runtime.sh +++ b/scripts/tester-runtime.sh @@ -147,7 +147,7 @@ for name,service in services.items(): for mount in service.get('volumes',[]): if isinstance(mount,str) or mount.get('type') not in ('volume','tmpfs'): raise SystemExit(f'{name}: host bind mounts are forbidden') for port in service.get('ports',[]): - if not isinstance(port,dict) or str(port.get('host_ip','')) != '127.0.0.1': raise SystemExit(f'{name}: published ports must bind loopback') + if not isinstance(port,dict) or str(port.get('host_ip','')) != '127.0.0.1' or port.get('protocol','tcp') != 'tcp': raise SystemExit(f'{name}: published ports must use TCP on loopback') ports.append((name,int(port.get('published',0)),int(port.get('target',0)))) if ports != [(route_service,route_port,ports[0][2] if ports else 0)] or not ports or ports[0][2] < 1: raise SystemExit('exactly one declared loopback route is required') if value.get('configs'): raise SystemExit('top-level configs are forbidden') @@ -238,8 +238,9 @@ remove_environment() { } inspect_environment() { - local target=$1 id compose route_service route_container status=running resource value bytes=0 mount expected running_state + local target=$1 id compose route_service status=running resource value bytes=0 mount expected running_state service configured_image local -a containers=() + local -A expected_images=() observed_services=() id=$(basename "$target" .state); secure_file "$target" 600 while IFS='=' read -r key value; do case $key in ENVIRONMENT|PROJECT|OWNER|ROUTE_PORT|EXPIRES_AT|SOURCE_REVISION|IMAGE_DIGESTS|UPDATED_AT) printf '%s=%s ' "$key" "$value" ;; esac @@ -247,18 +248,22 @@ inspect_environment() { compose=$(awk -F= '$1=="COMPOSE_FILE"{print substr($0,index($0,"=")+1)}' "$target") route_service=$(awk -F= '$1=="ROUTE_SERVICE"{print $2}' "$target") expected=$(python3 -c 'import json,sys; print(len(json.load(open(sys.argv[1]))["services"]))' "$compose") + while read -r service configured_image; do expected_images[$service]=$configured_image; done < <(python3 -c 'import json,sys; value=json.load(open(sys.argv[1])); print("\n".join("%s %s" % (name,service["image"]) for name,service in value["services"].items()))' "$compose") mapfile -t containers < <(docker compose -p "$(project_name "$id")" -f "$compose" ps -q) - route_container=$(docker compose -p "$(project_name "$id")" -f "$compose" ps -q "$route_service") || die 'route container inventory failed' [[ ${#containers[@]} == "$expected" ]] || status=unhealthy - [[ -n $route_container ]] || status=unhealthy for resource in "${containers[@]}"; do running_state=$(docker inspect --format '{{.State.Status}} {{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$resource") - if [[ $resource == "$route_container" ]]; then + read -r service configured_image < <(docker inspect --format '{{index .Config.Labels "com.docker.compose.service"}} {{.Config.Image}}' "$resource") + [[ -n $service ]] || { status=unhealthy; continue; } + [[ -n ${expected_images[$service]:-} && $configured_image == "${expected_images[$service]}" ]] || status=unhealthy + observed_services[$service]=$((${observed_services[$service]:-0} + 1)) + if [[ $service == "$route_service" ]]; then [[ $running_state == 'running healthy' ]] || status=unhealthy else [[ $running_state == 'running healthy' || $running_state == 'running none' ]] || status=unhealthy fi done + for service in "${!expected_images[@]}"; do [[ ${observed_services[$service]:-0} == 1 ]] || status=unhealthy; done inventory=$(docker ps -aq --filter "label=com.docker.compose.project=$(project_name "$id")") || die 'container inventory failed' while IFS= read -r resource; do [[ -n $resource ]] || continue From f1835871f278b2043d9bea2575f8eac35cd30f22 Mon Sep 17 00:00:00 2001 From: Nickfost <1572453+Nickfost@users.noreply.github.com> Date: Sun, 30 Aug 2026 04:21:30 -0500 Subject: [PATCH 37/42] fix: close tester policy and health gaps --- scripts/test-install-tester.sh | 7 ++++++- scripts/tester-runtime.sh | 11 ++++++++--- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/scripts/test-install-tester.sh b/scripts/test-install-tester.sh index a914294e..5e506c44 100755 --- a/scripts/test-install-tester.sh +++ b/scripts/test-install-tester.sh @@ -20,7 +20,8 @@ cp "$repo_root/scripts/fixtures/fake-tester-docker.sh" "$fake_bin/docker" chmod 0755 "$fake_bin/docker" cat >"$fake_bin/df" <<'EOF' #!/usr/bin/env bash -printf 'Filesystem 1024-blocks Used Available Capacity Mounted on\nfixture 100 20 80 20%% /fixture\n' +used=${FAKE_TESTER_DISK_USED_PERCENT:-20} +printf 'Filesystem 1024-blocks Used Available Capacity Mounted on\nfixture 100 %s 80 %s%% /fixture\n' "$used" "$used" EOF cat >"$fake_bin/systemctl" <<'EOF' #!/usr/bin/env bash @@ -116,6 +117,9 @@ if FAKE_TESTER_ROUTE_PORT=18093 "$runtime" --converge --environment bad-tagged-l write_environment bad-aliased-label 18094 printf 'x-key: &external-key label_file\nservices:\n web:\n *external-key: /root/credential.env\n' >"$root/etc/ci-fleet-tester/definitions/bad-aliased-label.yaml" if FAKE_TESTER_ROUTE_PORT=18094 "$runtime" --converge --environment bad-aliased-label >/dev/null 2>&1; then fail 'aliased Compose label_file was accepted before rendering'; fi +write_environment bad-flow-explicit-alias 18095 +printf '{x-key: &external-key include, ? *external-key : [{path: /root/other.yaml, env_file: /root/credential.env}], services: {}}\n' >"$root/etc/ci-fleet-tester/definitions/bad-flow-explicit-alias.yaml" +if FAKE_TESTER_ROUTE_PORT=18095 "$runtime" --converge --environment bad-flow-explicit-alias >/dev/null 2>&1; then fail 'flow-form explicit aliased Compose include was accepted before rendering'; fi write_environment interpolation 18090 TOKEN=must-not-render FAKE_TESTER_ROUTE_PORT=18090 FAKE_TESTER_POLICY=interpolation "$runtime" --converge --environment interpolation >/dev/null if grep -Fq must-not-render "$root/var/lib/ci-fleet-tester/environments/interpolation.compose.json"; then fail 'caller environment was interpolated into the Compose model'; fi @@ -129,6 +133,7 @@ if FAKE_TESTER_ROUTE_PORT=18080 FAKE_TESTER_CONTAINER_STATE='exited unhealthy' " if FAKE_TESTER_ROUTE_PORT=18080 FAKE_TESTER_CONTAINER_STATE='running none' "$runtime" --converge --environment preview-a >/dev/null 2>&1; then fail 'converge accepted a route without health evidence'; fi if FAKE_TESTER_ROUTE_PORT=18080 FAKE_TESTER_CONTAINER_STATE='running none' "$runtime" --health >/dev/null 2>&1; then fail 'health accepted a route without health evidence'; fi if FAKE_TESTER_ROUTE_PORT=18080 FAKE_TESTER_CONTAINER_IMAGE="registry.example/example/app@sha256:$(printf 'b%.0s' {1..64})" "$runtime" --health >/dev/null 2>&1; then fail 'health accepted a container using an unapproved image digest'; fi +if FAKE_TESTER_ROUTE_PORT=18080 FAKE_TESTER_DISK_USED_PERCENT=80 "$runtime" --health >/dev/null 2>&1; then fail 'scheduled health accepted Docker storage at the configured warning threshold'; fi write_environment service-map 18090 FAKE_TESTER_ROUTE_PORT=18090 FAKE_TESTER_POLICY=two-services "$runtime" --converge --environment service-map >/dev/null if FAKE_TESTER_ROUTE_PORT=18090 FAKE_TESTER_POLICY=two-services FAKE_TESTER_DUPLICATE_SERVICE=1 "$runtime" --inspect --environment service-map | grep -q 'STATUS=running'; then fail 'health accepted duplicate and missing service identities'; fi diff --git a/scripts/tester-runtime.sh b/scripts/tester-runtime.sh index 15ca86e3..a98eeed9 100755 --- a/scripts/tester-runtime.sh +++ b/scripts/tester-runtime.sh @@ -78,6 +78,11 @@ load_global() { [[ ${ENV_VALUES[CI_FLEET_TESTER_ISOLATION_ACK]:-} == test-only-no-production-authority ]] || die 'explicit test-only isolation acknowledgement is required' } +check_disk_threshold() { + used=$(df -P "$(root_path /var/lib/docker)" | awk 'NR==2{gsub(/%/,"",$5);print $5}') + [[ $used =~ ^[0-9]+$ && $used -lt $disk_warn ]] || die 'Docker storage exceeds configured warning threshold' +} + project_name() { printf 'ci-fleet-test-%s' "$1"; } state_path() { printf '%s/%s.state' "$state_dir" "$1"; } deployed_compose_path() { printf '%s/%s.compose.json' "$state_dir" "$1"; } @@ -113,7 +118,7 @@ validate_compose() { import re,sys text=open(sys.argv[1],encoding='utf-8').read() key=r'(?:(?:!!str|!<[^>\n]+>)[ \t]+)?(?:include|label_file|"(?:include|label_file)"|\x27(?:include|label_file)\x27)[ \t]*:' -explicit_key=r'(?m)^[ \t]*\?' +explicit_key=r'(?m)(?:^[ \t]*|[,{][ \t]*)\?' escaped_key=r'"[^"\n]*\\[^"\n]*"[ \t]*:' alias_key=r'(?m)(?:^[ \t]*|[,{][ \t]*)\*[A-Za-z0-9_-]+[ \t]*:' if re.search(r'(?m)^[ \t]*'+key,text) or re.search(r'[,{][ \t]*'+key,text) or re.search(escaped_key,text) or re.search(explicit_key,text) or re.search(alias_key,text): raise SystemExit('Compose include, label_file, or indirect mapping key is forbidden') @@ -301,8 +306,7 @@ case $action in docker compose version >/dev/null getent ahosts "$probe_host" >/dev/null || die 'test-host DNS probe failed' curl --fail --silent --show-error --head --max-time 10 --output /dev/null "$probe_url" || die 'test-host HTTPS/proxy probe failed' - used=$(df -P "$(root_path /var/lib/docker)" | awk 'NR==2{gsub(/%/,"",$5);print $5}') - [[ $used =~ ^[0-9]+$ && $used -lt $disk_warn ]] || die 'Docker storage exceeds configured warning threshold' + check_disk_threshold report "CHECK_OK max_environments=$max_environments disk_used_percent=$used" ;; --converge) converge ;; @@ -327,6 +331,7 @@ case $action in failed=0 for target in "$state_dir"/*.state; do [[ -e $target ]] || continue; inspect_environment "$target" | grep -q 'STATUS=running' || failed=1; done ((failed == 0)) || die 'one or more test environments are unhealthy' + check_disk_threshold report 'HEALTH_OK' ;; esac From 54287831b07681e64b446b3961a535787b4aaff1 Mon Sep 17 00:00:00 2001 From: Nickfost <1572453+Nickfost@users.noreply.github.com> Date: Sun, 30 Aug 2026 04:52:26 -0500 Subject: [PATCH 38/42] fix: close tester maintenance policy gaps --- host/systemd/ci-fleet-tester-cleanup.service | 1 - host/systemd/ci-fleet-tester-health.service | 1 - scripts/fixtures/fake-tester-docker.sh | 26 ++++++++-- scripts/install-tester.sh | 14 ++++-- scripts/test-install-tester.sh | 19 +++++++- scripts/tester-runtime.sh | 51 +++++++++++++++++++- 6 files changed, 96 insertions(+), 16 deletions(-) diff --git a/host/systemd/ci-fleet-tester-cleanup.service b/host/systemd/ci-fleet-tester-cleanup.service index 9f970274..4feaa34a 100644 --- a/host/systemd/ci-fleet-tester-cleanup.service +++ b/host/systemd/ci-fleet-tester-cleanup.service @@ -2,7 +2,6 @@ Description=Remove expired ci-fleet test environments After=docker.service Requires=docker.service -ConditionPathExists=/etc/ci-fleet-tester/tester.env [Service] Type=oneshot diff --git a/host/systemd/ci-fleet-tester-health.service b/host/systemd/ci-fleet-tester-health.service index 7440d67c..7920c16f 100644 --- a/host/systemd/ci-fleet-tester-health.service +++ b/host/systemd/ci-fleet-tester-health.service @@ -2,7 +2,6 @@ Description=Validate ci-fleet test environments After=docker.service Requires=docker.service -ConditionPathExists=/etc/ci-fleet-tester/tester.env [Service] Type=oneshot diff --git a/scripts/fixtures/fake-tester-docker.sh b/scripts/fixtures/fake-tester-docker.sh index 8f0d599a..8a7488e6 100755 --- a/scripts/fixtures/fake-tester-docker.sh +++ b/scripts/fixtures/fake-tester-docker.sh @@ -3,7 +3,11 @@ set -Eeuo pipefail printf '%s\n' "$*" >>"${FAKE_TESTER_DOCKER_LOG:?}" [[ -z ${FAKE_TESTER_EVENT_LOG:-} ]] || printf 'docker %s\n' "$*" >>"$FAKE_TESTER_EVENT_LOG" if [[ $1 == context && $2 == show ]]; then printf 'default\n'; exit 0; fi -if [[ $1 == info ]]; then printf '%s\n' "${FAKE_TESTER_DOCKER_ROOT:?}"; exit 0; fi +if [[ $1 == info ]]; then + if [[ " $* " == *'DefaultAddressPools'* ]]; then printf '[{"Base":"198.51.100.0/24","Size":28}]\n'; + else printf '%s\n' "${FAKE_TESTER_DOCKER_ROOT:?}"; fi + exit 0 +fi if [[ $1 == ps ]]; then [[ ${FAKE_TESTER_PS_FAIL:-0} != 1 ]] || exit 9 if [[ " $* " == *' label=com.docker.compose.project '* ]]; then @@ -29,17 +33,29 @@ if [[ $1 == inspect ]]; then fi if [[ $1 == volume && $2 == ls ]]; then [[ ${FAKE_TESTER_VOLUME_LS_FAIL:-0} != 1 ]] || exit 9 - if [[ " $* " == *' label=com.docker.compose.project=ci-fleet-test-'* ]]; then printf 'fixture-volume\n'; + if [[ " $* " == *' name=^'* && -n ${FAKE_TESTER_PREEXISTING_VOLUME:-} ]]; then printf '%s\n' "$FAKE_TESTER_PREEXISTING_VOLUME"; + elif [[ " $* " == *' label=com.docker.compose.project=ci-fleet-test-'* ]]; then printf 'fixture-volume\n'; elif [[ -n ${FAKE_TESTER_ORPHAN_VOLUME_PROJECT:-} ]]; then printf 'fixture-orphan-volume-id\n'; fi exit 0 fi if [[ $1 == volume && $2 == inspect ]]; then - if [[ " $* " == *'com.docker.compose.project'* ]]; then printf '%s\n' "${FAKE_TESTER_ORPHAN_VOLUME_PROJECT:-}"; + if [[ -n ${FAKE_TESTER_PREEXISTING_VOLUME:-} && ${*: -1} == "$FAKE_TESTER_PREEXISTING_VOLUME" && " $* " != *' --format '* ]]; then + printf '[{"Name":"%s","Driver":"local","Options":{"type":"none","device":"/","o":"bind"},"Labels":{"com.docker.compose.project":"ci-fleet-test-preview-a","com.docker.compose.volume":"data"}}]\n' "$FAKE_TESTER_PREEXISTING_VOLUME" + elif [[ " $* " == *'com.docker.compose.project'* ]]; then printf '%s\n' "${FAKE_TESTER_ORPHAN_VOLUME_PROJECT:-}"; else printf '%s\n' "${FAKE_TESTER_VOLUME_ROOT:?}"; fi exit 0 fi -if [[ $1 == network && $2 == ls ]]; then [[ -z ${FAKE_TESTER_ORPHAN_NETWORK_PROJECT:-} ]] || printf 'fixture-orphan-network-id\n'; exit 0; fi -if [[ $1 == network && $2 == inspect ]]; then printf '%s\n' "${FAKE_TESTER_ORPHAN_NETWORK_PROJECT:-}"; exit 0; fi +if [[ $1 == network && $2 == ls ]]; then + if [[ " $* " == *' name=^'* && -n ${FAKE_TESTER_PREEXISTING_NETWORK:-} ]]; then printf '%s\n' "$FAKE_TESTER_PREEXISTING_NETWORK"; + elif [[ -n ${FAKE_TESTER_ORPHAN_NETWORK_PROJECT:-} ]]; then printf 'fixture-orphan-network-id\n'; fi + exit 0 +fi +if [[ $1 == network && $2 == inspect ]]; then + if [[ -n ${FAKE_TESTER_PREEXISTING_NETWORK:-} && ${*: -1} == "$FAKE_TESTER_PREEXISTING_NETWORK" && " $* " != *' --format '* ]]; then + printf '[{"Name":"%s","Driver":"bridge","Options":{},"Labels":{"com.docker.compose.project":"ci-fleet-test-preview-a","com.docker.compose.network":"default"},"Internal":false,"Attachable":false,"Ingress":false,"EnableIPv6":false,"IPAM":{"Driver":"default","Options":{},"Config":[{"Subnet":"203.0.113.0/24","Gateway":"203.0.113.1"}]}}]\n' "$FAKE_TESTER_PREEXISTING_NETWORK" + else printf '%s\n' "${FAKE_TESTER_ORPHAN_NETWORK_PROJECT:-}"; fi + exit 0 +fi if [[ $1 == compose && $2 == version ]]; then printf 'Docker Compose version v2.fixture\n'; exit 0; fi if [[ $1 == compose && $2 == up && ${3:-} == --help ]]; then [[ ${FAKE_TESTER_NO_WAIT_TIMEOUT:-0} != 1 ]] && printf '%s\n' ' --wait-timeout int'; exit 0; fi if [[ $1 != compose ]]; then exit 2; fi diff --git a/scripts/install-tester.sh b/scripts/install-tester.sh index 7ee91fd1..d1525736 100755 --- a/scripts/install-tester.sh +++ b/scripts/install-tester.sh @@ -65,8 +65,10 @@ units=("${services[@]}" "${timers[@]}") tmpfiles_conf=ci-fleet-tester-lock.conf tmpfiles_dir=$(root_path /usr/lib/tmpfiles.d) -secure_file() { [[ -f $1 && ! -L $1 && $(stat -c %u "$1") == "$expected_uid" && $(stat -c %a "$1") == "$2" ]] || die "protected file is unsafe: $1"; } -secure_dir() { [[ -d $1 && ! -L $1 && $(stat -c %u "$1") == "$expected_uid" && $(stat -c %a "$1") == "$2" ]] || die "protected directory is unsafe: $1"; } +protected_file() { [[ -f $1 && ! -L $1 && $(stat -c %u "$1") == "$expected_uid" && $(stat -c %a "$1") == "$2" ]]; } +protected_dir() { [[ -d $1 && ! -L $1 && $(stat -c %u "$1") == "$expected_uid" && $(stat -c %a "$1") == "$2" ]]; } +secure_file() { protected_file "$1" "$2" || die "protected file is unsafe: $1"; } +secure_dir() { protected_dir "$1" "$2" || die "protected directory is unsafe: $1"; } remove_release_tree() { [[ ! -e $1 ]] || { [[ ${CI_FLEET_TESTING:-0} != 1 ]] || chmod -R u+w "$1"; rm -rf -- "$1"; }; } reject_git_replacements() { @@ -136,10 +138,12 @@ ensure_directories() { } release_complete() { - local path=$1 expected=$2 unit - [[ -d $path && ! -L $path && $(stat -c %u "$path") == "$expected_uid" && $(stat -c %a "$path") == 555 && -x $path/scripts/tester-runtime.sh && -x $path/scripts/tester-launcher.sh && -f $path/.ci-fleet-source-revision ]] || return 1 + local path=$1 expected=$2 directory file unit + for directory in "$path" "$path/scripts" "$path/host" "$path/host/systemd"; do protected_dir "$directory" 555 || return 1; done + for file in "$path/scripts/tester-runtime.sh" "$path/scripts/tester-launcher.sh"; do protected_file "$file" 555 || return 1; done + for file in "$path/.ci-fleet-source-revision" "$path/.ci-fleet-release.sha256"; do protected_file "$file" 444 || return 1; done [[ $(<"$path/.ci-fleet-source-revision") == "$expected" ]] || return 1 - for unit in "${units[@]}"; do [[ -f $path/host/systemd/$unit ]] || return 1; done + for unit in "${units[@]}" "$tmpfiles_conf"; do protected_file "$path/host/systemd/$unit" 444 || return 1; done (cd "$path" && sha256sum --status -c .ci-fleet-release.sha256) || return 1 } diff --git a/scripts/test-install-tester.sh b/scripts/test-install-tester.sh index 5e506c44..af7978d7 100755 --- a/scripts/test-install-tester.sh +++ b/scripts/test-install-tester.sh @@ -35,14 +35,19 @@ if [[ -n ${FAKE_TESTER_SYSTEMCTL_WAIT_READY:-} && $1 == start && " $* " != *' -- fi [[ -z ${FAKE_TESTER_SYSTEMCTL_FAIL:-} || " $* " != *" $FAKE_TESTER_SYSTEMCTL_FAIL "* ]] EOF -printf '#!/usr/bin/env bash\nexit 0\n' >"$fake_bin/curl" -printf '#!/usr/bin/env bash\nexit 0\n' >"$fake_bin/getent" +# shellcheck disable=SC2016 # Write the expansion for the fake to evaluate. +printf '%s\n' '#!/usr/bin/env bash' '[[ ${FAKE_TESTER_CURL_FAIL:-0} != 1 ]]' >"$fake_bin/curl" +# shellcheck disable=SC2016 # Write the expansion for the fake to evaluate. +printf '%s\n' '#!/usr/bin/env bash' '[[ ${FAKE_TESTER_GETENT_FAIL:-0} != 1 ]]' >"$fake_bin/getent" # shellcheck disable=SC2016 # Write the expansion for the fake to evaluate. printf '%s\n' '#!/usr/bin/env bash' '[[ ${FAKE_TESTER_TMPFILES_FAIL:-0} != 1 ]]' >"$fake_bin/systemd-tmpfiles" chmod 0755 "$fake_bin/df" "$fake_bin/systemctl" "$fake_bin/curl" "$fake_bin/getent" "$fake_bin/systemd-tmpfiles" export PATH="$fake_bin:$PATH" CI_FLEET_TESTING=1 CI_FLEET_ROOT_PREFIX=$root export FAKE_TESTER_DOCKER_ROOT=$root/var/lib/docker FAKE_TESTER_VOLUME_ROOT=$root/var/lib/fake-tester-volume FAKE_TESTER_DOCKER_LOG=$tmp/docker.log FAKE_TESTER_SYSTEMCTL_LOG=$tmp/systemctl.log export FAKE_TESTER_EVENT_LOG=$tmp/events.log +for service in ci-fleet-tester-health.service ci-fleet-tester-cleanup.service; do + if grep -q '^ConditionPathExists=/etc/ci-fleet-tester/tester.env$' "$repo_root/host/systemd/$service"; then fail "$service silently skips missing configuration"; fi +done standalone_installer=$tmp/install-tester.sh cp "$installer" "$standalone_installer" @@ -78,6 +83,8 @@ write_environment preview-a 18080 FAKE_TESTER_ROUTE_PORT=18080 "$runtime" --converge --environment preview-a | grep -Fq CONVERGED || fail 'converge failed' state=$root/var/lib/ci-fleet-tester/environments/preview-a.state [[ -f $state && $(stat -c %a "$state") == 600 ]] || fail 'state was not protected' +if FAKE_TESTER_ROUTE_PORT=18080 FAKE_TESTER_PREEXISTING_VOLUME=ci-fleet-test-preview-a_data "$runtime" --converge --environment preview-a >/dev/null 2>&1; then fail 'converge accepted a pre-existing bind-backed volume'; fi +if FAKE_TESTER_ROUTE_PORT=18080 FAKE_TESTER_PREEXISTING_NETWORK=ci-fleet-test-preview-a_default "$runtime" --converge --environment preview-a >/dev/null 2>&1; then fail 'converge accepted a pre-existing network outside approved IPAM'; fi inspect_output=$(FAKE_TESTER_ROUTE_PORT=18080 "$runtime" --inspect --environment preview-a) grep -q 'IMAGE_DIGESTS=sha256:[a-f0-9]\{64\}.*STATUS=running DISK_BYTES=[1-9][0-9]*' <<<"$inspect_output" || fail 'inspect did not report health and disk use' if FAKE_TESTER_PS_FAIL=1 "$runtime" --inspect --environment preview-a >/dev/null 2>&1; then fail 'container inventory failure was hidden'; fi @@ -134,6 +141,8 @@ if FAKE_TESTER_ROUTE_PORT=18080 FAKE_TESTER_CONTAINER_STATE='running none' "$run if FAKE_TESTER_ROUTE_PORT=18080 FAKE_TESTER_CONTAINER_STATE='running none' "$runtime" --health >/dev/null 2>&1; then fail 'health accepted a route without health evidence'; fi if FAKE_TESTER_ROUTE_PORT=18080 FAKE_TESTER_CONTAINER_IMAGE="registry.example/example/app@sha256:$(printf 'b%.0s' {1..64})" "$runtime" --health >/dev/null 2>&1; then fail 'health accepted a container using an unapproved image digest'; fi if FAKE_TESTER_ROUTE_PORT=18080 FAKE_TESTER_DISK_USED_PERCENT=80 "$runtime" --health >/dev/null 2>&1; then fail 'scheduled health accepted Docker storage at the configured warning threshold'; fi +if FAKE_TESTER_ROUTE_PORT=18080 FAKE_TESTER_GETENT_FAIL=1 "$runtime" --health >/dev/null 2>&1; then fail 'scheduled health ignored a failed DNS probe'; fi +if FAKE_TESTER_ROUTE_PORT=18080 FAKE_TESTER_CURL_FAIL=1 "$runtime" --health >/dev/null 2>&1; then fail 'scheduled health ignored a failed HTTPS probe'; fi write_environment service-map 18090 FAKE_TESTER_ROUTE_PORT=18090 FAKE_TESTER_POLICY=two-services "$runtime" --converge --environment service-map >/dev/null if FAKE_TESTER_ROUTE_PORT=18090 FAKE_TESTER_POLICY=two-services FAKE_TESTER_DUPLICATE_SERVICE=1 "$runtime" --inspect --environment service-map | grep -q 'STATUS=running'; then fail 'health accepted duplicate and missing service identities'; fi @@ -220,6 +229,12 @@ for service in ci-fleet-tester-health.service ci-fleet-tester-cleanup.service; d grep -Fq 'TimeoutStartSec=300' "$root/etc/systemd/system/$service" || fail "$service does not bound oneshot start time" done release=$root/opt/ci-fleet-tester/releases/$ref +chmod 0755 "$release/scripts/tester-runtime.sh" +if "$installer" --check --config /etc/ci-fleet-tester/tester.env >/dev/null 2>&1; then fail 'writable release executable passed check'; fi +chmod 0555 "$release/scripts/tester-runtime.sh" +chmod 0644 "$release/.ci-fleet-release.sha256" +if "$installer" --check --config /etc/ci-fleet-tester/tester.env >/dev/null 2>&1; then fail 'writable release manifest passed check'; fi +chmod 0444 "$release/.ci-fleet-release.sha256" # Reboot-creatable lock path: the tmpfiles.d drop-in ships in the release and # recreates the volatile lock directory, so maintenance units survive a reboot. [[ -f $release/host/systemd/ci-fleet-tester-lock.conf ]] || fail 'tmpfiles.d lock drop-in is missing from the release' diff --git a/scripts/tester-runtime.sh b/scripts/tester-runtime.sh index a98eeed9..94bdd726 100755 --- a/scripts/tester-runtime.sh +++ b/scripts/tester-runtime.sh @@ -83,6 +83,11 @@ check_disk_threshold() { [[ $used =~ ^[0-9]+$ && $used -lt $disk_warn ]] || die 'Docker storage exceeds configured warning threshold' } +check_network_probes() { + getent ahosts "$probe_host" >/dev/null || die 'test-host DNS probe failed' + curl --fail --silent --show-error --head --max-time 10 --output /dev/null "$probe_url" || die 'test-host HTTPS/proxy probe failed' +} + project_name() { printf 'ci-fleet-test-%s' "$1"; } state_path() { printf '%s/%s.state' "$state_dir" "$1"; } deployed_compose_path() { printf '%s/%s.compose.json' "$state_dir" "$1"; } @@ -182,6 +187,47 @@ PY ) } +validate_existing_resources() { + python3 - "$1" "$compose_project" "$(state_path "$environment")" <<'PY' +import ipaddress,json,os,subprocess,sys +model=json.load(open(sys.argv[1])); project=sys.argv[2]; tracked=os.path.isfile(sys.argv[3]) +def run(*args): + result=subprocess.run(args,text=True,capture_output=True) + if result.returncode: raise SystemExit(f'Docker resource inventory failed: {" ".join(args[:3])}') + return result.stdout +def inspect(kind,name): + value=json.loads(run('docker',kind,'inspect',name)) + if len(value) != 1 or value[0].get('Name') != name: raise SystemExit(f'{kind} identity is invalid: {name}') + return value[0] +pools=json.loads(run('docker','info','--format','{{json .DefaultAddressPools}}')) +for kind in ('volume','network'): + for logical,item in model.get(kind+'s',{}).items(): + name=item.get('name',f'{project}_{logical}') + existing=[value for value in run('docker',kind,'ls','-q','--filter',f'name=^{name}$').splitlines() if value == name] + if not existing: continue + if not tracked: raise SystemExit(f'untracked pre-existing Docker {kind} is forbidden: {name}') + actual=inspect(kind,name); labels=actual.get('Labels') or {} + if labels.get('com.docker.compose.project') != project or labels.get(f'com.docker.compose.{kind}') != logical: raise SystemExit(f'{kind} provenance is invalid: {name}') + if kind == 'volume': + if actual.get('Driver') != 'local' or actual.get('Options') not in (None,{}): raise SystemExit(f'volume configuration is invalid: {name}') + continue + if actual.get('Driver') != 'bridge' or actual.get('Options') not in (None,{}) or actual.get('Ingress'): raise SystemExit(f'network configuration is invalid: {name}') + if bool(actual.get('Internal')) != bool(item.get('internal')) or bool(actual.get('Attachable')) != bool(item.get('attachable')) or bool(actual.get('EnableIPv6')) != bool(item.get('enable_ipv6')): raise SystemExit(f'network configuration differs from the approved model: {name}') + ipam=actual.get('IPAM') or {} + if ipam.get('Driver') not in (None,'default') or ipam.get('Options') not in (None,{}): raise SystemExit(f'network IPAM is invalid: {name}') + ipv4=[] + for config in ipam.get('Config') or []: + if not isinstance(config,dict) or config.get('AuxAddress'): raise SystemExit(f'network IPAM is invalid: {name}') + try: subnet=ipaddress.ip_network(config.get('Subnet','')) + except ValueError: raise SystemExit(f'network IPAM is invalid: {name}') + if subnet.version == 4: ipv4.append((subnet,config.get('Gateway'))) + if len(ipv4) != 1: raise SystemExit(f'network IPv4 allocation is invalid: {name}') + subnet,gateway=ipv4[0] + approved=any(subnet.prefixlen == pool.get('Size') and subnet.subnet_of(ipaddress.ip_network(pool.get('Base',''))) for pool in pools) + if not approved or gateway is not None and ipaddress.ip_address(gateway) not in subnet: raise SystemExit(f'network IPAM is outside approved Docker address pools: {name}') +PY +} + check_port_unique() { local file key value other_port for file in "$state_dir"/*.state; do @@ -215,6 +261,7 @@ prepare_converge() { } apply_converge() { + validate_existing_resources "$prepared_rendered" || die 'pre-existing Docker resource validation failed' [[ -f $(state_path "$environment") ]] || install -m 0600 "$prepared_rendered" "$(deployed_compose_path "$environment")" write_state if ! docker compose -p "$compose_project" -f "$(deployed_compose_path "$environment")" up -d --remove-orphans --wait --wait-timeout 60; then @@ -304,8 +351,7 @@ case $action in --check) docker info --format '{{.DockerRootDir}}' >/dev/null docker compose version >/dev/null - getent ahosts "$probe_host" >/dev/null || die 'test-host DNS probe failed' - curl --fail --silent --show-error --head --max-time 10 --output /dev/null "$probe_url" || die 'test-host HTTPS/proxy probe failed' + check_network_probes check_disk_threshold report "CHECK_OK max_environments=$max_environments disk_used_percent=$used" ;; @@ -332,6 +378,7 @@ case $action in for target in "$state_dir"/*.state; do [[ -e $target ]] || continue; inspect_environment "$target" | grep -q 'STATUS=running' || failed=1; done ((failed == 0)) || die 'one or more test environments are unhealthy' check_disk_threshold + check_network_probes report 'HEALTH_OK' ;; esac From a91697b6c312c1796253982fb1b40ee7f13d78a2 Mon Sep 17 00:00:00 2001 From: Nickfost <1572453+Nickfost@users.noreply.github.com> Date: Sun, 30 Aug 2026 05:24:41 -0500 Subject: [PATCH 39/42] fix: harden tester maintenance validation --- scripts/fixtures/fake-tester-docker.sh | 10 +++++ scripts/install-tester.sh | 14 +++--- scripts/test-install-tester.sh | 23 ++++++++++ scripts/tester-runtime.sh | 59 ++++++++++++++++++++++++-- 4 files changed, 98 insertions(+), 8 deletions(-) diff --git a/scripts/fixtures/fake-tester-docker.sh b/scripts/fixtures/fake-tester-docker.sh index 8a7488e6..237e716e 100755 --- a/scripts/fixtures/fake-tester-docker.sh +++ b/scripts/fixtures/fake-tester-docker.sh @@ -26,6 +26,16 @@ if [[ $1 == inspect ]]; then service=web [[ ${*: -1} != fixture-worker-id || ${FAKE_TESTER_DUPLICATE_SERVICE:-0} == 1 ]] || service=worker printf '%s %s\n' "$service" "${FAKE_TESTER_CONTAINER_IMAGE:-registry.example/example/app@sha256:$(printf 'a%.0s' {1..64})}" + elif [[ " $* " != *' --format '* ]]; then + privileged=false; nano_cpus=500000000; memory=134217728; pids_limit=128; mounts='[{"Type":"volume","Destination":"/data","RW":true}]'; ports=$(printf '{"8080/tcp":[{"HostIp":"127.0.0.1","HostPort":"%s"}]}' "${FAKE_TESTER_ROUTE_PORT:-18080}") + case ${FAKE_TESTER_LIVE_POLICY:-valid} in + privileged) privileged=true ;; + bind) mounts='[{"Type":"bind","Source":"/","Destination":"/host"}]' ;; + broad-port) ports=$(printf '{"8080/tcp":[{"HostIp":"0.0.0.0","HostPort":"%s"}]}' "${FAKE_TESTER_ROUTE_PORT:-18080}") ;; + unbounded) nano_cpus=0; memory=0; pids_limit=0 ;; + esac + if [[ ${*: -1} == fixture-worker-id ]]; then mounts='[]'; ports='{}'; fi + printf '[{"HostConfig":{"Privileged":%s,"ReadonlyRootfs":true,"CapDrop":["ALL"],"SecurityOpt":["no-new-privileges:true"],"NanoCpus":%s,"Memory":%s,"PidsLimit":%s,"PortBindings":%s},"Mounts":%s}]\n' "$privileged" "$nano_cpus" "$memory" "$pids_limit" "$ports" "$mounts" else printf '%s\n' "${FAKE_TESTER_CONTAINER_STATE:-running healthy}" fi diff --git a/scripts/install-tester.sh b/scripts/install-tester.sh index d1525736..9499f3bd 100755 --- a/scripts/install-tester.sh +++ b/scripts/install-tester.sh @@ -88,10 +88,16 @@ acquire_lifecycle_lock() { } pin_local_docker() { + local socket_mode docker_context actual_root [[ -z ${DOCKER_HOST:-} && -z ${DOCKER_CONTEXT:-} ]] || die 'Docker environment selectors are forbidden' unset DOCKER_CONTEXT export DOCKER_HOST="unix://$docker_socket" [[ -S $docker_socket || ( ${CI_FLEET_TESTING:-0} == 1 && -e $docker_socket ) ]] || die 'local Docker socket is unavailable' + socket_mode=$(stat -c %a "$docker_socket") + [[ ! -L $docker_socket && $(stat -c %u "$docker_socket") == "$expected_uid" ]] || die 'local root-owned Docker socket is unavailable' + (( (8#$socket_mode & 0002) == 0 )) || die 'local Docker socket is writable outside its administration group' + docker_context=$(docker context show); [[ $docker_context == default ]] || die 'tester requires the local default Docker context' + actual_root=$(docker info --format '{{.DockerRootDir}}'); [[ $actual_root == "$docker_root" ]] || die 'Docker root does not match the local managed root' } reject_managed_compose_resources() { @@ -110,15 +116,13 @@ reject_managed_compose_resources() { } host_preflight() { - local os_release docker_context actual_root used + local os_release used os_release=$(root_path /etc/os-release) [[ -f $os_release ]] || die 'supported Debian os-release is missing' # shellcheck disable=SC1090 . "$os_release" [[ ${ID:-} == debian && ${VERSION_ID:-} =~ ^[0-9]+$ && ${VERSION_ID%%.*} -ge 12 ]] || die 'tester hosts require Debian 12 or newer' pin_local_docker - docker_context=$(docker context show); [[ $docker_context == default ]] || die 'tester requires the local default Docker context' - actual_root=$(docker info --format '{{.DockerRootDir}}'); [[ $actual_root == "$docker_root" ]] || die 'Docker root does not match the local managed root' docker compose version >/dev/null printf 'services: {}\n' | docker compose -f - config --format json >/dev/null || die 'Compose JSON rendering is unavailable' docker compose up --help | grep -q -- '--wait-timeout' || die 'Compose wait-timeout support is unavailable' @@ -201,10 +205,10 @@ install_launcher() { install -m 0555 "$1/scripts/tester-launcher.sh" "$stable_la remove_units() { local unit present_units=() present_timers=() for unit in "${units[@]}"; do - [[ ! -e $systemd_dir/$unit ]] || present_units+=("$unit") + [[ ! -e $systemd_dir/$unit && ! -L $systemd_dir/$unit ]] || present_units+=("$unit") done for unit in "${timers[@]}"; do - [[ ! -e $systemd_dir/$unit ]] || present_timers+=("$unit") + [[ ! -e $systemd_dir/$unit && ! -L $systemd_dir/$unit ]] || present_timers+=("$unit") done if (( ${#present_timers[@]} )); then systemctl disable --now "${present_timers[@]}" >/dev/null 2>&1 || return 1 diff --git a/scripts/test-install-tester.sh b/scripts/test-install-tester.sh index af7978d7..0a9f4bae 100755 --- a/scripts/test-install-tester.sh +++ b/scripts/test-install-tester.sh @@ -56,7 +56,18 @@ mkdir "$uninstall_bin" for command in bash chmod dirname env find flock grep id install mkdir rm stat; do ln -s "$(command -v "$command")" "$uninstall_bin/$command"; done ln -s "$fake_bin/docker" "$uninstall_bin/docker" ln -s "$fake_bin/systemctl" "$uninstall_bin/systemctl" +ln -s /missing "$root/etc/systemd/system/ci-fleet-tester-health.service" +ln -s /missing "$root/etc/systemd/system/ci-fleet-tester-health.timer" PATH=$uninstall_bin "$standalone_installer" --uninstall --config /etc/ci-fleet-tester/tester.env >/dev/null || fail 'standalone uninstall required installation-only tools or a Git checkout' +[[ ! -L $root/etc/systemd/system/ci-fleet-tester-health.service && ! -L $root/etc/systemd/system/ci-fleet-tester-health.timer ]] || fail 'uninstall left dangling tester unit symlinks' +mv "$root/var/run/docker.sock" "$root/var/run/docker.real" +ln -s docker.real "$root/var/run/docker.sock" +if PATH=$uninstall_bin "$standalone_installer" --uninstall --config /etc/ci-fleet-tester/tester.env >/dev/null 2>&1; then fail 'uninstall accepted a symlinked Docker endpoint'; fi +rm "$root/var/run/docker.sock"; mv "$root/var/run/docker.real" "$root/var/run/docker.sock" +if FAKE_TESTER_DOCKER_ROOT=/remote/docker PATH=$uninstall_bin "$standalone_installer" --uninstall --config /etc/ci-fleet-tester/tester.env >/dev/null 2>&1; then fail 'uninstall accepted a Docker endpoint with the wrong managed root'; fi +chmod 0666 "$root/var/run/docker.sock" +if PATH=$uninstall_bin "$standalone_installer" --uninstall --config /etc/ci-fleet-tester/tester.env >/dev/null 2>&1; then fail 'uninstall accepted a world-writable Docker control socket'; fi +chmod 0600 "$root/var/run/docker.sock" write_environment() { local id=$1 port=$2 @@ -69,6 +80,9 @@ write_environment() { } "$runtime" --check | grep -Fq CHECK_OK || fail 'runtime preflight failed' +chmod 0666 "$root/var/run/docker.sock" +if "$runtime" --check >/dev/null 2>&1; then fail 'runtime accepted a world-writable Docker control socket'; fi +chmod 0600 "$root/var/run/docker.sock" [[ $(stat -c %a "$root/run/lock") == 1777 && $(stat -c %a "$root/run/lock/ci-fleet-tester") == 755 ]] || fail 'runtime changed shared lock-directory permissions' rm -rf "$root/run/lock/ci-fleet-tester"; mkdir "$tmp/lock-target"; chmod 700 "$tmp/lock-target"; ln -s "$tmp/lock-target" "$root/run/lock/ci-fleet-tester" if "$runtime" --check >/dev/null 2>&1; then fail 'symlinked lifecycle lock directory was accepted'; fi @@ -127,6 +141,9 @@ if FAKE_TESTER_ROUTE_PORT=18094 "$runtime" --converge --environment bad-aliased- write_environment bad-flow-explicit-alias 18095 printf '{x-key: &external-key include, ? *external-key : [{path: /root/other.yaml, env_file: /root/credential.env}], services: {}}\n' >"$root/etc/ci-fleet-tester/definitions/bad-flow-explicit-alias.yaml" if FAKE_TESTER_ROUTE_PORT=18095 "$runtime" --converge --environment bad-flow-explicit-alias >/dev/null 2>&1; then fail 'flow-form explicit aliased Compose include was accepted before rendering'; fi +write_environment bad-anchored-key 18096 +printf '&external include: [{path: /root/other.yaml, env_file: /root/credential.env}]\nservices: {}\n' >"$root/etc/ci-fleet-tester/definitions/bad-anchored-key.yaml" +if FAKE_TESTER_ROUTE_PORT=18096 "$runtime" --converge --environment bad-anchored-key >/dev/null 2>&1; then fail 'anchored Compose include key was accepted before rendering'; fi write_environment interpolation 18090 TOKEN=must-not-render FAKE_TESTER_ROUTE_PORT=18090 FAKE_TESTER_POLICY=interpolation "$runtime" --converge --environment interpolation >/dev/null if grep -Fq must-not-render "$root/var/lib/ci-fleet-tester/environments/interpolation.compose.json"; then fail 'caller environment was interpolated into the Compose model'; fi @@ -140,6 +157,9 @@ if FAKE_TESTER_ROUTE_PORT=18080 FAKE_TESTER_CONTAINER_STATE='exited unhealthy' " if FAKE_TESTER_ROUTE_PORT=18080 FAKE_TESTER_CONTAINER_STATE='running none' "$runtime" --converge --environment preview-a >/dev/null 2>&1; then fail 'converge accepted a route without health evidence'; fi if FAKE_TESTER_ROUTE_PORT=18080 FAKE_TESTER_CONTAINER_STATE='running none' "$runtime" --health >/dev/null 2>&1; then fail 'health accepted a route without health evidence'; fi if FAKE_TESTER_ROUTE_PORT=18080 FAKE_TESTER_CONTAINER_IMAGE="registry.example/example/app@sha256:$(printf 'b%.0s' {1..64})" "$runtime" --health >/dev/null 2>&1; then fail 'health accepted a container using an unapproved image digest'; fi +for live_policy in privileged bind broad-port unbounded; do + if FAKE_TESTER_ROUTE_PORT=18080 FAKE_TESTER_LIVE_POLICY=$live_policy "$runtime" --health >/dev/null 2>&1; then fail "health accepted unsafe live container settings: $live_policy"; fi +done if FAKE_TESTER_ROUTE_PORT=18080 FAKE_TESTER_DISK_USED_PERCENT=80 "$runtime" --health >/dev/null 2>&1; then fail 'scheduled health accepted Docker storage at the configured warning threshold'; fi if FAKE_TESTER_ROUTE_PORT=18080 FAKE_TESTER_GETENT_FAIL=1 "$runtime" --health >/dev/null 2>&1; then fail 'scheduled health ignored a failed DNS probe'; fi if FAKE_TESTER_ROUTE_PORT=18080 FAKE_TESTER_CURL_FAIL=1 "$runtime" --health >/dev/null 2>&1; then fail 'scheduled health ignored a failed HTTPS probe'; fi @@ -190,6 +210,9 @@ for id in preview-a expired-a expired-b; do done "$runtime" --cleanup >/dev/null [[ ! -e $state && ! -e $root/var/lib/ci-fleet-tester/environments/expired-a.state && ! -e $root/var/lib/ci-fleet-tester/environments/expired-b.state ]] || fail 'expired environment survived cleanup' +if FAKE_TESTER_ORPHAN_PROJECT=ci-fleet-test-orphan "$runtime" --cleanup >/dev/null 2>&1; then fail 'scheduled cleanup ignored an orphaned Compose container project'; fi +if FAKE_TESTER_ORPHAN_VOLUME_PROJECT=ci-fleet-test-orphan "$runtime" --cleanup >/dev/null 2>&1; then fail 'scheduled cleanup ignored an orphaned Compose volume project'; fi +if FAKE_TESTER_ORPHAN_NETWORK_PROJECT=ci-fleet-test-orphan "$runtime" --cleanup >/dev/null 2>&1; then fail 'scheduled cleanup ignored an orphaned Compose network project'; fi [[ ${CI_FLEET_TESTER_RUNTIME_ONLY:-0} != 1 ]] || { printf 'TESTER_RUNTIME_TESTS_OK\n'; exit 0; } diff --git a/scripts/tester-runtime.sh b/scripts/tester-runtime.sh index 94bdd726..309d7549 100755 --- a/scripts/tester-runtime.sh +++ b/scripts/tester-runtime.sh @@ -126,7 +126,8 @@ key=r'(?:(?:!!str|!<[^>\n]+>)[ \t]+)?(?:include|label_file|"(?:include|label_fil explicit_key=r'(?m)(?:^[ \t]*|[,{][ \t]*)\?' escaped_key=r'"[^"\n]*\\[^"\n]*"[ \t]*:' alias_key=r'(?m)(?:^[ \t]*|[,{][ \t]*)\*[A-Za-z0-9_-]+[ \t]*:' -if re.search(r'(?m)^[ \t]*'+key,text) or re.search(r'[,{][ \t]*'+key,text) or re.search(escaped_key,text) or re.search(explicit_key,text) or re.search(alias_key,text): raise SystemExit('Compose include, label_file, or indirect mapping key is forbidden') +anchor_key=r'(?m)(?:^[ \t]*|[,{][ \t]*)&[A-Za-z0-9_-]+[ \t]+' +if re.search(r'(?m)^[ \t]*'+key,text) or re.search(r'[,{][ \t]*'+key,text) or re.search(escaped_key,text) or re.search(explicit_key,text) or re.search(alias_key,text) or re.search(anchor_key,text): raise SystemExit('Compose include, label_file, or indirect mapping key is forbidden') PY then return 1; fi if [[ ${CI_FLEET_TESTING:-0} == 1 ]]; then for variable in ${!FAKE_@}; do clean_environment+=("$variable=${!variable}"); done; fi @@ -289,6 +290,53 @@ remove_environment() { report "REMOVED environment=$id" } +reject_orphaned_projects() { + local kind inventory resource project format + local -a inspect_command + for kind in container volume network; do + case $kind in + container) inventory=$(docker ps -aq --filter label=com.docker.compose.project) || die "Docker Compose $kind inventory could not be inspected"; inspect_command=(docker inspect); format='{{index .Config.Labels "com.docker.compose.project"}}' ;; + volume) inventory=$(docker volume ls -q --filter label=com.docker.compose.project) || die "Docker Compose $kind inventory could not be inspected"; inspect_command=(docker volume inspect); format='{{index .Labels "com.docker.compose.project"}}' ;; + network) inventory=$(docker network ls -q --filter label=com.docker.compose.project) || die "Docker Compose $kind inventory could not be inspected"; inspect_command=(docker network inspect); format='{{index .Labels "com.docker.compose.project"}}' ;; + esac + while IFS= read -r resource; do + [[ -n $resource ]] || continue + project=$("${inspect_command[@]}" --format "$format" "$resource") || die "Docker Compose $kind identity could not be inspected" + [[ $project != ci-fleet-test-* ]] && continue + [[ $project =~ ^ci-fleet-test-([a-z0-9][a-z0-9-]{0,62})$ && -f $(state_path "${BASH_REMATCH[1]}") ]] || die "orphaned tester Compose project exists: $project" + done <<<"$inventory" + done +} + +live_container_matches() { + local compose=$1 service=$2 resource=$3 inspection rc=0 + inspection=$(mktemp) + if ! docker inspect "$resource" >"$inspection"; then rm -f "$inspection"; return 1; fi + python3 - "$compose" "$service" "$inspection" <<'PY' || rc=$? +import json,sys +model=json.load(open(sys.argv[1])); service=model['services'][sys.argv[2]] +value=json.load(open(sys.argv[3])); actual=value[0] if len(value)==1 else {}; host=actual.get('HostConfig') or {} +security=lambda values: sorted(str(value).replace('=true',':true') for value in values or []) +expected_ports={} +for port in service.get('ports',[]): + key=f"{int(port['target'])}/{port.get('protocol','tcp')}" + expected_ports.setdefault(key,[]).append({'HostIp':str(port.get('host_ip','')),'HostPort':str(port['published'])}) +expected_mounts=[(mount['type'],'',mount['target'],not mount.get('read_only',False)) for mount in service.get('volumes',[])] +for secret in service.get('secrets',[]): + item={'source':secret,'target':secret} if isinstance(secret,str) else secret + expected_mounts.append(('bind',model['secrets'][item['source']]['file'],f"/run/secrets/{item.get('target',item['source'])}",False)) +expected_mounts=sorted(expected_mounts) +actual_mounts=sorted((mount.get('Type'),mount.get('Source','') if mount.get('Type')=='bind' else '',mount.get('Destination'),bool(mount.get('RW'))) for mount in actual.get('Mounts') or []) +valid=(host.get('Privileged') is False and host.get('ReadonlyRootfs') is True and set(host.get('CapDrop') or [])=={'ALL'} and + not host.get('CapAdd') and security(host.get('SecurityOpt'))==security(service['security_opt']) and + host.get('NanoCpus')==int(service['cpus']*1_000_000_000) and host.get('Memory')==service['mem_limit'] and + host.get('PidsLimit')==service['pids_limit'] and (host.get('PortBindings') or {})==expected_ports and actual_mounts==expected_mounts) +raise SystemExit(0 if valid else 1) +PY + rm -f "$inspection" + return "$rc" +} + inspect_environment() { local target=$1 id compose route_service status=running resource value bytes=0 mount expected running_state service configured_image local -a containers=() @@ -308,6 +356,7 @@ inspect_environment() { read -r service configured_image < <(docker inspect --format '{{index .Config.Labels "com.docker.compose.service"}} {{.Config.Image}}' "$resource") [[ -n $service ]] || { status=unhealthy; continue; } [[ -n ${expected_images[$service]:-} && $configured_image == "${expected_images[$service]}" ]] || status=unhealthy + live_container_matches "$compose" "$service" "$resource" || status=unhealthy observed_services[$service]=$((${observed_services[$service]:-0} + 1)) if [[ $service == "$route_service" ]]; then [[ $running_state == 'running healthy' ]] || status=unhealthy @@ -332,10 +381,13 @@ inspect_environment() { load_global docker_socket=$(root_path /var/run/docker.sock) if [[ ${CI_FLEET_TESTING:-0} == 1 ]]; then - [[ -f $docker_socket && ! -L $docker_socket ]] || die 'local Docker socket is unavailable' + [[ -f $docker_socket ]] || die 'local Docker socket is unavailable' else - [[ -S $docker_socket && ! -L $docker_socket && $(stat -c %u "$docker_socket") == 0 ]] || die 'local root-owned Docker socket is unavailable' + [[ -S $docker_socket ]] || die 'local Docker socket is unavailable' fi +socket_mode=$(stat -c %a "$docker_socket") +[[ ! -L $docker_socket && $(stat -c %u "$docker_socket") == "$expected_uid" ]] || die 'local root-owned Docker socket is unavailable' +(( (8#$socket_mode & 0002) == 0 )) || die 'local Docker socket is writable outside its administration group' unset DOCKER_CONTEXT export DOCKER_HOST="unix://$docker_socket" [[ $(docker info --format '{{.DockerRootDir}}') == "$(root_path /var/lib/docker)" ]] || die 'Docker daemon root is not the expected local path' @@ -370,6 +422,7 @@ case $action in ((expires > now)) || remove_environment "$(basename "$target" .state)" ); then failed=1; fi done + reject_orphaned_projects ((failed == 0)) || die 'one or more expired environments could not be removed' report 'CLEANUP_OK' ;; From 589fc4e2847dbdd2d223caca592076cb28d7bb23 Mon Sep 17 00:00:00 2001 From: Nickfost <1572453+Nickfost@users.noreply.github.com> Date: Sun, 30 Aug 2026 05:29:24 -0500 Subject: [PATCH 40/42] test: preserve rollback route fixture --- scripts/test-install-tester.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/test-install-tester.sh b/scripts/test-install-tester.sh index 0a9f4bae..0513b189 100755 --- a/scripts/test-install-tester.sh +++ b/scripts/test-install-tester.sh @@ -380,7 +380,7 @@ pre_rollback_release=$(readlink -f "$root/opt/ci-fleet-tester/current") chmod u+w "$pre_rollback_release/scripts/tester-runtime.sh" printf '# corrupt before rollback\n' >>"$pre_rollback_release/scripts/tester-runtime.sh" chmod 0555 "$pre_rollback_release/scripts/tester-runtime.sh" -"$installer" --rollback --config /etc/ci-fleet-tester/tester.env | grep -Fq ROLLBACK_OK || fail 'rollback failed' +FAKE_TESTER_ROUTE_PORT=18086 "$installer" --rollback --config /etc/ci-fleet-tester/tester.env | grep -Fq ROLLBACK_OK || fail 'rollback failed' [[ $(readlink -f "$root/opt/ci-fleet-tester/current") == "$root/opt/ci-fleet-tester/releases/$old" ]] || fail 'rollback selected the wrong release' [[ $(<"$root/var/lib/ci-fleet-tester/last-known-good") == "$old" ]] || fail 'rollback recorded a corrupt prior release' [[ $(sha256sum "$root/var/lib/ci-fleet-tester/environments/rollback-env.state") == "$rollback_state_hash" ]] || fail 'rollback changed active environment state' From 57db054a6d5f3f096f7683d2430a7122697a842c Mon Sep 17 00:00:00 2001 From: Nickfost <1572453+Nickfost@users.noreply.github.com> Date: Sun, 30 Aug 2026 05:49:50 -0500 Subject: [PATCH 41/42] fix: permit tester health probe sockets --- host/systemd/ci-fleet-tester-health.service | 2 +- scripts/test-install-tester.sh | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/host/systemd/ci-fleet-tester-health.service b/host/systemd/ci-fleet-tester-health.service index 7920c16f..500069aa 100644 --- a/host/systemd/ci-fleet-tester-health.service +++ b/host/systemd/ci-fleet-tester-health.service @@ -15,6 +15,6 @@ ProtectHome=yes ProtectSystem=strict ReadWritePaths=/var/lib/ci-fleet-tester -/run/lock/ci-fleet-tester TimeoutStartSec=300 -RestrictAddressFamilies=AF_UNIX +RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 LockPersonality=yes MemoryDenyWriteExecute=yes diff --git a/scripts/test-install-tester.sh b/scripts/test-install-tester.sh index 0513b189..adf3d8d6 100755 --- a/scripts/test-install-tester.sh +++ b/scripts/test-install-tester.sh @@ -48,6 +48,8 @@ export FAKE_TESTER_EVENT_LOG=$tmp/events.log for service in ci-fleet-tester-health.service ci-fleet-tester-cleanup.service; do if grep -q '^ConditionPathExists=/etc/ci-fleet-tester/tester.env$' "$repo_root/host/systemd/$service"; then fail "$service silently skips missing configuration"; fi done +grep -Fxq 'RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6' "$repo_root/host/systemd/ci-fleet-tester-health.service" || fail 'health service blocks required network probe sockets' +grep -Fxq 'RestrictAddressFamilies=AF_UNIX' "$repo_root/host/systemd/ci-fleet-tester-cleanup.service" || fail 'cleanup service permits network sockets' standalone_installer=$tmp/install-tester.sh cp "$installer" "$standalone_installer" From de021238546681391d8af8ba58c348bcf5d5cb54 Mon Sep 17 00:00:00 2001 From: Nickfost <1572453+Nickfost@users.noreply.github.com> Date: Mon, 31 Aug 2026 01:55:39 -0500 Subject: [PATCH 42/42] fix: scope tester runtime prerequisites to each action --- scripts/test-install-tester.sh | 4 ++++ scripts/tester-runtime.sh | 4 +++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/scripts/test-install-tester.sh b/scripts/test-install-tester.sh index adf3d8d6..7c82f43b 100755 --- a/scripts/test-install-tester.sh +++ b/scripts/test-install-tester.sh @@ -215,6 +215,10 @@ done if FAKE_TESTER_ORPHAN_PROJECT=ci-fleet-test-orphan "$runtime" --cleanup >/dev/null 2>&1; then fail 'scheduled cleanup ignored an orphaned Compose container project'; fi if FAKE_TESTER_ORPHAN_VOLUME_PROJECT=ci-fleet-test-orphan "$runtime" --cleanup >/dev/null 2>&1; then fail 'scheduled cleanup ignored an orphaned Compose volume project'; fi if FAKE_TESTER_ORPHAN_NETWORK_PROJECT=ci-fleet-test-orphan "$runtime" --cleanup >/dev/null 2>&1; then fail 'scheduled cleanup ignored an orphaned Compose network project'; fi +cleanup_bin=$tmp/cleanup-bin +mkdir "$cleanup_bin" +for command in awk basename bash chmod cmp date df dirname docker du env find flock grep id install mkdir mktemp mv python3 readlink rm stat wc; do ln -s "$(command -v "$command")" "$cleanup_bin/$command"; done +PATH=$cleanup_bin "$runtime" --cleanup >/dev/null || fail 'cleanup required unrelated network-probe tools' [[ ${CI_FLEET_TESTER_RUNTIME_ONLY:-0} != 1 ]] || { printf 'TESTER_RUNTIME_TESTS_OK\n'; exit 0; } diff --git a/scripts/tester-runtime.sh b/scripts/tester-runtime.sh index 309d7549..5d8b8c27 100755 --- a/scripts/tester-runtime.sh +++ b/scripts/tester-runtime.sh @@ -33,7 +33,9 @@ while (($#)); do done [[ -n $action ]] || { usage; exit 2; } case $action in --converge|--reset|--remove|--inspect) [[ $environment =~ ^[a-z0-9][a-z0-9-]{0,62}$ ]] || die 'environment ID is invalid' ;; *) [[ -z $environment ]] || die '--environment is not valid for this action' ;; esac -for command in awk basename chmod cmp curl date df dirname docker du env find flock getent grep install mktemp mv python3 readlink rm stat wc; do command -v "$command" >/dev/null || die "required command is unavailable: $command"; done +required_commands='awk basename chmod cmp date df dirname docker du env find flock grep install mkdir mktemp mv python3 readlink rm stat wc' +case $action in --check|--health) required_commands+=' curl getent' ;; esac +for command in $required_commands; do command -v "$command" >/dev/null || die "required command is unavailable: $command"; done secure_directory() { local path=$1 mode=$2