From 5e8d58166ccee3d846ceceaa4aad30c10865e623 Mon Sep 17 00:00:00 2001 From: Nick Foster Date: Sat, 18 Jul 2026 17:20:38 -0500 Subject: [PATCH 01/57] Add Git-authored controller desired state --- .gitignore | 28 +- AGENTS.md | 2 + README.md | 25 +- docs/ADDING-A-HOST.md | 82 +-- docs/ARCHITECTURE.md | 16 +- docs/DESIRED-STATE.md | 153 +++++ docs/HOST-MAINTENANCE.md | 28 +- docs/PROJECT-STANDARD.md | 9 +- docs/PUBLIC-PRIVATE-CONFIGURATION.md | 3 +- docs/README.md | 8 +- docs/ROADMAP.md | 2 + docs/SECRETS.md | 9 +- host/ci-fleet.env.example | 22 - host/host.env.example | 8 + host/systemd/ci-fleet-cleanup.service | 4 +- host/systemd/ci-fleet-drift.service | 10 + host/systemd/ci-fleet-drift.timer | 11 + host/systemd/ci-fleet-health.service | 4 +- scripts/check-installed-state.sh | 18 + scripts/desired_state.py | 274 +++++++++ scripts/healthcheck.sh | 14 +- scripts/install-worker-controller.sh | 557 ++++++++++++++++++ scripts/preflight.sh | 37 +- scripts/test-install-worker-controller.sh | 176 ++++++ scripts/test_desired_state.py | 142 +++++ scripts/validate.sh | 8 +- templates/config-repository/AGENTS.md | 9 +- templates/config-repository/README.md | 118 +++- .../examples/multi-host/fleet.json | 36 +- templates/config-repository/fleet.json | 22 +- templates/config-repository/fleet.schema.json | 53 +- templates/config-repository/scripts/init.py | 62 +- .../config-repository/scripts/test_policy.py | 85 ++- .../config-repository/scripts/validate.py | 131 +++- 34 files changed, 1958 insertions(+), 208 deletions(-) create mode 100644 docs/DESIRED-STATE.md delete mode 100644 host/ci-fleet.env.example create mode 100644 host/host.env.example create mode 100644 host/systemd/ci-fleet-drift.service create mode 100644 host/systemd/ci-fleet-drift.timer create mode 100755 scripts/check-installed-state.sh create mode 100755 scripts/desired_state.py create mode 100755 scripts/install-worker-controller.sh create mode 100755 scripts/test-install-worker-controller.sh create mode 100755 scripts/test_desired_state.py diff --git a/.gitignore b/.gitignore index 29eb047a..7a60b85e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,26 +1,2 @@ -# Local deployment configuration -.env -.env.* -!.env.example -*.local -secrets/ -private/ -inventory/ - -# Credentials and keys -*.pem -*.key -*.p12 -*.pfx - -# Runtime state -_work/ -runner-data/ -logs/ -tmp/ - -# Editors and operating systems -.vscode/ -.idea/ -.DS_Store -Thumbs.db +__pycache__/ +*.pyc diff --git a/AGENTS.md b/AGENTS.md index 00593ae8..db0bed70 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -41,6 +41,8 @@ Agents modifying this repository or adapting a project MUST apply these document - Do not install project language runtimes in the generic runner image. - Do not put repository names, project allowlists, or project-specific logic in runner images or host configuration. - Keep project workflows independent of host, site, instance, and scale-set names; route through shared capability labels. +- Keep worker capacity in reviewed private infrastructure configuration. Application workflows submit all independent jobs and must not use `max-parallel` to model fleet size. +- Treat schema-v3 controller desired state as authoritative for routing, capacity, lifecycle, and engine revision; host-local files contain only rendered state and credentials. - Do not allow projects to select privileged runner groups through untrusted inputs. ## Verification expectations diff --git a/README.md b/README.md index 18516e30..25d336b2 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ Use one shared pool of disposable CI workers across multiple trusted private repositories, Docker hosts, virtual machines, bare-metal computers, home labs, remote sites, or VPS providers. Projects bring their own Dockerized build and test environment; fleet hosts stay generic. -> **Status:** Experimental. The first controller host is deployed, and the first manually dispatched runner pilot is pending. The project is not production-ready. +> **Status:** Experimental. Ephemeral runner pilots and multi-job workloads have been proven, and the schema-v3 Git-authored controller installer is available for reviewed adoption testing. The project is not production-ready. ## What problem does this solve? @@ -93,7 +93,7 @@ GitHub runner-group policy decides which repositories may schedule work. A share | --- | --- | --- | | Public fleet repository | Runner image, controller, lifecycle, setup, health checks, scoped cleanup, standards, examples | Real credentials, private host inventory, production configuration | | Project repository | Test Dockerfile, services, fixtures, migrations, test plan, `scripts/ci/run.sh` | Fleet controller credentials or host-specific setup | -| Private installation configuration | Organization settings, repository authorization, host capacity, network policy, monitoring, secrets | Project runtime dependencies or test logic | +| Private installation configuration | Organization settings, repository authorization, logical controller state, capacity budgets, network policy, and required secret names | Secret values, host addresses, project runtime dependencies, or test logic | A public application can use the same fleet indirectly. Its public repository keeps pull-request validation unprivileged, while a separate private delivery repository checks out an approved immutable commit and performs protected CI, release, or deployment work. The public repository itself never receives privileged runner-group access or fleet credentials. See [Public projects, private delivery, and private configuration](docs/PUBLIC-PRIVATE-CONFIGURATION.md). @@ -137,9 +137,9 @@ Read-only validation, repository-writing releases, staging, production deploymen | Public architecture, standards, examples, and migration rules | Available | | Docker runner and controller prototype | Available | | First isolated controller host | Deployed | -| First manually dispatched private-repository pilot job | Pending | -| One-command `sudo ./setup.sh` installer | Accepted design; not implemented | -| MailThisForMe migration | Planned after pilot | +| First manually dispatched private-repository pilot job | Proven | +| Schema-v3 worker-controller installer | Available for experimental install and adoption | +| MailThisForMe migration | Parallel task-matrix validation in progress | | TF2 Recommendation Engine migration | Planned after pilot | | Reusable tester and deployer components | Planned | | Production-ready release | Not yet | @@ -169,6 +169,7 @@ Supported deployment shapes include virtual machines, dedicated physical machine | Browse every guide, concept, standard, and example | [Documentation index](docs/README.md) | | Evaluate the current prototype safely | [Live pilot runbook](docs/LIVE-PILOT.md) | | Add another Docker host or location | [How to add a host](docs/ADDING-A-HOST.md) | +| Install, adopt, check, upgrade, or remove a controller | [Git-authored controller desired state](docs/DESIRED-STATE.md) | | Add a trusted private repository | [How to add a project](docs/ADDING-A-PROJECT.md) | | Convert existing GitHub Actions CI | [Migrating existing CI](docs/MIGRATING-EXISTING-CI.md) | | Build a compatible project contract | [Project CI standard](docs/PROJECT-STANDARD.md) | @@ -236,17 +237,21 @@ Use this fleet only for explicitly trusted repositories and workflows. Keep depl Read [SECURITY.md](SECURITY.md) and the [secrets model](docs/SECRETS.md) before registration or deployment. -## Installation direction +## Install or adopt a controller -The accepted operator experience is: +After creating the GitHub App/runner group and placing the host-local identity file, run one command on the target machine with a reviewed private configuration commit: ```bash -sudo ./setup.sh +sudo ./scripts/install-worker-controller.sh \ + --install \ + --config-repo example-org/example-fleet-config \ + --ref 1111111111111111111111111111111111111111 \ + --controller example-ci-01 ``` -That command will eventually bootstrap or validate GitHub configuration, install the controller and maintenance services, verify health, and support safe reruns, repair, upgrades, and removal directly from the target host. Phone interaction should be limited to unavoidable, non-secret GitHub approval links. +Use `--adopt` for an existing manually installed controller. The same script provides `--check`, `--upgrade`, `--rollback`, and `--uninstall`. It validates schema-v3 desired state, renders only non-secret host configuration, installs the pinned engine revision, drains before replacement, records a rollback checkpoint, and installs health, scoped-cleanup, and drift timers. -**That installer is not implemented yet.** For the experimental version, follow the [live pilot runbook](docs/LIVE-PILOT.md) and [deployment prototype](docs/DEPLOYMENT-PROTOTYPE.md). +The installer remains experimental and intentionally does not create GitHub credentials or broaden runner-group access. Phone-friendly GitHub bootstrap is tracked separately. Read [Git-authored controller desired state](docs/DESIRED-STATE.md) before using it on a live host; retain the [live pilot runbook](docs/LIVE-PILOT.md) for first-job proof. ## Examples diff --git a/docs/ADDING-A-HOST.md b/docs/ADDING-A-HOST.md index 3e42e1f5..e39361b2 100644 --- a/docs/ADDING-A-HOST.md +++ b/docs/ADDING-A-HOST.md @@ -1,6 +1,6 @@ # How to add a fleet host -Use this runbook to add a Proxmox VM, physical computer, remote-site machine, or VPS to an existing ci-fleet installation. +Use this runbook to add a virtual machine, physical computer, remote-site machine, or VPS to an existing ci-fleet installation. ## Outcome @@ -73,62 +73,41 @@ Before placing credentials on the host: A remote site needs outbound connectivity; it does not need project-specific inbound access. If one location is offline, compatible jobs remain queued for another available host. -## 4. Install a pinned ci-fleet checkout +## 4. Declare the host in private Git configuration -Use a reviewed immutable commit: +Create or update one schema-v3 controller entry in the organization's private configuration repository. Give it a unique logical ID and scale-set name, assign its shared runner pool, start with reviewed capacity, and pin a full ci-fleet engine commit. -```bash -sudo install -d -m 0755 /opt/ci-fleet -# Populate /opt/ci-fleet from a reviewed ci-fleet commit. -git -C /opt/ci-fleet rev-parse HEAD -``` +Run that repository's strict validator before merge. Do not put the machine's address, VM ID, storage, backup identifier, or credential in Git. -Do not follow a moving branch for unattended controller updates. Fleet image changes use reviewed rolling replacement and keep the previous version available for rollback. +See [Git-authored controller desired state](DESIRED-STATE.md) for the complete contract. -## 5. Create host-local configuration +## 5. Prepare host-local identity -Start from `deploy/ci-fleet.env.example`: +From a reviewed ci-fleet checkout: ```bash sudo install -d -m 0700 /etc/ci-fleet/secrets -sudo install -m 0600 deploy/ci-fleet.env.example /etc/ci-fleet/ci-fleet.env +sudo install -m 0600 host/host.env.example /etc/ci-fleet/host.env +sudo install -m 0600 /secure/source/github-app.pem /etc/ci-fleet/secrets/github-app.pem ``` -Set: - -- a unique `CI_FLEET_INSTANCE`; -- a unique `CI_FLEET_SCALE_SET_NAME`; -- the fleet's shared `CI_FLEET_LABELS`; -- the shared `CI_FLEET_RUNNER_GROUP`; -- host-specific maximum concurrency and runner CPU/memory limits; -- `CI_FLEET_MIN_RUNNERS=0` so idle runners do not persist. - -Repository names do not belong in this file. +Edit `/etc/ci-fleet/host.env` locally. Prefer a separately generated GitHub App private key per host so one location can be revoked without replacing every host key. Never expose the key to runner or project containers. -Provision the GitHub App private key directly into `/etc/ci-fleet/secrets/github-app.pem` with mode `0600`. Prefer a separately generated App private key per host so one location can be revoked without replacing every host key. Never expose the key to runner or project containers. - -## 6. Validate before starting +## 6. Install or adopt with one command ```bash -cd /opt/ci-fleet -set -a -. /etc/ci-fleet/ci-fleet.env -set +a -scripts/preflight.sh -docker compose -f deploy/compose.yaml build runner-image controller -scripts/preflight.sh +sudo ./scripts/install-worker-controller.sh \ + --install \ + --config-repo example-org/example-fleet-config \ + --ref 1111111111111111111111111111111111111111 \ + --controller example-ci-01 ``` -Continue only when preflight reports `PREFLIGHT_OK warnings=0`. - -## 7. Start at zero and prove one job +Use `--adopt` when converting an existing manual controller. The installer validates the complete configuration, renders host-local runtime state, installs the pinned engine, creates a controller checkpoint, builds images, installs maintenance timers, and verifies health. -Start with maximum concurrency one even if the host will eventually run more: +The configuration repository credential, when required, must be read-only and host-side. Credentials are never accepted in command arguments. -```bash -docker compose -f deploy/compose.yaml up -d --no-deps controller -docker compose -f deploy/compose.yaml logs --tail=100 controller -``` +## 7. Prove one job Confirm no runner exists while idle. Dispatch one manual, read-only job from an authorized private project and verify: @@ -141,15 +120,15 @@ Confirm no runner exists while idle. Dispatch one manual, read-only job from an Follow [Live pilot](LIVE-PILOT.md) for the complete proof and rollback. -## 8. Enable unattended operations +## 8. Verify unattended operations -Install the health and cleanup timers from [Host maintenance](HOST-MAINTENANCE.md). Monitor controller state, disk thresholds, Docker health, last successful job, cleanup failures, and pending reboot state. +The installer enables health, scoped cleanup, and pinned desired-state drift timers. Run each service once and inspect its journal as described in [Host maintenance](HOST-MAINTENANCE.md). Monitor controller state, disk thresholds, Docker health, last successful job, cleanup failures, drift, and pending reboot state. The steady-state host should need no project-specific edits. Adding a project changes its repository, private fleet configuration, and GitHub runner-group policy—not this host. -## 9. Increase capacity safely +## 9. Increase capacity safely through Git -Raise `CI_FLEET_MAX_RUNNERS` only after measuring CPU, memory, disk, network, cache growth, collisions, cancellation, and cleanup. Keep explicit per-runner limits. +Raise the controller's schema-v3 `max_runners` and its pool `capacity_budget` through a reviewed private configuration change only after measuring CPU, memory, disk, network, cache growth, collisions, cancellation, and cleanup. Keep explicit per-runner limits. Keep the one-runner pilot preflight unchanged. For any later increase, follow [Post-pilot capacity promotion](CAPACITY-PROMOTION.md), run `scripts/capacity-preflight.sh` before and after the controller-only recreation, and retain the new maximum only when the separately authorized workload and cleanup proof satisfy the predeclared resource policy. @@ -157,12 +136,13 @@ Use another host or location when it improves failure tolerance. Give every adde ## 10. Drain, replace, or remove a host -1. Stop new capacity by setting maximum capacity to zero or stopping the controller. -2. Confirm no managed runner is active. -3. Run cleanup in dry-run mode, review candidates, then apply scoped cleanup. -4. Verify the host's unique scale set is absent from GitHub. -5. Revoke that host's GitHub App private key. -6. Securely delete host-local credentials. -7. Remove or restore the machine according to the site's recovery policy. +1. Merge a private configuration change setting the controller to `drained`. +2. Apply that exact commit with `install-worker-controller.sh --upgrade`. +3. Confirm zero managed runners and run scoped cleanup. +4. Run `install-worker-controller.sh --uninstall` when the host will not return. +5. Verify the host's unique scale set is absent from GitHub. +6. Revoke that host's GitHub App private key and explicitly remove preserved credentials. +7. Set the controller to `disabled` or remove it from private configuration. +8. Delete or repurpose the machine according to the site's recovery policy. Other hosts and project workflows continue using the shared routing label. Replacing one host must not require editing any project workflow. diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index dae470f3..e0ce9a0e 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -38,7 +38,8 @@ A host may run multiple runners when resource measurements and project isolation flowchart LR A["ci-fleet"] --> B["Runner lifecycle"] C["Project repo"] --> D["Test environment"] - E["Host config"] --> F["Secrets and capacity"] + E["Private Git config"] --> F["Routing and capacity desired state"] + G["Host-local config"] --> H["Credentials and rendered runtime state"] ``` ### Fleet repository @@ -71,19 +72,19 @@ Each project is responsible for: - run-scoped cleanup; - release and deployment behavior. -### Deployment-local configuration +### Private installation configuration Each installation is responsible for: - real organization and runner-group names; -- host capacity; +- logical controller identities and capacity budgets; +- pinned controller engine revisions; - private network rules; -- secret provisioning; -- host inventory; +- required secret names; - monitoring destinations; - maintenance windows. -Deployment-local values must not be committed to this public repository. +These secret-free values belong in a reviewed private schema-v3 configuration repository. Machine addresses, VM IDs, storage and backup identifiers, credentials, secret values, and rendered runtime files remain host-local or in an external secret manager. See [Git-authored controller desired state](DESIRED-STATE.md). ## Target runner lifecycle @@ -127,7 +128,7 @@ The same fleet interface should support: - one Docker host with one or more runners; - several Docker hosts with one or more runners each; -- Proxmox virtual machines; +- virtual machines; - dedicated physical computers; - remote-site computers; - VPS infrastructure. @@ -152,6 +153,7 @@ Host-wide pruning must not run as an uncoordinated per-job operation. Hard cance - Host security updates are applied automatically. - Reboots drain runner capacity before interrupting the host. - Fleet updates use rolling replacement and an explicit rollback version. +- Controller routing, capacity, lifecycle, and engine updates use reviewed full commits from private desired-state configuration; hosts never follow an unreviewed moving branch. ## Project adoption diff --git a/docs/DESIRED-STATE.md b/docs/DESIRED-STATE.md new file mode 100644 index 00000000..f102fe43 --- /dev/null +++ b/docs/DESIRED-STATE.md @@ -0,0 +1,153 @@ +# Git-authored controller desired state + +Schema v3 makes a reviewed private configuration repository the authority for controller routing, capacity, lifecycle, and engine revision. The target host keeps credentials and rendered runtime state outside Git. + +## Responsibility boundary + +| Location | Owns | +| --- | --- | +| Public ci-fleet repository | Schema, validator, renderer, installer, controller, maintenance, and fictional examples | +| Private configuration repository | Real project allowlists, runner pools, logical controller IDs, capacity budgets, controller state, resource limits, and pinned engine revisions | +| Root-owned host files | GitHub App identity, private key path, rendered environment, checkpoints, and installation state | +| Application repository | Independent CI tasks and shards; never fleet size or controller identity | + +Host addresses, VM IDs, storage names, backup identifiers, SSH details, tokens, private keys, and rendered `.env` files are rejected from the Git-authored configuration. + +## Schema v3 + +Each runner pool declares: + +- a logical GitHub runner group; +- stable shared routing labels; +- explicitly allowed repositories; +- `public_repositories: false`; +- an infrastructure `capacity_budget`; +- `job_submission_policy: all-independent-jobs`. + +Each controller has a unique object key and declares: + +- its pool and logical location; +- `active`, `drained`, or `disabled` state; +- a unique scale-set name; +- `experimental`, `stable`, or `retiring` lifecycle; +- a full pinned ci-fleet engine commit; +- minimum and maximum runners; +- CPU cores and memory per ephemeral runner. + +Active and drained controllers reserve their configured maximum against the pool budget. A drained controller has zero effective runtime capacity but keeps its reservation, so an undrain cannot silently overcommit the pool. Disabled controllers reserve no capacity. + +The authoritative fictional contract is in [`templates/config-repository`](../templates/config-repository/README.md). + +## Application parallelism rule + +Application workflows submit every independent task and shard. They must not use GitHub Actions `strategy.max-parallel` to represent how many fleet workers exist. GitHub queues excess jobs; private infrastructure policy decides how many run simultaneously. + +An application may use a concurrency limit only for a documented external-system restriction such as a vendor rate limit or a single-writer test fixture. That exception must not be based on current worker count. + +## Prepare host-local identity + +Create the root-owned file once: + +```bash +sudo install -d -m 0700 /etc/ci-fleet/secrets +sudo install -m 0600 host/host.env.example /etc/ci-fleet/host.env +sudo install -m 0600 /secure/source/github-app.pem /etc/ci-fleet/secrets/github-app.pem +``` + +Edit `/etc/ci-fleet/host.env` locally. It contains only the GitHub App client ID, installation ID, private-key path, and runner TTL. It is never committed. The PEM remains separate and is never printed by the installer. + +GitHub App and runner-group creation remain the bootstrap responsibility tracked by [issue #27](https://github.com/RandomDevelopment/ci-fleet/issues/27). The installer fails closed when those prerequisites are absent. + +## Install a fresh controller + +Run the command from a reviewed checkout of ci-fleet on the target Linux Docker machine: + +```bash +sudo ./scripts/install-worker-controller.sh \ + --install \ + --config-repo example-org/example-fleet-config \ + --ref 1111111111111111111111111111111111111111 \ + --controller example-ci-01 +``` + +`--ref` must be a full configuration commit SHA. The installer never follows a moving branch. For a private remote repository, configure a narrowly scoped read-only Git credential on the host before running the command. Do not embed credentials in the URL or command line. A local pinned Git checkout is also accepted. + +The installer: + +1. fetches only the requested configuration commit; +2. validates the complete schema and capacity relationships; +3. selects exactly one logical controller; +4. renders `/etc/ci-fleet/ci-fleet.env` without secret values; +5. fetches and verifies the pinned public engine commit; +6. builds the pinned runner and controller images; +7. creates a root-only controller checkpoint; +8. installs health, cleanup, and pinned-state drift timers; +9. starts the controller only when its desired state is active; +10. verifies runtime health and records a redacted installation state. + +A successful second `--install` run reports `NO_CHANGE` and performs no unnecessary replacement. + +## Adopt an existing controller + +Use adoption for a manually installed controller: + +```bash +sudo ./scripts/install-worker-controller.sh \ + --adopt \ + --config-repo example-org/example-fleet-config \ + --ref 1111111111111111111111111111111111111111 \ + --controller example-ci-01 +``` + +If `/etc/ci-fleet/host.env` does not exist, adoption extracts only the approved host identity fields from the existing root-owned `/etc/ci-fleet/ci-fleet.env`. It never copies project settings or secret values into Git. The running controller is paused, existing managed runners are allowed to finish, and activation proceeds only after the host is idle. + +## Check, upgrade, roll back, and remove + +```bash +# Read-only comparison with one pinned configuration commit +sudo ./scripts/install-worker-controller.sh --check \ + --config-repo example-org/example-fleet-config \ + --ref 1111111111111111111111111111111111111111 \ + --controller example-ci-01 + +# Apply a newer reviewed configuration and/or pinned engine +sudo ./scripts/install-worker-controller.sh --upgrade \ + --config-repo example-org/example-fleet-config \ + --ref 2222222222222222222222222222222222222222 \ + --controller example-ci-01 + +# Restore the latest root-only controller checkpoint +sudo ./scripts/install-worker-controller.sh --rollback + +# Drain and remove controller services and rendered state +sudo ./scripts/install-worker-controller.sh --uninstall +``` + +Uninstall removes the controller, timers, rendered environment, and active installation marker. It deliberately preserves `/etc/ci-fleet/host.env`, the secrets directory, and checkpoints so credential destruction and audit retention remain explicit operator decisions. + +No mode uses global Docker prune or removes unrelated workloads. + +## Drift and reviewed updates + +`ci-fleet-drift.timer` checks the host every fifteen minutes against the exact configuration SHA recorded at installation. It detects host edits, missing releases, runtime-state mismatch, altered metadata, and missing maintenance timers without applying changes. + +The host does not automatically follow or execute a moving branch. A new configuration becomes effective only when an operator or authorized external controller supplies its reviewed full commit SHA to `--upgrade`. Automatic dispatchers may watch a protected branch and invoke that exact command after merge, using read-only repository contents permission; their identity must remain host-side and unavailable to job runners. + +## Drain and retirement + +1. Merge a private configuration change setting the controller to `drained`. +2. Run `--upgrade` with that reviewed commit. +3. Verify zero managed runners and zero effective capacity. +4. Remove the controller's GitHub registration and revoke its host identity when it will not return. +5. Set it to `disabled` or remove its declaration in a later reviewed change. +6. Delete or repurpose the machine according to the installation's infrastructure policy. + +Legacy project-specific hosts remain until CI, promotion, and deployment no longer reference them. Deleting or replacing a generic controller does not require application workflow changes because projects route through shared labels. + +## Failure and recovery behavior + +Before mutation, the installer records the prior rendered environment, installation metadata, and release target under `/var/lib/ci-fleet/checkpoints`. Build and validation happen before the active release changes. A failed activation or health check restores the latest known-good controller checkpoint. + +These controller checkpoints do not replace machine backups. Operators still create and verify VM snapshots, physical-host recovery media, or equivalent infrastructure backups according to their local policy. + +This contract implements the engine portions of [issue #32](https://github.com/RandomDevelopment/ci-fleet/issues/32) and integrates the installer, documentation, phone-first bootstrap, and capacity work tracked by [#21](https://github.com/RandomDevelopment/ci-fleet/issues/21), [#24](https://github.com/RandomDevelopment/ci-fleet/issues/24), [#27](https://github.com/RandomDevelopment/ci-fleet/issues/27), and [#30](https://github.com/RandomDevelopment/ci-fleet/issues/30). diff --git a/docs/HOST-MAINTENANCE.md b/docs/HOST-MAINTENANCE.md index 8e1faf28..72e0a145 100644 --- a/docs/HOST-MAINTENANCE.md +++ b/docs/HOST-MAINTENANCE.md @@ -9,6 +9,7 @@ Fleet hosts are generic Docker infrastructure. They must receive operating-syste - Schedule reboots only after draining the controller. - Run health checks every five minutes. - Run scoped cleanup daily. +- Compare installed state with its pinned Git configuration every fifteen minutes. - Never schedule `docker system prune`. - Alert before Docker storage reaches 80%; treat 90% as critical. @@ -25,25 +26,22 @@ sudo unattended-upgrade --dry-run --debug Review `/etc/apt/apt.conf.d/50unattended-upgrades` and confirm only the intended Debian security origins are enabled. Keep automatic reboot disabled; a generic CI host may still be executing a job when a package requests reboot. -## Install fleet timers +## Fleet timers -After the repository is installed at `/opt/ci-fleet` and `/etc/ci-fleet/ci-fleet.env` exists: +`scripts/install-worker-controller.sh` installs and enables all three timer pairs: -```bash -sudo install -m 0644 host/systemd/ci-fleet-health.service /etc/systemd/system/ -sudo install -m 0644 host/systemd/ci-fleet-health.timer /etc/systemd/system/ -sudo install -m 0644 host/systemd/ci-fleet-cleanup.service /etc/systemd/system/ -sudo install -m 0644 host/systemd/ci-fleet-cleanup.timer /etc/systemd/system/ -sudo systemctl daemon-reload -sudo systemctl enable --now ci-fleet-health.timer ci-fleet-cleanup.timer -``` +- `ci-fleet-health.timer` checks Docker, disk, and the controller's desired runtime state; +- `ci-fleet-cleanup.timer` removes only expired inactive fleet-owned resources; +- `ci-fleet-drift.timer` compares the installation with the exact pinned configuration commit without applying changes. Run each service manually once before relying on its timer: ```bash sudo systemctl start ci-fleet-health.service +sudo systemctl start ci-fleet-drift.service sudo journalctl -u ci-fleet-health.service --since today -sudo /opt/ci-fleet/scripts/cleanup.sh +sudo journalctl -u ci-fleet-drift.service --since today +sudo /opt/ci-fleet/current/scripts/cleanup.sh sudo systemctl start ci-fleet-cleanup.service ``` @@ -51,13 +49,15 @@ The manual cleanup command is intentionally a dry-run. Enable the applying servi ## Reboot procedure -1. Stop new capacity by stopping the controller. +1. Merge and apply a reviewed desired-state change setting the controller to `drained`, or otherwise pause the controller and wait for zero managed runners. 2. Confirm no managed runner container is active. 3. Apply updates and reboot. 4. Confirm Docker, disk, time synchronization, DNS, and outbound GitHub connectivity. -5. Run `scripts/preflight.sh`. -6. Start the controller and confirm `MIN=0` produces no idle container. +5. Run `scripts/install-worker-controller.sh --check` against the installed configuration commit. +6. Apply a reviewed `active` desired-state commit and confirm `MIN=0` produces no idle container. ## Dependency maintenance Dependabot proposes updates for GitHub Actions, Go modules, and both Dockerfiles. Those pull requests must pass inert validation and be reviewed before merge. This preserves unattended host security patching without silently changing the runner control plane. + +Controller and runner engine updates are also pinned. A merged private configuration change is applied with `install-worker-controller.sh --upgrade`; the host never follows a moving engine or configuration branch automatically. See [Git-authored controller desired state](DESIRED-STATE.md). diff --git a/docs/PROJECT-STANDARD.md b/docs/PROJECT-STANDARD.md index 24939b02..1e4e833b 100644 --- a/docs/PROJECT-STANDARD.md +++ b/docs/PROJECT-STANDARD.md @@ -50,6 +50,12 @@ Ordinary CI MUST target a wall-clock duration of five minutes or less when suffi Forty-five test-minutes divided among nine workers is a theoretical five-minute lower bound. Real plans generally need more than nine shards because job setup consumes part of the ceiling and test work is not perfectly balanced. Adding workers reduces wall-clock time only while runnable shards remain queued. +## Infrastructure-owned capacity + +Application workflows MUST submit every independent task and shard and MUST NOT use `strategy.max-parallel` to represent the number of available fleet workers. Excess jobs remain safely queued by GitHub. Pool budgets and controller maxima belong in the private schema-v3 infrastructure configuration described in [Git-authored controller desired state](DESIRED-STATE.md). + +A project MAY limit concurrency for a documented external-system restriction, such as an API rate limit or a single-writer fixture, only when that limit is independent of fleet size. Workflow-level concurrency groups MAY still cancel obsolete runs or serialize operations that are inherently unsafe to overlap. + ## Host independence A project CI job: @@ -109,7 +115,8 @@ Ordinary CI: - MUST NOT receive deployment, release, production, or internal-network credentials; - MUST NOT push branches, tags, releases, packages, or commits; - MUST set `timeout-minutes: 5` on every ordinary task-matrix job; -- SHOULD use concurrency controls appropriate to the project; +- SHOULD use concurrency groups to cancel obsolete runs or protect inherently single-writer operations; +- MUST NOT use concurrency settings to encode current fleet capacity; - MUST treat pull-request code as untrusted unless repository policy explicitly establishes otherwise. A job requiring write permission is not ordinary CI. It MUST be a separate job or workflow routed to an appropriate privileged runner group. diff --git a/docs/PUBLIC-PRIVATE-CONFIGURATION.md b/docs/PUBLIC-PRIVATE-CONFIGURATION.md index 12af79fd..32404da7 100644 --- a/docs/PUBLIC-PRIVATE-CONFIGURATION.md +++ b/docs/PUBLIC-PRIVATE-CONFIGURATION.md @@ -40,7 +40,7 @@ A private application repository may use the restricted runner group directly. A | Public application repository | Source code, public tests, public Dockerfiles, and unprivileged workflows | | Private application repository | Source code and workflows authorized to use an appropriate private runner group | | Private delivery repository for a public project | Approved source mappings, CI and deployment workflows, environment policy, promotion rules, and required secret names | -| Private organization configuration | Repository mappings, logical host groups, environment policy, capacity, image names, and internal operating notes | +| Private organization configuration | Repository mappings, logical controllers and host groups, environment policy, capacity budgets, pinned engine revisions, image names, and required secret names | | Secret manager, GitHub environment, or host-local protected file | Actual keys, tokens, passwords, and deployment credentials | Each organization generates its own private installation configuration from the public [configuration template](../templates/config-repository/README.md). The resulting private repository is an implementation detail of that organization and is not required to be accessible to users of the public project. @@ -78,6 +78,7 @@ The private delivery repository is the authorized GitHub Actions identity. Fleet - Private configuration may declare required secret names but never their values. - GitHub App keys, tokens, project secrets, and deployment credentials stay outside Git history. - Real private configuration is validated against the same public schema used by examples. +- Real machine addresses, VM IDs, storage/backup identifiers, credentials, and rendered runtime files remain outside both public and private Git history. - A host remains generic; adding a repository changes policy and project configuration, not every worker image. ## Secret locations diff --git a/docs/README.md b/docs/README.md index 31a77eaf..9deab26b 100644 --- a/docs/README.md +++ b/docs/README.md @@ -11,6 +11,7 @@ Use this index to find ci-fleet concepts, requirements, examples, and step-by-st | Decide whether it fits my infrastructure | [Architecture](ARCHITECTURE.md) and the root [README](../README.md) | | Try the experimental implementation safely | [Live pilot runbook](LIVE-PILOT.md) | | Add another Docker host, VM, computer, or VPS | [Adding a host](ADDING-A-HOST.md) | +| Manage controller configuration from Git | [Git-authored controller desired state](DESIRED-STATE.md) | | Add a private project to the shared runner pool | [Adding a project](ADDING-A-PROJECT.md) | | Convert an existing GitHub Actions workflow | [Migrating existing CI](MIGRATING-EXISTING-CI.md) | | Make a project compliant | [Project CI standard](PROJECT-STANDARD.md) and [compliance checklist](COMPLIANCE-CHECKLIST.md) | @@ -33,7 +34,8 @@ Use this index to find ci-fleet concepts, requirements, examples, and step-by-st | Scale set | One controller's uniquely named runner capacity advertised to GitHub. | | Shared routing label | The stable `runs-on` capability used by compatible projects, regardless of which physical host accepts the job. | | Test shard | One bounded slice of a larger test suite, designed to run independently and usually finish within five minutes. | -| Private delivery configuration | Organization names, repository allowlists, host inventory, capacity, network policy, and credentials kept outside this public repository. | +| Private delivery configuration | Repository allowlists, logical controllers, capacity budgets, environment policy, and required secret names kept in a private but secret-free repository. | +| Desired state | A reviewed schema-v3 declaration of runner pools and logical controllers stored in a private, secret-free Git repository. | ## Design and boundaries @@ -51,6 +53,7 @@ These pages are normative for compatible projects and hosts: - [Migration procedure](MIGRATING-EXISTING-CI.md) - [Compliance checklist](COMPLIANCE-CHECKLIST.md) - [Host maintenance standard](HOST-MAINTENANCE.md) +- [Git-authored controller desired state](DESIRED-STATE.md) - [Secrets model](SECRETS.md) - [Security policy](../SECURITY.md) @@ -58,12 +61,13 @@ These pages are normative for compatible projects and hosts: - [Run the live pilot](LIVE-PILOT.md) - [Add a host](ADDING-A-HOST.md) +- [Install, adopt, check, upgrade, roll back, or remove a controller](DESIRED-STATE.md) - [Add a project](ADDING-A-PROJECT.md) - [Deploy the current experimental prototype](DEPLOYMENT-PROTOTYPE.md) - [Maintain, drain, clean, update, and reboot hosts](HOST-MAINTENANCE.md) - [Use the public configuration-repository scaffold](../templates/config-repository/README.md) -The accepted target is a single target-host command, `sudo ./setup.sh`. Until that installer is implemented and released, follow the versioned pilot and deployment procedures above rather than assuming the future interface exists. +The schema-v3 worker-controller installer is implemented as `scripts/install-worker-controller.sh`. It remains experimental; use the [desired-state guide](DESIRED-STATE.md), pin both configuration and engine commits, and retain the manual pilot runbook for isolated first-job proof. ## Project integration examples diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index e86a4993..63bc64a1 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -62,6 +62,8 @@ Exit condition: both projects pass on the new fleet without removing the old pat ## Phase 5: Distributed capacity +- Manage runner pools and controllers through the schema-v3 private desired-state contract. +- Enroll or adopt hosts with the idempotent worker-controller installer. - Add a second independently provisioned Docker host. - Verify identical deployment and recovery. - Add demand-based scaling. diff --git a/docs/SECRETS.md b/docs/SECRETS.md index 7a8f2948..85dcf084 100644 --- a/docs/SECRETS.md +++ b/docs/SECRETS.md @@ -18,22 +18,25 @@ Project test or integration secrets belong in GitHub repository or environment s Normal validation should use no secret when possible. Deployment credentials must not be available to the normal shared validation pool. -### Host-local secrets +### Host-local identity and secrets A single-host prototype may use root-owned files outside the repository checkout, for example: ```text /etc/ci-fleet/secrets/github-app.pem +/etc/ci-fleet/host.env /etc/ci-fleet/ci-fleet.env ``` -Credential files should be owned by root, readable only by their intended service, and mounted only into the controller. +`host.env` contains the App client ID, installation ID, private-key path, and runner TTL. `ci-fleet.env` is rendered from reviewed desired state plus those approved host fields. Neither contains the PEM or application secrets, but both remain root-owned mode `0600` because they describe the controller identity and private installation. + +Credential files should be owned by root, readable only by their intended service, and mounted only into the controller. The installer never accepts a private key, token, or Git credential in a command-line argument. For a distributed fleet, use a secret manager or encrypted configuration system with independently revocable host identities. ## Docker Compose pattern -A future controller service may receive a host-local secret as a mounted file: +The controller service receives its host-local private key as a mounted file: ```yaml services: diff --git a/host/ci-fleet.env.example b/host/ci-fleet.env.example deleted file mode 100644 index fae86d89..00000000 --- a/host/ci-fleet.env.example +++ /dev/null @@ -1,22 +0,0 @@ -# Proposed configuration names only. -# This is not yet a stable or runnable interface. -# Never place real secrets in this file. - -CI_FLEET_GITHUB_ORGANIZATION=example-organization -CI_FLEET_RUNNER_GROUP=trusted-private-ci -CI_FLEET_SCALE_SET=ci-fleet-experimental - -CI_FLEET_MIN_RUNNERS=0 -CI_FLEET_MAX_RUNNERS=1 - -CI_FLEET_RUNNER_CPUS=4 -CI_FLEET_RUNNER_MEMORY=8g - -CI_FLEET_BUILD_CACHE_LIMIT=40GB -CI_FLEET_DISK_ALERT_PERCENT=75 -CI_FLEET_DISK_DRAIN_PERCENT=85 -CI_FLEET_DISK_EMERGENCY_PERCENT=90 - -# Long-lived credentials must be mounted from a root-owned file or -# retrieved from an external secret manager. They must not be placed here. -CI_FLEET_GITHUB_APP_KEY_FILE=/run/secrets/github_app_private_key diff --git a/host/host.env.example b/host/host.env.example new file mode 100644 index 00000000..e862c18a --- /dev/null +++ b/host/host.env.example @@ -0,0 +1,8 @@ +# Copy to /etc/ci-fleet/host.env and set mode 0600. +# These values stay on the controller host and are never rendered from Git. +# The private key itself must be root-owned mode 0600 at the path below. + +CI_FLEET_GITHUB_APP_CLIENT_ID=Iv1.EXAMPLE +CI_FLEET_GITHUB_APP_INSTALLATION_ID=12345678 +CI_FLEET_GITHUB_APP_PRIVATE_KEY_FILE=/etc/ci-fleet/secrets/github-app.pem +CI_FLEET_RUNNER_TTL=6h diff --git a/host/systemd/ci-fleet-cleanup.service b/host/systemd/ci-fleet-cleanup.service index 1bd47e1f..5eb060f4 100644 --- a/host/systemd/ci-fleet-cleanup.service +++ b/host/systemd/ci-fleet-cleanup.service @@ -6,6 +6,6 @@ Wants=docker.service [Service] Type=oneshot User=root -WorkingDirectory=/opt/ci-fleet +WorkingDirectory=/opt/ci-fleet/current EnvironmentFile=/etc/ci-fleet/ci-fleet.env -ExecStart=/opt/ci-fleet/scripts/cleanup.sh --apply +ExecStart=/opt/ci-fleet/current/scripts/cleanup.sh --apply diff --git a/host/systemd/ci-fleet-drift.service b/host/systemd/ci-fleet-drift.service new file mode 100644 index 00000000..50e8704c --- /dev/null +++ b/host/systemd/ci-fleet-drift.service @@ -0,0 +1,10 @@ +[Unit] +Description=Check ci-fleet host against its pinned Git desired state +After=docker.service network-online.target +Wants=docker.service network-online.target + +[Service] +Type=oneshot +User=root +WorkingDirectory=/opt/ci-fleet/manager/current +ExecStart=/opt/ci-fleet/manager/current/scripts/check-installed-state.sh diff --git a/host/systemd/ci-fleet-drift.timer b/host/systemd/ci-fleet-drift.timer new file mode 100644 index 00000000..4c197a40 --- /dev/null +++ b/host/systemd/ci-fleet-drift.timer @@ -0,0 +1,11 @@ +[Unit] +Description=Check pinned ci-fleet desired state every fifteen minutes + +[Timer] +OnBootSec=5min +OnUnitActiveSec=15min +AccuracySec=1min +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/host/systemd/ci-fleet-health.service b/host/systemd/ci-fleet-health.service index 00480e25..b33bc8fc 100644 --- a/host/systemd/ci-fleet-health.service +++ b/host/systemd/ci-fleet-health.service @@ -6,6 +6,6 @@ Wants=docker.service [Service] Type=oneshot User=root -WorkingDirectory=/opt/ci-fleet +WorkingDirectory=/opt/ci-fleet/manager/current EnvironmentFile=/etc/ci-fleet/ci-fleet.env -ExecStart=/opt/ci-fleet/scripts/healthcheck.sh +ExecStart=/opt/ci-fleet/manager/current/scripts/healthcheck.sh diff --git a/scripts/check-installed-state.sh b/scripts/check-installed-state.sh new file mode 100755 index 00000000..bfcc5e5b --- /dev/null +++ b/scripts/check-installed-state.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +state_file=${CI_FLEET_INSTALL_STATE_FILE:-/var/lib/ci-fleet/install-state.json} +installer=${CI_FLEET_INSTALLER:-/opt/ci-fleet/manager/current/scripts/install-worker-controller.sh} + +[[ -f "$state_file" ]] || { echo "ERROR: installed desired-state record is missing: $state_file" >&2; exit 2; } +command -v jq >/dev/null || { echo 'ERROR: jq is required' >&2; exit 2; } +[[ -x "$installer" ]] || { echo "ERROR: installer is unavailable: $installer" >&2; exit 2; } + +controller=$(jq -er .controller "$state_file") +config_repository=$(jq -er .config_repository "$state_file") +config_ref=$(jq -er .config_ref "$state_file") + +exec "$installer" --check \ + --config-repo "$config_repository" \ + --ref "$config_ref" \ + --controller "$controller" diff --git a/scripts/desired_state.py b/scripts/desired_state.py new file mode 100755 index 00000000..77ae16ad --- /dev/null +++ b/scripts/desired_state.py @@ -0,0 +1,274 @@ +#!/usr/bin/env python3 +"""Validate, select, and render schema-v3 ci-fleet controller desired state.""" + +from __future__ import annotations + +import argparse +import importlib.util +import json +import os +import re +import stat +import sys +import tempfile +from pathlib import Path +from typing import Any + + +ROOT = Path(__file__).resolve().parents[1] +TEMPLATE_VALIDATOR = ROOT / "templates" / "config-repository" / "scripts" / "validate.py" +COMMIT_SHA = re.compile(r"^[0-9a-f]{40}$") +SAFE_IDENTIFIER = re.compile(r"^[A-Za-z0-9._-]+$") +SAFE_DURATION = re.compile(r"^[1-9][0-9]*(?:s|m|h)$") +SAFE_ABSOLUTE_PATH = re.compile(r"^/[A-Za-z0-9._/-]+$") +SAFE_ENV_VALUE = re.compile(r"^[A-Za-z0-9._/:,-]+$") +HOST_REQUIRED = { + "CI_FLEET_GITHUB_APP_CLIENT_ID", + "CI_FLEET_GITHUB_APP_INSTALLATION_ID", + "CI_FLEET_GITHUB_APP_PRIVATE_KEY_FILE", +} +HOST_OPTIONAL = {"CI_FLEET_RUNNER_TTL"} + + +class DesiredStateError(ValueError): + """A safe operator-facing desired-state error.""" + + +def load_template_validator(): + spec = importlib.util.spec_from_file_location("ci_fleet_template_validator", TEMPLATE_VALIDATOR) + if spec is None or spec.loader is None: + raise DesiredStateError("public configuration validator could not be loaded") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def load_and_validate_config(path: Path) -> dict[str, Any]: + module = load_template_validator() + validation = module.Validation() + config = module.load_json(path, validation) + if config is not None: + module.scan_secret_material(config, validation) + module.validate_config(config, validation, False) + if validation.errors: + raise DesiredStateError("configuration rejected:\n" + "\n".join(f"- {error}" for error in validation.errors)) + if not isinstance(config, dict) or config.get("schema_version") != 3: + raise DesiredStateError("configuration must use schema_version 3") + return config + + +def parse_env(path: Path, *, allow_unknown: bool) -> dict[str, str]: + try: + lines = path.read_text(encoding="utf-8").splitlines() + except FileNotFoundError as exc: + raise DesiredStateError(f"host configuration does not exist: {path}") from exc + values: dict[str, str] = {} + for number, raw in enumerate(lines, start=1): + line = raw.strip() + if not line or line.startswith("#"): + continue + if "=" not in line: + raise DesiredStateError(f"{path}:{number}: expected NAME=value") + name, value = line.split("=", 1) + if not re.fullmatch(r"[A-Z][A-Z0-9_]*", name): + raise DesiredStateError(f"{path}:{number}: invalid variable name") + if name in values: + raise DesiredStateError(f"{path}:{number}: duplicate variable {name}") + if any(character in value for character in "\r\n\0"): + raise DesiredStateError(f"{path}:{number}: multiline values are forbidden") + if not allow_unknown and name not in HOST_REQUIRED | HOST_OPTIONAL: + raise DesiredStateError(f"{path}:{number}: unsupported host-local variable {name}") + values[name] = value + return values + + +def validate_host_values(values: dict[str, str]) -> dict[str, str]: + missing = sorted(HOST_REQUIRED - values.keys()) + if missing: + raise DesiredStateError("host configuration is missing: " + ", ".join(missing)) + client_id = values["CI_FLEET_GITHUB_APP_CLIENT_ID"] + installation_id = values["CI_FLEET_GITHUB_APP_INSTALLATION_ID"] + key_file = values["CI_FLEET_GITHUB_APP_PRIVATE_KEY_FILE"] + if not SAFE_IDENTIFIER.fullmatch(client_id): + raise DesiredStateError("GitHub App client ID contains unsupported characters") + if not installation_id.isdigit() or int(installation_id) < 1: + raise DesiredStateError("GitHub App installation ID must be a positive integer") + if not SAFE_ABSOLUTE_PATH.fullmatch(key_file): + raise DesiredStateError("GitHub App private-key path must be an absolute shell-safe path") + ttl = values.get("CI_FLEET_RUNNER_TTL", "6h") + if not SAFE_DURATION.fullmatch(ttl): + raise DesiredStateError("runner TTL must be a positive duration ending in s, m, or h") + return { + "CI_FLEET_GITHUB_APP_CLIENT_ID": client_id, + "CI_FLEET_GITHUB_APP_INSTALLATION_ID": installation_id, + "CI_FLEET_GITHUB_APP_PRIVATE_KEY_FILE": key_file, + "CI_FLEET_RUNNER_TTL": ttl, + } + + +def select_controller(config: dict[str, Any], controller_id: str) -> tuple[dict[str, Any], dict[str, Any]]: + controllers = config["controllers"] + if controller_id not in controllers: + available = ", ".join(sorted(controllers)) + raise DesiredStateError(f"controller {controller_id!r} is not declared; available: {available}") + controller = controllers[controller_id] + pool = config["runner_pools"][controller["pool"]] + return controller, pool + + +def build_rendered_env( + config: dict[str, Any], + controller_id: str, + host_values: dict[str, str], + *, + config_repository: str, + config_ref: str, + docker_gid: int, +) -> tuple[dict[str, str], dict[str, Any]]: + controller, pool = select_controller(config, controller_id) + engine_commit = controller["engine_ref"] + if not COMMIT_SHA.fullmatch(config_ref): + raise DesiredStateError("configuration ref must be a full lowercase commit SHA") + if docker_gid < 0: + raise DesiredStateError("Docker socket GID must be a non-negative integer") + if not SAFE_ENV_VALUE.fullmatch(config_repository): + raise DesiredStateError("configuration repository identity must be shell-safe") + + state = controller["state"] + configured_max = controller["max_runners"] + effective_max = configured_max if state == "active" else 0 + short_commit = engine_commit[:12] + rendered = { + "CI_FLEET_CAPACITY_BUDGET": str(pool["capacity_budget"]), + "CI_FLEET_CONFIGURED_MAX_RUNNERS": str(configured_max), + "CI_FLEET_CONFIG_REF": config_ref, + "CI_FLEET_CONFIG_REPOSITORY": config_repository, + "CI_FLEET_CONTROLLER_IMAGE": f"ci-fleet-controller:{short_commit}", + "CI_FLEET_CONTROLLER_STATE": state, + "CI_FLEET_DESIRED_STATE_SCHEMA": "3", + "CI_FLEET_DOCKER_GID": str(docker_gid), + "CI_FLEET_ENGINE_REF": engine_commit, + "CI_FLEET_GITHUB_URL": f"https://github.com/{config['organization']['slug']}", + "CI_FLEET_INSTANCE": controller_id, + "CI_FLEET_LABELS": ",".join(pool["routing_labels"]), + "CI_FLEET_MAX_RUNNERS": str(effective_max), + "CI_FLEET_MIN_RUNNERS": str(controller["min_runners"] if state == "active" else 0), + "CI_FLEET_RUNNER_CPUS": str(controller["runner_resources"]["cpu_cores"]), + "CI_FLEET_RUNNER_GROUP": pool["runner_group"], + "CI_FLEET_RUNNER_IMAGE": f"ci-fleet-runner:{short_commit}", + "CI_FLEET_RUNNER_MEMORY_MIB": str(controller["runner_resources"]["memory_mib"]), + "CI_FLEET_SCALE_SET_NAME": controller["scale_set_name"], + "CI_FLEET_VERSION": short_commit, + **validate_host_values(host_values), + } + for name, value in rendered.items(): + if not SAFE_ENV_VALUE.fullmatch(value): + raise DesiredStateError(f"rendered value for {name} contains unsafe characters") + metadata = { + "schema_version": 1, + "controller": controller_id, + "controller_state": state, + "pool": controller["pool"], + "location": controller["location"], + "lifecycle": controller["lifecycle"], + "scale_set_name": controller["scale_set_name"], + "configured_max_runners": configured_max, + "effective_max_runners": effective_max, + "capacity_budget": pool["capacity_budget"], + "config_repository": config_repository, + "config_ref": config_ref, + "engine_ref": engine_commit, + "engine_repository": config["organization"]["delivery_engine"], + } + return rendered, metadata + + +def write_private(path: Path, content: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + descriptor, temporary_name = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent, text=True) + temporary = Path(temporary_name) + try: + os.fchmod(descriptor, stat.S_IRUSR | stat.S_IWUSR) + with os.fdopen(descriptor, "w", encoding="utf-8") as handle: + handle.write(content) + handle.flush() + os.fsync(handle.fileno()) + os.replace(temporary, path) + finally: + temporary.unlink(missing_ok=True) + + +def render_env(values: dict[str, str]) -> str: + return "".join(f"{name}={values[name]}\n" for name in sorted(values)) + + +def command_validate(args: argparse.Namespace) -> None: + config = load_and_validate_config(args.config) + print(f"DESIRED_STATE_OK schema={config['schema_version']} controllers={len(config['controllers'])}") + + +def command_extract_host(args: argparse.Namespace) -> None: + values = parse_env(args.source, allow_unknown=True) + selected = validate_host_values({name: value for name, value in values.items() if name in HOST_REQUIRED | HOST_OPTIONAL}) + write_private(args.output, render_env(selected)) + print(f"HOST_CONFIG_WRITTEN path={args.output}") + + +def command_render(args: argparse.Namespace) -> None: + config = load_and_validate_config(args.config) + host_values = parse_env(args.host_config, allow_unknown=False) + values, metadata = build_rendered_env( + config, + args.controller, + host_values, + config_repository=args.config_repository, + config_ref=args.config_ref, + docker_gid=args.docker_gid, + ) + write_private(args.output, render_env(values)) + write_private(args.metadata_output, json.dumps(metadata, indent=2, sort_keys=True) + "\n") + print( + "DESIRED_STATE_RENDERED " + f"controller={metadata['controller']} state={metadata['controller_state']} " + f"config_ref={metadata['config_ref']} engine_ref={metadata['engine_ref']}" + ) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + subparsers = parser.add_subparsers(dest="command", required=True) + + validate = subparsers.add_parser("validate", help="validate a complete schema-v3 configuration") + validate.add_argument("--config", type=Path, required=True) + validate.set_defaults(function=command_validate) + + extract = subparsers.add_parser("extract-host-env", help="extract approved host-local values during adoption") + extract.add_argument("--source", type=Path, required=True) + extract.add_argument("--output", type=Path, required=True) + extract.set_defaults(function=command_extract_host) + + render = subparsers.add_parser("render", help="render one controller into a host-local runtime environment") + render.add_argument("--config", type=Path, required=True) + render.add_argument("--controller", required=True) + render.add_argument("--host-config", type=Path, required=True) + render.add_argument("--config-repository", required=True) + render.add_argument("--config-ref", required=True) + render.add_argument("--docker-gid", type=int, required=True) + render.add_argument("--output", type=Path, required=True) + render.add_argument("--metadata-output", type=Path, required=True) + render.set_defaults(function=command_render) + return parser.parse_args() + + +def main() -> int: + args = parse_args() + try: + args.function(args) + except DesiredStateError as exc: + print(f"ERROR: {exc}", file=sys.stderr) + return 2 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/healthcheck.sh b/scripts/healthcheck.sh index f50621bc..4763d5da 100755 --- a/scripts/healthcheck.sh +++ b/scripts/healthcheck.sh @@ -4,6 +4,7 @@ set -Eeuo pipefail warn_percent=${CI_FLEET_DISK_WARN_PERCENT:-80} critical_percent=${CI_FLEET_DISK_CRITICAL_PERCENT:-90} controller_name=${CI_FLEET_CONTROLLER_CONTAINER:-ci-fleet-controller-1} +desired_state=${CI_FLEET_CONTROLLER_STATE:-active} status=0 emit() { printf '%s\n' "$*"; } @@ -13,15 +14,22 @@ if ! command -v docker >/dev/null || ! docker info >/dev/null 2>&1; then fi emit "OK docker_available" -used=$(df -P /var/lib/docker 2>/dev/null | awk 'NR==2 {gsub(/%/, "", $5); print $5}') +docker_root=${CI_FLEET_DOCKER_ROOT:-/var/lib/docker} +used=$(df -P "$docker_root" 2>/dev/null | awk 'NR==2 {gsub(/%/, "", $5); print $5}' || true) if [[ -z "$used" ]]; then used=$(df -P / | awk 'NR==2 {gsub(/%/, "", $5); print $5}'); fi if ((used >= critical_percent)); then emit "CRITICAL disk_used_percent=$used"; status=2 elif ((used >= warn_percent)); then emit "WARN disk_used_percent=$used"; ((status < 1)) && status=1 else emit "OK disk_used_percent=$used"; fi controller_state=$(docker inspect --format '{{.State.Status}}' "$controller_name" 2>/dev/null || true) -if [[ "$controller_state" == running ]]; then emit "OK controller_state=running" -else emit "CRITICAL controller_state=${controller_state:-missing}"; status=2; fi +if [[ "$desired_state" == active ]]; then + if [[ "$controller_state" == running ]]; then emit "OK controller_state=running" + else emit "CRITICAL controller_state=${controller_state:-missing} desired_state=active"; status=2; fi +elif [[ -z "$controller_state" || "$controller_state" == exited || "$controller_state" == created ]]; then + emit "OK controller_state=${controller_state:-missing} desired_state=$desired_state" +else + emit "CRITICAL controller_state=$controller_state desired_state=$desired_state"; status=2 +fi stale=$(docker ps -aq --filter label=io.randomdevelopment.ci-fleet.managed=true --filter status=exited | wc -l | tr -d ' ') if ((stale > 0)); then emit "WARN inactive_managed_containers=$stale"; ((status < 1)) && status=1 diff --git a/scripts/install-worker-controller.sh b/scripts/install-worker-controller.sh new file mode 100755 index 00000000..f0241932 --- /dev/null +++ b/scripts/install-worker-controller.sh @@ -0,0 +1,557 @@ +#!/usr/bin/env bash +set -Eeuo pipefail +set +x + +repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +mode= +config_repo= +config_ref= +controller_id= +host_config_arg= +root_prefix=${CI_FLEET_ROOT_PREFIX:-} +testing=${CI_FLEET_TESTING:-0} +transaction_active=false +checkpoint_dir= + +usage() { + cat >&2 <<'EOF' +usage: + install-worker-controller.sh --check|--install|--adopt|--upgrade \ + --config-repo OWNER/REPOSITORY|PATH --ref FULL_COMMIT_SHA \ + --controller CONTROLLER_ID [--host-config PATH] + + install-worker-controller.sh --rollback + install-worker-controller.sh --uninstall + +Modes are mutually exclusive. Remote private repositories use the target host's +preconfigured read-only Git credentials; credentials are never accepted in URLs +or command-line arguments. +EOF +} + +note() { printf '%s\n' "$*"; } +die() { + printf 'ERROR: %s\n' "$*" >&2 + if [[ ${transaction_active:-false} == true ]] && declare -F restore_checkpoint >/dev/null; then + restore_checkpoint || true + transaction_active=false + fi + exit 2 +} + +while (($#)); do + case "$1" in + --check|--install|--adopt|--upgrade|--rollback|--uninstall) + [[ -z "$mode" ]] || die 'select exactly one operating mode' + mode=${1#--} + shift + ;; + --config-repo) + (($# >= 2)) || die '--config-repo requires a value' + config_repo=$2 + shift 2 + ;; + --ref) + (($# >= 2)) || die '--ref requires a value' + config_ref=$2 + shift 2 + ;; + --controller) + (($# >= 2)) || die '--controller requires a value' + controller_id=$2 + shift 2 + ;; + --host-config) + (($# >= 2)) || die '--host-config requires a value' + host_config_arg=$2 + shift 2 + ;; + -h|--help) + usage + exit 0 + ;; + *) + usage + die "unknown argument: $1" + ;; + esac +done + +[[ -n "$mode" ]] || { usage; die 'an explicit operating mode is required'; } +if [[ -n "$root_prefix" && "$testing" != 1 ]]; then + die 'CI_FLEET_ROOT_PREFIX is test-only and requires CI_FLEET_TESTING=1' +fi +if [[ "$testing" != 1 && ${EUID:-$(id -u)} -ne 0 ]]; then + die 'run this installer as root' +fi + +root_path() { printf '%s%s' "$root_prefix" "$1"; } + +install_root=$(root_path /opt/ci-fleet) +releases_dir=$install_root/releases +current_link=$install_root/current +manager_root=$install_root/manager +manager_releases=$manager_root/releases +manager_current=$manager_root/current +etc_dir=$(root_path /etc/ci-fleet) +rendered_env=$etc_dir/ci-fleet.env +default_host_config=$etc_dir/host.env +host_config=${host_config_arg:-$default_host_config} +state_root=$(root_path /var/lib/ci-fleet) +state_file=$state_root/install-state.json +checkpoints_dir=$state_root/checkpoints +systemd_dir=$(root_path /etc/systemd/system) + +temporary=$(mktemp -d) +cleanup_temporary() { rm -rf "$temporary"; } +trap cleanup_temporary EXIT + +require_commands() { + local command + for command in git python3 docker jq tar install cmp readlink systemctl stat awk grep date; do + command -v "$command" >/dev/null || die "$command is required" + done + docker info >/dev/null 2>&1 || die 'Docker daemon is unavailable' + docker compose version >/dev/null 2>&1 || die 'Docker Compose v2 is unavailable' +} + +validate_common_arguments() { + [[ -n "$config_repo" ]] || die '--config-repo is required for this mode' + [[ "$config_ref" =~ ^[0-9a-f]{40}$ ]] || die '--ref must be a full lowercase commit SHA' + [[ "$controller_id" =~ ^[a-z0-9][a-z0-9-]{0,62}$ ]] || die '--controller must be a lowercase logical ID' + if [[ "$config_repo" == *://* || "$config_repo" == *@* ]]; then + die '--config-repo must not contain a URL or embedded credentials; use OWNER/REPOSITORY or a local path' + fi +} + +resolve_config() { + local resolved checkout + candidate_config=$temporary/fleet.json + if [[ -d "$config_repo/.git" ]]; then + resolved=$(git -C "$config_repo" rev-parse "$config_ref^{commit}" 2>/dev/null || true) + [[ "$resolved" == "$config_ref" ]] || die 'local configuration repository does not contain the requested commit' + git -C "$config_repo" show "$config_ref:fleet.json" >"$candidate_config" || die 'fleet.json is absent at the requested configuration commit' + config_identity=$config_repo + return + fi + [[ "$config_repo" =~ ^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$ ]] || die '--config-repo must be OWNER/REPOSITORY or a local Git checkout' + checkout=$temporary/config-repository + git init -q "$checkout" + git -C "$checkout" remote add origin "https://github.com/${config_repo}.git" + if ! GIT_TERMINAL_PROMPT=0 git -C "$checkout" fetch -q --depth=1 origin "$config_ref"; then + die 'configuration fetch failed; configure a read-only credential on this host or use a local pinned checkout' + fi + resolved=$(git -C "$checkout" rev-parse 'FETCH_HEAD^{commit}') + [[ "$resolved" == "$config_ref" ]] || die 'fetched configuration commit does not match --ref' + git -C "$checkout" show "$config_ref:fleet.json" >"$candidate_config" || die 'fleet.json is absent at the requested configuration commit' + config_identity=$config_repo +} + +prepare_host_config() { + effective_host_config=$host_config + if [[ -f "$host_config" ]]; then + return + fi + if [[ -f "$rendered_env" && ( "$mode" == adopt || "$mode" == check ) ]]; then + if [[ "$mode" == check ]]; then + effective_host_config=$temporary/host.env + else + install -d -m 0700 "$etc_dir" + fi + python3 "$repo_root/scripts/desired_state.py" extract-host-env \ + --source "$rendered_env" --output "$effective_host_config" + return + fi + die "host-local GitHub App configuration is missing: $host_config" +} + +verify_host_files() { + local mode_bits owner expected_owner key_file + mode_bits=$(stat -c '%a' "$effective_host_config") + [[ "$mode_bits" == 600 ]] || die "host configuration must have mode 0600: $effective_host_config" + key_file=$(awk -F= '$1 == "CI_FLEET_GITHUB_APP_PRIVATE_KEY_FILE" {print substr($0, index($0, "=") + 1)}' "$effective_host_config") + [[ -n "$key_file" && -f "$key_file" ]] || die 'GitHub App PEM file is missing' + mode_bits=$(stat -c '%a' "$key_file") + owner=$(stat -c '%u' "$key_file") + expected_owner=0 + [[ "$testing" != 1 ]] || expected_owner=$(id -u) + [[ "$mode_bits" == 600 && "$owner" == "$expected_owner" ]] || die 'GitHub App PEM must be owned by root and have mode 0600' +} + +docker_gid() { + if [[ "$testing" == 1 && -n ${CI_FLEET_DOCKER_GID_OVERRIDE:-} ]]; then + printf '%s' "$CI_FLEET_DOCKER_GID_OVERRIDE" + return + fi + stat -c '%g' /var/run/docker.sock +} + +render_candidate() { + candidate_env=$temporary/ci-fleet.env + candidate_metadata=$temporary/metadata.json + python3 "$repo_root/scripts/desired_state.py" render \ + --config "$candidate_config" \ + --controller "$controller_id" \ + --host-config "$effective_host_config" \ + --config-repository "$config_identity" \ + --config-ref "$config_ref" \ + --docker-gid "$(docker_gid)" \ + --output "$candidate_env" \ + --metadata-output "$candidate_metadata" + target_state=$(jq -r .controller_state "$candidate_metadata") + engine_ref=$(jq -r .engine_ref "$candidate_metadata") + engine_repository=$(jq -r .engine_repository "$candidate_metadata") + release_dir=$releases_dir/$engine_ref +} + +compose() { + local release=$1 env=$2 + shift 2 + docker compose --env-file "$env" -f "$release/deploy/compose.yaml" "$@" +} + +controller_status() { + docker inspect --format '{{.State.Status}}' ci-fleet-controller-1 2>/dev/null || true +} + +current_runtime_release() { + if [[ -L "$current_link" ]]; then + readlink -f "$current_link" + elif [[ -f "$install_root/deploy/compose.yaml" ]]; then + printf '%s' "$install_root" + fi +} + +managed_runner_count() { + docker ps -q \ + --filter label=io.randomdevelopment.ci-fleet.managed=true \ + --filter label=io.randomdevelopment.ci-fleet.kind=runner | wc -l | tr -d ' ' +} + +runtime_matches() { + local expected=$1 status + status=$(controller_status) + if [[ "$expected" == active ]]; then + [[ "$status" == running ]] + else + [[ -z "$status" || "$status" == exited || "$status" == created ]] + fi +} + +state_matches() { + [[ -f "$state_file" ]] || return 1 + jq 'del(.installed_at)' "$state_file" >"$temporary/installed-metadata.json" || return 1 + cmp -s "$candidate_metadata" "$temporary/installed-metadata.json" +} + +release_matches() { + [[ -L "$current_link" ]] || return 1 + [[ $(readlink -f "$current_link") == $(readlink -f "$release_dir") ]] +} + +systemd_matches() { + [[ -L "$manager_current" ]] || return 1 + [[ -f "$systemd_dir/ci-fleet-health.service" && -f "$systemd_dir/ci-fleet-cleanup.service" && -f "$systemd_dir/ci-fleet-drift.service" ]] || return 1 + systemctl is-enabled --quiet ci-fleet-health.timer ci-fleet-cleanup.timer ci-fleet-drift.timer || return 1 + systemctl is-active --quiet ci-fleet-health.timer ci-fleet-cleanup.timer ci-fleet-drift.timer +} + +drift_count() { + local count=0 + [[ -f "$rendered_env" ]] && cmp -s "$candidate_env" "$rendered_env" || { note 'DRIFT rendered_environment'; count=$((count + 1)); } + release_matches || { note 'DRIFT engine_release'; count=$((count + 1)); } + state_matches || { note 'DRIFT install_state'; count=$((count + 1)); } + runtime_matches "$target_state" || { note 'DRIFT controller_runtime'; count=$((count + 1)); } + systemd_matches || { note 'DRIFT maintenance_timers'; count=$((count + 1)); } + DRIFT_COUNT=$count +} + +install_release() { + local archive marker_commit checkout resolved + if [[ -d "$release_dir" ]]; then + [[ -f "$release_dir/.ci-fleet-engine-ref" ]] || die "existing release has no engine marker: $release_dir" + marker_commit=$(<"$release_dir/.ci-fleet-engine-ref") + [[ "$marker_commit" == "$engine_ref" ]] || die "existing release marker does not match $engine_ref" + return + fi + install -d -m 0755 "$releases_dir" "$release_dir" + archive=$temporary/engine.tar + if [[ -d "$repo_root/.git" && $(git -C "$repo_root" rev-parse 'HEAD^{commit}') == "$engine_ref" ]]; then + git -C "$repo_root" archive --format=tar --output "$archive" HEAD + else + [[ "$engine_repository" =~ ^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$ ]] || die 'delivery engine repository is invalid' + checkout=$temporary/engine-repository + git init -q "$checkout" + git -C "$checkout" remote add origin "https://github.com/${engine_repository}.git" + GIT_TERMINAL_PROMPT=0 git -C "$checkout" fetch -q --depth=1 origin "$engine_ref" || die 'pinned ci-fleet engine commit could not be fetched' + resolved=$(git -C "$checkout" rev-parse 'FETCH_HEAD^{commit}') + [[ "$resolved" == "$engine_ref" ]] || die 'fetched ci-fleet engine commit does not match desired state' + git -C "$checkout" archive --format=tar --output "$archive" FETCH_HEAD + fi + tar -xf "$archive" -C "$release_dir" + printf '%s\n' "$engine_ref" >"$temporary/engine-ref" + install -m 0644 "$temporary/engine-ref" "$release_dir/.ci-fleet-engine-ref" +} + +install_manager() { + local manager_commit manager_release archive marker + if [[ -f "$repo_root/.ci-fleet-engine-ref" ]]; then + manager_commit=$(<"$repo_root/.ci-fleet-engine-ref") + else + [[ -d "$repo_root/.git" ]] || die 'installer manager source is not a Git checkout or installed manager release' + manager_commit=$(git -C "$repo_root" rev-parse 'HEAD^{commit}') + fi + [[ "$manager_commit" =~ ^[0-9a-f]{40}$ ]] || die 'installer manager commit is invalid' + manager_release=$manager_releases/$manager_commit + if [[ ! -d "$manager_release" ]]; then + [[ -d "$repo_root/.git" ]] || die 'a new installer manager release requires a Git checkout' + install -d -m 0755 "$manager_releases" "$manager_release" + archive=$temporary/manager.tar + git -C "$repo_root" archive --format=tar --output "$archive" HEAD + tar -xf "$archive" -C "$manager_release" + printf '%s\n' "$manager_commit" >"$temporary/manager-ref" + install -m 0644 "$temporary/manager-ref" "$manager_release/.ci-fleet-engine-ref" + else + marker=$(<"$manager_release/.ci-fleet-engine-ref") + [[ "$marker" == "$manager_commit" ]] || die 'existing installer manager marker is inconsistent' + fi + install -d -m 0755 "$manager_root" + ln -sfn "$manager_release" "$temporary/manager-current" + mv -Tf "$temporary/manager-current" "$manager_current" +} + +run_candidate_preflight() { + ( + set -a + # shellcheck disable=SC1090 + . "$candidate_env" + set +a + CI_FLEET_TESTING=$testing "$repo_root/scripts/preflight.sh" --managed + ) +} + +build_candidate() { + run_candidate_preflight + compose "$release_dir" "$candidate_env" config --quiet + [[ "$target_state" == disabled ]] || compose "$release_dir" "$candidate_env" build runner-image controller +} + +make_checkpoint() { + local timestamp target + timestamp=$(date -u +%Y%m%dT%H%M%SZ) + checkpoint_dir=$checkpoints_dir/${timestamp}-$$ + install -d -m 0700 "$checkpoint_dir" + [[ ! -f "$rendered_env" ]] || install -m 0600 "$rendered_env" "$checkpoint_dir/ci-fleet.env" + [[ ! -f "$state_file" ]] || install -m 0600 "$state_file" "$checkpoint_dir/install-state.json" + target=$(current_runtime_release) + if [[ -n "$target" ]]; then + printf '%s\n' "$target" >"$temporary/release-target" + install -m 0600 "$temporary/release-target" "$checkpoint_dir/release-target" + fi + note "CHECKPOINT_CREATED path=$checkpoint_dir" +} + +drain_current() { + local deadline count old_release status + status=$(controller_status) + [[ "$status" == running ]] || return 0 + [[ -f "$rendered_env" ]] || die 'cannot safely drain a running controller without its rendered environment' + old_release=$(current_runtime_release) + [[ -n "$old_release" && -f "$old_release/deploy/compose.yaml" ]] || die 'cannot locate the running controller Compose release for safe adoption' + compose "$old_release" "$rendered_env" pause controller >/dev/null + deadline=$((SECONDS + ${CI_FLEET_DRAIN_TIMEOUT_SECONDS:-300})) + while :; do + count=$(managed_runner_count) + if [[ "$count" == 0 ]]; then + break + fi + if ((SECONDS >= deadline)); then + compose "$old_release" "$rendered_env" unpause controller >/dev/null || true + die "drain timed out with $count managed runner(s) still present" + fi + sleep 2 + done + compose "$old_release" "$rendered_env" stop controller >/dev/null + note 'DRAIN_OK managed_runners=0' +} + +install_systemd_units() { + install -d -m 0755 "$systemd_dir" + install -m 0644 "$repo_root/host/systemd/ci-fleet-health.service" "$systemd_dir/" + install -m 0644 "$repo_root/host/systemd/ci-fleet-health.timer" "$systemd_dir/" + install -m 0644 "$repo_root/host/systemd/ci-fleet-cleanup.service" "$systemd_dir/" + install -m 0644 "$repo_root/host/systemd/ci-fleet-cleanup.timer" "$systemd_dir/" + install -m 0644 "$repo_root/host/systemd/ci-fleet-drift.service" "$systemd_dir/" + install -m 0644 "$repo_root/host/systemd/ci-fleet-drift.timer" "$systemd_dir/" + systemctl daemon-reload + systemctl enable --now ci-fleet-health.timer ci-fleet-cleanup.timer ci-fleet-drift.timer >/dev/null +} + +activate_candidate() { + install -d -m 0700 "$etc_dir" "$state_root" "$checkpoints_dir" + install -m 0600 "$candidate_env" "$rendered_env" + ln -sfn "$release_dir" "$temporary/current" + mv -Tf "$temporary/current" "$current_link" + install_manager + install_systemd_units + if [[ "$target_state" == active ]]; then + compose "$release_dir" "$rendered_env" up -d --no-deps controller + sleep "${CI_FLEET_STARTUP_WAIT_SECONDS:-2}" + runtime_matches active || die 'controller did not remain running after activation' + if ! ( + set -a + # shellcheck disable=SC1090 + . "$rendered_env" + set +a + "$repo_root/scripts/healthcheck.sh" + ); then + die 'post-activation health check failed' + fi + else + compose "$release_dir" "$rendered_env" stop controller >/dev/null 2>&1 || true + fi + jq --arg installed_at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" '. + {installed_at: $installed_at}' \ + "$candidate_metadata" >"$temporary/install-state.json" + install -m 0600 "$temporary/install-state.json" "$state_file" +} + +restore_checkpoint() { + local target restored_state + [[ -n "$checkpoint_dir" && -d "$checkpoint_dir" ]] || return 1 + set +e + if [[ -f "$checkpoint_dir/ci-fleet.env" ]]; then + install -m 0600 "$checkpoint_dir/ci-fleet.env" "$rendered_env" + else + rm -f "$rendered_env" + fi + if [[ -f "$checkpoint_dir/install-state.json" ]]; then + install -m 0600 "$checkpoint_dir/install-state.json" "$state_file" + else + rm -f "$state_file" + fi + if [[ -f "$checkpoint_dir/release-target" ]]; then + target=$(<"$checkpoint_dir/release-target") + if [[ -d "$target" ]]; then + ln -sfn "$target" "$temporary/rollback-current" + mv -Tf "$temporary/rollback-current" "$current_link" + release_dir=$target + install_systemd_units + if [[ -f "$rendered_env" ]]; then + restored_state=$(awk -F= '$1 == "CI_FLEET_CONTROLLER_STATE" {print $2}' "$rendered_env") + if [[ "$restored_state" == active ]]; then + compose "$release_dir" "$rendered_env" up -d --no-deps controller + fi + fi + fi + else + rm -f "$current_link" + fi + set -e + note "ROLLBACK_RESTORED checkpoint=$checkpoint_dir" +} + +on_error() { + local status=$? + if $transaction_active; then + restore_checkpoint || true + fi + exit "$status" +} +trap on_error ERR + +perform_check() { + local count + drift_count + count=$DRIFT_COUNT + if ((count > 0)); then + note "CHECK_FAILED drift=$count" + exit 3 + fi + note "CHECK_OK controller=$controller_id config_ref=$config_ref engine_ref=$engine_ref state=$target_state" +} + +perform_converge() { + local count existing_status + if [[ "$mode" == upgrade && ! -f "$state_file" ]]; then + die '--upgrade requires an existing managed installation; use --install or --adopt' + fi + existing_status=$(controller_status) + if [[ "$mode" == adopt && ! -f "$rendered_env" && ! -f "$state_file" && -z "$existing_status" ]]; then + die '--adopt requires an existing controller or configuration; use --install for a fresh host' + fi + if [[ "$mode" == install && -f "$rendered_env" && ! -f "$state_file" ]]; then + die 'an unmanaged controller configuration exists; use --adopt' + fi + drift_count + count=$DRIFT_COUNT + if ((count == 0)); then + note "NO_CHANGE controller=$controller_id config_ref=$config_ref engine_ref=$engine_ref state=$target_state" + return + fi + install_release + build_candidate + make_checkpoint + transaction_active=true + drain_current + activate_candidate + transaction_active=false + note "CONVERGED mode=$mode controller=$controller_id config_ref=$config_ref engine_ref=$engine_ref state=$target_state" +} + +latest_checkpoint() { + find "$checkpoints_dir" -mindepth 1 -maxdepth 1 -type d -printf '%T@ %p\n' 2>/dev/null | sort -nr | awk 'NR == 1 {print $2}' +} + +perform_rollback() { + checkpoint_dir=$(latest_checkpoint) + [[ -n "$checkpoint_dir" ]] || die 'no controller checkpoint is available' + drain_current + restore_checkpoint || die 'checkpoint restoration failed' + note "ROLLBACK_OK checkpoint=$checkpoint_dir" +} + +perform_uninstall() { + local old_release= + [[ ! -L "$current_link" ]] || old_release=$(readlink -f "$current_link") + make_checkpoint + transaction_active=true + drain_current + if [[ -n "$old_release" && -f "$rendered_env" ]]; then + compose "$old_release" "$rendered_env" down --remove-orphans || true + ( + set -a + # shellcheck disable=SC1090 + . "$rendered_env" + set +a + "$old_release/scripts/cleanup.sh" --apply --instance "${CI_FLEET_INSTANCE:-}" || true + ) + fi + systemctl disable --now ci-fleet-health.timer ci-fleet-cleanup.timer ci-fleet-drift.timer >/dev/null 2>&1 || true + rm -f \ + "$systemd_dir/ci-fleet-health.service" "$systemd_dir/ci-fleet-health.timer" \ + "$systemd_dir/ci-fleet-cleanup.service" "$systemd_dir/ci-fleet-cleanup.timer" \ + "$systemd_dir/ci-fleet-drift.service" "$systemd_dir/ci-fleet-drift.timer" + systemctl daemon-reload + rm -f "$current_link" "$rendered_env" "$state_file" + rm -f "$manager_current" + transaction_active=false + note "UNINSTALL_OK host_config_preserved=$host_config secrets_preserved=$etc_dir/secrets" +} + +require_commands +case "$mode" in + check|install|adopt|upgrade) + validate_common_arguments + resolve_config + prepare_host_config + verify_host_files + render_candidate + if [[ "$mode" == check ]]; then perform_check; else perform_converge; fi + ;; + rollback) + perform_rollback + ;; + uninstall) + perform_uninstall + ;; +esac diff --git a/scripts/preflight.sh b/scripts/preflight.sh index 7a418bd2..7751d516 100755 --- a/scripts/preflight.sh +++ b/scripts/preflight.sh @@ -3,6 +3,17 @@ set -Eeuo pipefail failures=0 warnings=0 +managed=false + +usage() { printf 'usage: %s [--managed]\n' "$0" >&2; } +while (($#)); do + case "$1" in + --managed) managed=true ;; + -h|--help) usage; exit 0 ;; + *) usage; exit 2 ;; + esac + shift +done ok() { printf 'OK %s\n' "$*"; } warn() { printf 'WARN %s\n' "$*"; warnings=$((warnings + 1)); } @@ -24,8 +35,14 @@ for name in "${required[@]}"; do if [[ -n "${!name:-}" ]]; then ok "$name is set"; else fail "$name is missing"; fi done -if [[ "${CI_FLEET_MIN_RUNNERS:-0}" != 0 ]]; then fail "CI_FLEET_MIN_RUNNERS must be 0 for the pilot"; fi -if [[ "${CI_FLEET_MAX_RUNNERS:-1}" != 1 ]]; then fail "CI_FLEET_MAX_RUNNERS must be 1 for the pilot"; fi +if [[ "${CI_FLEET_MIN_RUNNERS:-0}" != 0 ]]; then fail "CI_FLEET_MIN_RUNNERS must be 0"; fi +if $managed; then + if [[ ! "${CI_FLEET_MAX_RUNNERS:-}" =~ ^[0-9]+$ ]]; then + fail "CI_FLEET_MAX_RUNNERS must be a non-negative integer for a managed host" + fi +else + if [[ "${CI_FLEET_MAX_RUNNERS:-1}" != 1 ]]; then fail "CI_FLEET_MAX_RUNNERS must be 1 for the pilot"; fi +fi if [[ "${CI_FLEET_SCALE_SET_NAME:-}" != *"${CI_FLEET_INSTANCE:-__missing__}"* ]]; then fail "scale-set name must include the unique fleet instance ID" fi @@ -39,7 +56,12 @@ if command -v docker >/dev/null; then if docker compose version >/dev/null 2>&1; then ok "Docker Compose plugin is available"; else fail "Docker Compose plugin is unavailable"; fi fi -socket_gid=$(stat -c '%g' /var/run/docker.sock 2>/dev/null || true) +docker_socket=${CI_FLEET_DOCKER_SOCKET:-/var/run/docker.sock} +if [[ "${CI_FLEET_TESTING:-0}" == 1 && -n "${CI_FLEET_DOCKER_GID_OVERRIDE:-}" ]]; then + socket_gid=$CI_FLEET_DOCKER_GID_OVERRIDE +else + socket_gid=$(stat -c '%g' "$docker_socket" 2>/dev/null || true) +fi if [[ -n "$socket_gid" && "$socket_gid" == "${CI_FLEET_DOCKER_GID:-}" ]]; then ok "Docker socket group matches CI_FLEET_DOCKER_GID" else @@ -50,13 +72,16 @@ secret=${CI_FLEET_GITHUB_APP_PRIVATE_KEY_FILE:-} if [[ -f "$secret" ]]; then mode=$(stat -c '%a' "$secret") owner=$(stat -c '%u' "$secret") - if [[ "$mode" == 600 && "$owner" == 0 ]]; then ok "GitHub App PEM is root-owned mode 0600" - else fail "GitHub App PEM must be root-owned mode 0600 (found uid=$owner mode=$mode)"; fi + expected_owner=0 + [[ "${CI_FLEET_TESTING:-0}" != 1 ]] || expected_owner=$(id -u) + if [[ "$mode" == 600 && "$owner" == "$expected_owner" ]]; then ok "GitHub App PEM owner and mode are valid" + else fail "GitHub App PEM must be owned by the installer account with mode 0600 (found uid=$owner mode=$mode)"; fi else fail "GitHub App PEM file is missing" fi -disk_used=$(df -P /var/lib/docker 2>/dev/null | awk 'NR==2 {gsub(/%/, "", $5); print $5}') +docker_root=${CI_FLEET_DOCKER_ROOT:-/var/lib/docker} +disk_used=$(df -P "$docker_root" 2>/dev/null | awk 'NR==2 {gsub(/%/, "", $5); print $5}' || true) if [[ -z "$disk_used" ]]; then disk_used=$(df -P / | awk 'NR==2 {gsub(/%/, "", $5); print $5}'); fi if ((disk_used >= 80)); then fail "Docker filesystem is ${disk_used}% full"; else ok "Docker filesystem is ${disk_used}% full"; fi diff --git a/scripts/test-install-worker-controller.sh b/scripts/test-install-worker-controller.sh new file mode 100755 index 00000000..274205c8 --- /dev/null +++ b/scripts/test-install-worker-controller.sh @@ -0,0 +1,176 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +tmp=$(mktemp -d) +trap 'rm -rf "$tmp"' EXIT +fake_bin=$tmp/bin +mkdir -p "$fake_bin" + +cat >"$fake_bin/docker" <<'EOF' +#!/usr/bin/env bash +set -u +state=${FAKE_DOCKER_STATE:?} +case "${1:-}" in + info) exit 0 ;; + inspect) + [[ -f "$state" ]] || exit 1 + if [[ "$*" == *'.State.Status'* ]]; then printf 'running\n'; else printf 'running\n'; fi + ;; + ps) exit 0 ;; + volume|network) + [[ "${2:-}" == ls ]] && exit 0 + [[ "${2:-}" == inspect ]] && exit 1 + exit 0 + ;; + compose) + if [[ "${2:-}" == version ]]; then exit 0; fi + command= + for argument in "$@"; do + case "$argument" in config|build|up|stop|pause|unpause|down|logs|rm) command=$argument ;; esac + done + case "$command" in + up) + if [[ -n "${FAKE_FAIL_UP_ONCE:-}" && -f "$FAKE_FAIL_UP_ONCE" ]]; then + rm -f "$FAKE_FAIL_UP_ONCE" + exit 42 + fi + : >"$state" + ;; + stop|down|rm) rm -f "$state" ;; + config|build|pause|unpause|logs) ;; + *) exit 1 ;; + esac + ;; + *) exit 1 ;; +esac +EOF + +cat >"$fake_bin/systemctl" <<'EOF' +#!/usr/bin/env bash +exit 0 +EOF +chmod 700 "$fake_bin/docker" "$fake_bin/systemctl" + +export PATH="$fake_bin:$PATH" +export FAKE_DOCKER_STATE=$tmp/docker-controller-running +export CI_FLEET_TESTING=1 +export CI_FLEET_DOCKER_GID_OVERRIDE=998 +export CI_FLEET_STARTUP_WAIT_SECONDS=0 +export CI_FLEET_DRAIN_TIMEOUT_SECONDS=2 + +fail() { printf 'FAIL %s\n' "$*" >&2; exit 1; } +expect_success() { + local output + output=$("$@" 2>&1) || fail "expected success: $*; output=$output" + printf '%s\n' "$output" +} +expect_failure() { + local expected=$1 output + shift + if output=$("$@" 2>&1); then fail "expected failure: $*"; fi + grep -Fq -- "$expected" <<<"$output" || fail "missing failure [$expected]: $output" +} + +engine_ref=$(git -C "$repo_root" rev-parse 'HEAD^{commit}') +config_repo=$tmp/config-repo +git init -q "$config_repo" +git -C "$config_repo" config user.name fixture +git -C "$config_repo" config user.email fixture@example.invalid + +write_config() { + local state=$1 maximum=$2 budget=$3 + python3 - "$repo_root/templates/config-repository/fleet.json" "$config_repo/fleet.json" "$engine_ref" "$state" "$maximum" "$budget" <<'PY' +import json +import sys +source, target, engine_ref, state, maximum, budget = sys.argv[1:] +value = json.load(open(source, encoding="utf-8")) +controller = value["controllers"]["example-ci-01"] +controller["engine_ref"] = engine_ref +controller["state"] = state +controller["max_runners"] = int(maximum) +value["runner_pools"]["trusted-ci"]["capacity_budget"] = int(budget) +with open(target, "w", encoding="utf-8") as handle: + json.dump(value, handle, indent=2) + handle.write("\n") +PY + git -C "$config_repo" add fleet.json + git -C "$config_repo" commit -q -m "fixture $state $maximum" + git -C "$config_repo" rev-parse HEAD +} + +root=$tmp/host +export CI_FLEET_ROOT_PREFIX=$root +export CI_FLEET_DOCKER_ROOT=$root/var/lib/docker +mkdir -p "$root/etc/ci-fleet/secrets" "$CI_FLEET_DOCKER_ROOT" +pem=$root/etc/ci-fleet/secrets/github-app.pem +printf 'fixture only\n' >"$pem" +chmod 600 "$pem" +host_config=$root/etc/ci-fleet/host.env +printf '%s\n' \ + 'CI_FLEET_GITHUB_APP_CLIENT_ID=Iv1.EXAMPLE' \ + 'CI_FLEET_GITHUB_APP_INSTALLATION_ID=123456' \ + "CI_FLEET_GITHUB_APP_PRIVATE_KEY_FILE=$pem" \ + 'CI_FLEET_RUNNER_TTL=6h' >"$host_config" +chmod 600 "$host_config" + +ref_one=$(write_config active 1 1) +installer=$repo_root/scripts/install-worker-controller.sh +base_args=(--config-repo "$config_repo" --controller example-ci-01) + +first=$(expect_success "$installer" --install "${base_args[@]}" --ref "$ref_one") +grep -Fq 'CONVERGED mode=install' <<<"$first" || fail 'fresh install did not converge' +[[ -L "$root/opt/ci-fleet/current" && -f "$root/var/lib/ci-fleet/install-state.json" ]] || fail 'fresh install state is incomplete' +[[ -f "$FAKE_DOCKER_STATE" ]] || fail 'active controller was not started' + +second=$(expect_success "$installer" --install "${base_args[@]}" --ref "$ref_one") +grep -Fq 'NO_CHANGE' <<<"$second" || fail 'idempotent rerun changed the host' +expect_success "$installer" --check "${base_args[@]}" --ref "$ref_one" >/dev/null + +printf '\n' >>"$root/etc/ci-fleet/ci-fleet.env" +expect_failure 'DRIFT rendered_environment' "$installer" --check "${base_args[@]}" --ref "$ref_one" +expect_success "$installer" --install "${base_args[@]}" --ref "$ref_one" >/dev/null + +ref_two=$(write_config active 2 2) +export FAKE_FAIL_UP_ONCE=$tmp/fail-up-once +: >"$FAKE_FAIL_UP_ONCE" +expect_failure 'ROLLBACK_RESTORED' "$installer" --upgrade "${base_args[@]}" --ref "$ref_two" +unset FAKE_FAIL_UP_ONCE +grep -Fq 'CI_FLEET_MAX_RUNNERS=1' "$root/etc/ci-fleet/ci-fleet.env" || fail 'failed activation did not restore capacity one' +expect_success "$installer" --upgrade "${base_args[@]}" --ref "$ref_two" >/dev/null +grep -Fq 'CI_FLEET_MAX_RUNNERS=2' "$root/etc/ci-fleet/ci-fleet.env" || fail 'upgrade did not apply capacity two' + +expect_success "$installer" --rollback >/dev/null +grep -Fq 'CI_FLEET_MAX_RUNNERS=1' "$root/etc/ci-fleet/ci-fleet.env" || fail 'rollback did not restore capacity one' + +ref_three=$(write_config drained 2 2) +expect_success "$installer" --upgrade "${base_args[@]}" --ref "$ref_three" >/dev/null +grep -Fq 'CI_FLEET_CONTROLLER_STATE=drained' "$root/etc/ci-fleet/ci-fleet.env" || fail 'drained state was not rendered' +grep -Fq 'CI_FLEET_MAX_RUNNERS=0' "$root/etc/ci-fleet/ci-fleet.env" || fail 'drained controller retained effective capacity' +[[ ! -f "$FAKE_DOCKER_STATE" ]] || fail 'drained controller remained running' + +expect_success "$installer" --uninstall >/dev/null +[[ ! -e "$root/opt/ci-fleet/current" && ! -e "$root/var/lib/ci-fleet/install-state.json" ]] || fail 'uninstall left active installation state' +[[ -f "$host_config" && -f "$pem" ]] || fail 'uninstall removed preserved host credentials' + +adopt_root=$tmp/adopt-host +export CI_FLEET_ROOT_PREFIX=$adopt_root +export FAKE_DOCKER_STATE=$tmp/adopt-controller-running +mkdir -p "$adopt_root/etc/ci-fleet/secrets" "$adopt_root/opt/ci-fleet/deploy" +adopt_pem=$adopt_root/etc/ci-fleet/secrets/github-app.pem +printf 'fixture only\n' >"$adopt_pem" +chmod 600 "$adopt_pem" +cp "$repo_root/deploy/compose.yaml" "$adopt_root/opt/ci-fleet/deploy/compose.yaml" +printf '%s\n' \ + 'CI_FLEET_GITHUB_APP_CLIENT_ID=Iv1.EXAMPLE' \ + 'CI_FLEET_GITHUB_APP_INSTALLATION_ID=123456' \ + "CI_FLEET_GITHUB_APP_PRIVATE_KEY_FILE=$adopt_pem" \ + 'CI_FLEET_RUNNER_TTL=6h' >"$adopt_root/etc/ci-fleet/ci-fleet.env" +chmod 600 "$adopt_root/etc/ci-fleet/ci-fleet.env" +: >"$FAKE_DOCKER_STATE" + +adopt=$(expect_success "$installer" --adopt "${base_args[@]}" --ref "$ref_one") +grep -Fq 'CONVERGED mode=adopt' <<<"$adopt" || fail 'adoption did not converge' +[[ -f "$adopt_root/etc/ci-fleet/host.env" ]] || fail 'adoption did not separate host-local values' + +printf 'INSTALLER_TESTS_OK\n' diff --git a/scripts/test_desired_state.py b/scripts/test_desired_state.py new file mode 100755 index 00000000..8dce0aa2 --- /dev/null +++ b/scripts/test_desired_state.py @@ -0,0 +1,142 @@ +#!/usr/bin/env python3 +"""Deterministic tests for schema-v3 controller selection and rendering.""" + +from __future__ import annotations + +import copy +import json +import tempfile +import unittest +from pathlib import Path + +from desired_state import ( + DesiredStateError, + build_rendered_env, + load_and_validate_config, + parse_env, + validate_host_values, +) + + +ROOT = Path(__file__).resolve().parents[1] +CONFIG_COMMIT = "1" * 40 + + +def config() -> dict: + return json.loads((ROOT / "templates" / "config-repository" / "fleet.json").read_text(encoding="utf-8")) + + +def host_values() -> dict[str, str]: + return { + "CI_FLEET_GITHUB_APP_CLIENT_ID": "Iv1.EXAMPLE", + "CI_FLEET_GITHUB_APP_INSTALLATION_ID": "123456", + "CI_FLEET_GITHUB_APP_PRIVATE_KEY_FILE": "/etc/ci-fleet/secrets/github-app.pem", + "CI_FLEET_RUNNER_TTL": "6h", + } + + +class DesiredStateTests(unittest.TestCase): + def render(self, value: dict | None = None): + return build_rendered_env( + value or config(), + "example-ci-01", + host_values(), + config_repository="example-org/example-fleet-config", + config_ref=CONFIG_COMMIT, + docker_gid=998, + ) + + def test_active_controller_renders_configured_capacity(self) -> None: + environment, metadata = self.render() + self.assertEqual(environment["CI_FLEET_MAX_RUNNERS"], "1") + self.assertEqual(environment["CI_FLEET_CONFIGURED_MAX_RUNNERS"], "1") + self.assertEqual(environment["CI_FLEET_LABELS"], "docker-ci") + self.assertEqual(metadata["controller_state"], "active") + + def test_drained_controller_renders_zero_effective_capacity(self) -> None: + value = config() + value["controllers"]["example-ci-01"]["state"] = "drained" + environment, metadata = self.render(value) + self.assertEqual(environment["CI_FLEET_MAX_RUNNERS"], "0") + self.assertEqual(environment["CI_FLEET_CONFIGURED_MAX_RUNNERS"], "1") + self.assertEqual(metadata["effective_max_runners"], 0) + + def test_disabled_controller_renders_zero_effective_capacity(self) -> None: + value = config() + value["controllers"]["example-ci-01"]["state"] = "disabled" + environment, _ = self.render(value) + self.assertEqual(environment["CI_FLEET_MAX_RUNNERS"], "0") + + def test_missing_controller_fails_closed(self) -> None: + with self.assertRaisesRegex(DesiredStateError, "is not declared"): + build_rendered_env( + config(), + "missing-ci-01", + host_values(), + config_repository="example-org/example-fleet-config", + config_ref=CONFIG_COMMIT, + docker_gid=998, + ) + + def test_capacity_overcommit_is_rejected_by_public_contract(self) -> None: + value = config() + value["controllers"]["example-ci-01"]["max_runners"] = 2 + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "fleet.json" + path.write_text(json.dumps(value), encoding="utf-8") + with self.assertRaisesRegex(DesiredStateError, "capacity_budget"): + load_and_validate_config(path) + + def test_project_max_parallel_is_rejected(self) -> None: + value = config() + value["projects"]["example-app"]["ci_contract"]["max_parallel"] = 1 + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "fleet.json" + path.write_text(json.dumps(value), encoding="utf-8") + with self.assertRaisesRegex(DesiredStateError, "max_parallel"): + load_and_validate_config(path) + + def test_unknown_host_local_variable_is_rejected(self) -> None: + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "host.env" + path.write_text( + "CI_FLEET_GITHUB_APP_CLIENT_ID=Iv1.EXAMPLE\n" + "CI_FLEET_GITHUB_APP_INSTALLATION_ID=123\n" + "CI_FLEET_GITHUB_APP_PRIVATE_KEY_FILE=/safe/key.pem\n" + "PROJECT_SECRET=forbidden\n", + encoding="utf-8", + ) + with self.assertRaisesRegex(DesiredStateError, "unsupported host-local variable"): + parse_env(path, allow_unknown=False) + + def test_duplicate_host_local_variable_is_rejected(self) -> None: + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "host.env" + path.write_text( + "CI_FLEET_GITHUB_APP_CLIENT_ID=first\n" + "CI_FLEET_GITHUB_APP_CLIENT_ID=second\n", + encoding="utf-8", + ) + with self.assertRaisesRegex(DesiredStateError, "duplicate variable"): + parse_env(path, allow_unknown=False) + + def test_host_values_require_absolute_key_path(self) -> None: + values = copy.deepcopy(host_values()) + values["CI_FLEET_GITHUB_APP_PRIVATE_KEY_FILE"] = "relative.pem" + with self.assertRaisesRegex(DesiredStateError, "absolute shell-safe"): + validate_host_values(values) + + def test_shell_metacharacter_in_config_identity_is_rejected(self) -> None: + with self.assertRaisesRegex(DesiredStateError, "must be shell-safe"): + build_rendered_env( + config(), + "example-ci-01", + host_values(), + config_repository="/tmp/config$(id)", + config_ref=CONFIG_COMMIT, + docker_gid=998, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/validate.sh b/scripts/validate.sh index 9628d645..816848eb 100755 --- a/scripts/validate.sh +++ b/scripts/validate.sh @@ -5,11 +5,17 @@ repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) cd "$repo_root" for script in scripts/*.sh examples/project/scripts/ci/*.sh; do bash -n "$script"; done -python3 -m py_compile .github/actions/plan/plan.py .github/actions/plan/test_plan.py +python3 -m py_compile \ + .github/actions/plan/plan.py \ + .github/actions/plan/test_plan.py \ + scripts/desired_state.py \ + scripts/test_desired_state.py python3 .github/actions/plan/test_plan.py +python3 scripts/test_desired_state.py python3 .github/actions/plan/plan.py --plan examples/project/scripts/ci/plan.json --group fast >/dev/null python3 .github/actions/plan/plan.py --plan examples/project/scripts/ci/plan.json --group full >/dev/null scripts/test-capacity-preflight.sh +scripts/test-install-worker-controller.sh tmp=$(mktemp) trap 'rm -f "$tmp"' EXIT diff --git a/templates/config-repository/AGENTS.md b/templates/config-repository/AGENTS.md index d340d357..c6d46a8a 100644 --- a/templates/config-repository/AGENTS.md +++ b/templates/config-repository/AGENTS.md @@ -2,7 +2,7 @@ ## Purpose -This repository maps projects to CI pools, deployment environments, logical host groups, and container images. It never stores secret values. +This repository maps projects to CI pools, Git-authored controller desired state, deployment environments, logical host groups, and container images. It never stores secret values or host-local infrastructure details. ## Required verification @@ -15,11 +15,14 @@ Before committing configuration changes, run: ## Hard rules - Never add real `.env` files, credentials, tokens, private keys, cookies, or passwords. +- Never add addresses, VM IDs, storage identifiers, backup identifiers, SSH details, or rendered runtime configuration. - Do not weaken `public_repositories: false` for Docker-socket runner pools. +- Infrastructure configuration owns capacity. Application workflows submit all independent jobs and do not use `max-parallel` to model fleet size. +- The sum of active and drained controller maxima must not exceed the pool capacity budget. +- Controller engine revisions and reusable workflows must be pinned to full reviewed commit SHAs. - Production environments must require approval and must not deploy automatically. - CI runner hosts and application deployment hosts are separate roles. - Image promotion uses immutable digests; do not rebuild separately for production. -- Reusable workflows must be pinned to a full reviewed commit SHA. - Keep ordinary task jobs at a five-minute hard ceiling and expected shard payload at four minutes or less. - Preserve deterministic task/shard isolation; Compose identity must include task and shard as well as run identity. -- Update schema and validator together. +- Update schema, initializer, validator, tests, examples, and documentation together. diff --git a/templates/config-repository/README.md b/templates/config-repository/README.md index 9c536c13..952e1535 100644 --- a/templates/config-repository/README.md +++ b/templates/config-repository/README.md @@ -1,31 +1,42 @@ # ci-fleet configuration template -This is the public, secret-free starting point for an organization's private `ci-fleet` configuration repository. It records which trusted projects may use each CI pool, which logical deployment environments exist, and the standardized commands every project must expose. +This is the public, secret-free starting point for an organization's private `ci-fleet` configuration repository. It records which trusted projects may use each CI pool, the reviewed desired state for controller machines, infrastructure capacity budgets, logical deployment environments, and the standardized commands every project must expose. -It does **not** contain runner registration tokens, deploy credentials, private keys, host addresses, or `.env` files. +It does **not** contain runner registration tokens, deploy credentials, private keys, host addresses, VM IDs, storage names, backup identifiers, or `.env` files. ```mermaid flowchart LR - E[Public ci-fleet engine] -->|pinned reusable workflow| P[Private project repos] - C[Private config created from this template] -->|logical policy| P - P -->|task matrix| R[Trusted Docker CI pool] + E[Public ci-fleet engine] -->|pinned engine commit| C[Private configuration] + C -->|pool policy and capacity budget| A[Controller at site A] + C -->|pool policy and capacity budget| B[Controller at site B] + P[Authorized project repositories] -->|all independent jobs| G[GitHub runner group] + A --> G + B --> G + G --> R[Ephemeral Docker workers] P -->|approved image digest| D[Development hosts] - P -->|manual approval + image digest| X[Production hosts] - S[GitHub Environments / host secret store] -. secret values .-> P + P -->|manual approval and same digest| X[Production hosts] + S[GitHub Environments or host secret store] -. secret values .-> P classDef public fill:#dff4ff,stroke:#1570a6,color:#102a43 classDef private fill:#fff3cd,stroke:#9a6700,color:#3d2b00 class E public - class C,P,R,D,X,S private + class C,A,B,P,G,R,D,X,S private ``` ## Start a private organization configuration 1. Create a **private** repository from this public template. -2. Clone it and initialize the first project: +2. Clone it and initialize the first project and controller: ```bash - ./scripts/init.sh --organization your-org --project your-app + ./scripts/init.sh \ + --organization your-org \ + --project your-app \ + --controller ci-01 \ + --location primary-site \ + --capacity-budget 1 \ + --max-runners 1 \ + --engine-ref ``` 3. Edit `fleet.json` to add the organization's real logical mappings. @@ -35,23 +46,77 @@ flowchart LR ./scripts/validate.sh --strict ``` -5. Configure secret **values** in GitHub Environments or the deployment host's secret manager. The repository stores only names such as `DEPLOY_AUTH`. +5. Configure secret **values** in GitHub Environments, root-owned host files, or an external secret manager. The repository stores only names such as `DEPLOY_AUTH`. -The initializer refuses to replace a configured file unless `--force` is explicit. Run `./scripts/init.sh --help` for repository, registry, label, and output options. +The initializer refuses to replace a configured file unless `--force` is explicit. Run `./scripts/init.sh --help` for repository, registry, runner-group, controller, location, capacity, resource, and output options. + +## Schema v3: Git-authored controller desired state + +`fleet.json` is the reviewed authority for logical controller state. Each entry in `controllers` has a unique ID and declares: + +- its runner pool and logical location; +- whether it is `active`, `drained`, or `disabled`; +- its unique GitHub scale-set name; +- an `experimental`, `stable`, or `retiring` lifecycle; +- the full reviewed ci-fleet commit SHA it runs; +- minimum and maximum runners; +- CPU and memory available to each ephemeral runner. + +The controller ID is how a target host selects its declaration. A location is a non-sensitive logical slug such as `primary-site` or `remote-site`, never an address. Runtime-generated configuration and credentials remain host-local. + +### Pool capacity is infrastructure policy + +Each runner pool has a `capacity_budget`. The validator totals the maximum capacity of every active or drained controller assigned to the pool and rejects overcommit. Drained capacity remains reserved so an undrain cannot silently exceed the reviewed budget. Disabled controllers do not reserve capacity. + +Application repositories do not encode the number of available workers. They submit all independent tasks and shards. Do not use GitHub Actions `strategy.max-parallel` to model fleet size; controllers and the private configuration decide how many jobs run simultaneously. An application may limit concurrency only for a separately documented external-system constraint, not worker availability. + +This separation lets one infrastructure change add, remove, drain, or resize controllers without editing every project workflow. + +## Target-host installation and adoption + +The public ci-fleet engine owns the host installer. Its intended interface consumes one logical controller from a pinned private configuration revision: + +```bash +sudo ./scripts/install-worker-controller.sh \ + --config-repo example-org/example-fleet-config \ + --controller example-ci-01 \ + --ref \ + --install +``` + +Use `--adopt` instead of `--install` to bring an existing controller under Git-authored desired state. The engine contract also provides `--check`, `--upgrade`, `--rollback`, and `--uninstall` modes. + +The command runs on the target Linux Docker machine. It validates the pinned configuration, renders host-local runtime state, preserves root-owned secrets, drains before disruptive changes, verifies a recoverable checkpoint, installs maintenance services, checks health, and reports drift without exposing credentials. OpenClaw or another agent may invoke it, but no agent is required. + +The engine-side implementation is `scripts/install-worker-controller.sh` in the parent ci-fleet repository and remains experimental. This configuration repository never installs a controller by itself. + +## Drain, retire, and delete a controller host + +Host retirement is an explicit reviewed transition: + +1. Change the controller state to `drained` and merge the private configuration change. +2. Converge the host and verify that it accepts no new work and has no active runner. +3. Remove only fleet-owned residue and verify replacement capacity. +4. Unregister its scale set and revoke that host's credentials. +5. Change the declaration to `disabled` or remove it in a later reviewed change. +6. Delete or repurpose the machine according to the installation's declared infrastructure policy. + +Deleting one generic controller must not require application workflow changes. Legacy project-specific hosts should remain only until CI, promotion, and deployment no longer reference them. ## Hard rules -- Public repositories never receive access to the trusted self-hosted runner pool. +- Public repositories never receive direct access to the trusted self-hosted runner pool. - Every project publishes `scripts/ci/plan.json` and implements `./scripts/ci/run.sh --shard INDEX/TOTAL` in its own Docker-defined test environment. - `./scripts/ci/run.sh fast` and `full` remain aggregate developer commands; fleet scheduling expands their named tasks across available workers. - Every matrix job has a five-minute hard timeout, while expected test payload targets four minutes or less to reserve startup and reporting time. +- Application workflows submit all independent jobs; infrastructure configuration alone controls worker capacity. - CI runner pools and deployment host groups are separate trust roles. - Production deployment is manual and requires GitHub Environment approval. -- Reusable workflows and third-party actions are pinned to immutable commits. +- Controller engine revisions, reusable workflows, and third-party actions are pinned to immutable commits. - Configuration contains logical identifiers only. Secret values, private host details, and credentials never enter Git. - Promoted artifacts are container image digests; production does not rebuild a different image. -`fleet.schema.json` provides editor completion and structural documentation. `scripts/validate.py` is the authoritative dependency-free policy check, including relationships JSON Schema cannot express clearly. +`fleet.schema.json` provides editor completion and structural documentation. `scripts/validate.py` is the authoritative dependency-free policy check, including cross-object relationships JSON Schema cannot express clearly. ## Five-minute parallelism contract @@ -61,9 +126,9 @@ Projects divide their total test-minutes into independent named tasks and determ flowchart LR P[plan.json] --> M[GitHub matrix] M --> A[lint] - M --> B[unit 1/4] - M --> C[unit 2/4] - M --> D[integration 1/3] + M --> B[unit 1 of 4] + M --> C[unit 2 of 4] + M --> D[integration 1 of 3] M --> E[other independent shards] ``` @@ -73,21 +138,22 @@ Adding workers reduces wall-clock time only while independent shards remain queu | Path | Purpose | |---|---| -| `fleet.json` | Fictional, valid reference configuration to initialize or replace | +| `fleet.json` | Fictional, valid schema-v3 configuration with one controller | | `fleet.schema.json` | JSON Schema draft 2020-12 editor contract | -| `scripts/init.sh` | Safe first-project initializer | -| `scripts/validate.sh` | Structural, policy, and secret-boundary validation | +| `scripts/init.sh` | Safe first-project and first-controller initializer | +| `scripts/validate.sh` | Structural, relational, capacity, and secret-boundary validation | | `scripts/test_policy.py` | Regression tests proving unsafe configurations fail closed | -| `examples/multi-host/fleet.json` | Fictional two-project, multi-host topology | +| `examples/multi-host/fleet.json` | Fictional two-project, two-location controller topology | | `SECURITY.md` | Secret handling and vulnerability reporting | | `AGENTS.md` | Non-negotiable rules for humans and coding agents | ## Public and private boundary -| Safe in this public template | Belongs in the private config repo | Belongs only in a secret store | +| Safe in this public template | Belongs in the private config repo | Belongs only outside Git | |---|---|---| -| Schema, validator, fictional examples | Real repository names and logical host-group names | Tokens, passwords, private keys | -| Standard CI entrypoint names | Environment policy and allowed repository lists | Host addresses and SSH material | -| Reusable workflow references | Required secret **names** | `.env` contents and app credentials | +| Schema, validator, fictional examples | Real repository names and runner-group policy | Tokens, passwords, private keys | +| Standard CI entrypoint names | Logical controller IDs and locations | Host addresses, VM IDs, SSH material | +| Controller state and lifecycle vocabulary | Capacity budgets and per-runner limits | Rendered runtime configuration | +| Reusable engine interface | Required secret **names** | Secret values and application credentials | The public engine and this template use the [Unlicense](LICENSE). See [THIRD_PARTY_NOTICES.md](THIRD_PARTY_NOTICES.md) before copying third-party material into a derived repository. diff --git a/templates/config-repository/examples/multi-host/fleet.json b/templates/config-repository/examples/multi-host/fleet.json index 603ee21f..40b7c0d5 100644 --- a/templates/config-repository/examples/multi-host/fleet.json +++ b/templates/config-repository/examples/multi-host/fleet.json @@ -1,6 +1,6 @@ { "$schema": "../../fleet.schema.json", - "schema_version": 2, + "schema_version": 3, "organization": { "slug": "sample-company", "registry": "ghcr.io/sample-company", @@ -9,10 +9,42 @@ }, "runner_pools": { "trusted-ci": { + "runner_group": "sample-trusted-ci", "routing_labels": ["docker-ci"], "allowed_repositories": ["sample-company/api", "sample-company/web"], "public_repositories": false, - "max_concurrent_jobs": 4 + "capacity_budget": 6, + "job_submission_policy": "all-independent-jobs" + } + }, + "controllers": { + "sample-ci-primary": { + "pool": "trusted-ci", + "location": "primary-site", + "state": "active", + "scale_set_name": "sample-ci-primary", + "lifecycle": "stable", + "engine_ref": "3d118432dce86d702fa1b78ecadcfcee750bfd9a", + "min_runners": 0, + "max_runners": 4, + "runner_resources": { + "cpu_cores": 4, + "memory_mib": 8192 + } + }, + "sample-ci-remote": { + "pool": "trusted-ci", + "location": "remote-site", + "state": "active", + "scale_set_name": "sample-ci-remote", + "lifecycle": "stable", + "engine_ref": "3d118432dce86d702fa1b78ecadcfcee750bfd9a", + "min_runners": 0, + "max_runners": 2, + "runner_resources": { + "cpu_cores": 2, + "memory_mib": 4096 + } } }, "host_groups": { diff --git a/templates/config-repository/fleet.json b/templates/config-repository/fleet.json index 0a3177aa..c7788d99 100644 --- a/templates/config-repository/fleet.json +++ b/templates/config-repository/fleet.json @@ -1,6 +1,6 @@ { "$schema": "./fleet.schema.json", - "schema_version": 2, + "schema_version": 3, "organization": { "slug": "example-org", "registry": "ghcr.io/example-org", @@ -9,10 +9,28 @@ }, "runner_pools": { "trusted-ci": { + "runner_group": "example-trusted-ci", "routing_labels": ["docker-ci"], "allowed_repositories": ["example-org/example-app"], "public_repositories": false, - "max_concurrent_jobs": 1 + "capacity_budget": 1, + "job_submission_policy": "all-independent-jobs" + } + }, + "controllers": { + "example-ci-01": { + "pool": "trusted-ci", + "location": "example-site-a", + "state": "active", + "scale_set_name": "example-ci-01", + "lifecycle": "experimental", + "engine_ref": "3d118432dce86d702fa1b78ecadcfcee750bfd9a", + "min_runners": 0, + "max_runners": 1, + "runner_resources": { + "cpu_cores": 2, + "memory_mib": 4096 + } } }, "host_groups": { diff --git a/templates/config-repository/fleet.schema.json b/templates/config-repository/fleet.schema.json index 9dd012e8..feebece2 100644 --- a/templates/config-repository/fleet.schema.json +++ b/templates/config-repository/fleet.schema.json @@ -4,10 +4,10 @@ "title": "ci-fleet organization configuration", "type": "object", "additionalProperties": false, - "required": ["schema_version", "organization", "runner_pools", "host_groups", "environments", "projects"], + "required": ["schema_version", "organization", "runner_pools", "controllers", "host_groups", "environments", "projects"], "properties": { "$schema": {"type": "string"}, - "schema_version": {"const": 2}, + "schema_version": {"const": 3}, "organization": { "type": "object", "additionalProperties": false, @@ -22,34 +22,73 @@ "runner_pools": { "type": "object", "minProperties": 1, + "propertyNames": {"$ref": "#/$defs/slug"}, "additionalProperties": {"$ref": "#/$defs/runner_pool"} }, + "controllers": { + "type": "object", + "minProperties": 1, + "propertyNames": {"$ref": "#/$defs/slug"}, + "additionalProperties": {"$ref": "#/$defs/controller"} + }, "host_groups": { "type": "object", "minProperties": 1, + "propertyNames": {"$ref": "#/$defs/slug"}, "additionalProperties": {"$ref": "#/$defs/host_group"} }, "environments": { "type": "object", "minProperties": 1, + "propertyNames": {"$ref": "#/$defs/slug"}, "additionalProperties": {"$ref": "#/$defs/environment"} }, "projects": { "type": "object", "minProperties": 1, + "propertyNames": {"$ref": "#/$defs/slug"}, "additionalProperties": {"$ref": "#/$defs/project"} } }, "$defs": { + "slug": {"type": "string", "pattern": "^[a-z0-9][a-z0-9-]{0,62}$"}, + "repository": {"type": "string", "pattern": "^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$"}, "runner_pool": { "type": "object", "additionalProperties": false, - "required": ["routing_labels", "allowed_repositories", "public_repositories", "max_concurrent_jobs"], + "required": ["runner_group", "routing_labels", "allowed_repositories", "public_repositories", "capacity_budget", "job_submission_policy"], "properties": { - "routing_labels": {"type": "array", "minItems": 1, "uniqueItems": true, "items": {"type": "string", "pattern": "^[a-z0-9][a-z0-9-]{0,62}$"}}, - "allowed_repositories": {"type": "array", "minItems": 1, "uniqueItems": true, "items": {"type": "string", "pattern": "^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$"}}, + "runner_group": {"$ref": "#/$defs/slug"}, + "routing_labels": {"type": "array", "minItems": 1, "uniqueItems": true, "items": {"$ref": "#/$defs/slug"}}, + "allowed_repositories": {"type": "array", "minItems": 1, "uniqueItems": true, "items": {"$ref": "#/$defs/repository"}}, "public_repositories": {"const": false}, - "max_concurrent_jobs": {"type": "integer", "minimum": 1} + "capacity_budget": {"type": "integer", "minimum": 1}, + "job_submission_policy": {"const": "all-independent-jobs"} + } + }, + "controller": { + "type": "object", + "additionalProperties": false, + "required": ["pool", "location", "state", "scale_set_name", "lifecycle", "engine_ref", "min_runners", "max_runners", "runner_resources"], + "properties": { + "pool": {"$ref": "#/$defs/slug"}, + "location": {"$ref": "#/$defs/slug"}, + "state": {"enum": ["active", "drained", "disabled"]}, + "scale_set_name": {"$ref": "#/$defs/slug"}, + "lifecycle": {"enum": ["experimental", "stable", "retiring"]}, + "engine_ref": {"type": "string", "pattern": "^[0-9a-f]{40}$"}, + "min_runners": {"type": "integer", "minimum": 0}, + "max_runners": {"type": "integer", "minimum": 1}, + "runner_resources": {"$ref": "#/$defs/runner_resources"} + } + }, + "runner_resources": { + "type": "object", + "additionalProperties": false, + "required": ["cpu_cores", "memory_mib"], + "properties": { + "cpu_cores": {"type": "integer", "minimum": 1}, + "memory_mib": {"type": "integer", "minimum": 512} } }, "host_group": { @@ -77,7 +116,7 @@ "additionalProperties": false, "required": ["repository", "image", "ci_pool", "ci_contract", "deployments"], "properties": { - "repository": {"type": "string", "pattern": "^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$"}, + "repository": {"$ref": "#/$defs/repository"}, "image": {"type": "string", "pattern": "^[a-z0-9.-]+/[a-z0-9._/-]+$"}, "ci_pool": {"type": "string"}, "ci_contract": { diff --git a/templates/config-repository/scripts/init.py b/templates/config-repository/scripts/init.py index 0a761d61..7790b0fa 100755 --- a/templates/config-repository/scripts/init.py +++ b/templates/config-repository/scripts/init.py @@ -12,7 +12,16 @@ ROOT = Path(__file__).resolve().parents[1] ORG_SLUG = re.compile(r"^[a-z0-9][a-z0-9-]{0,38}$") -PROJECT_SLUG = re.compile(r"^[a-z0-9][a-z0-9-]{0,62}$") +SLUG = re.compile(r"^[a-z0-9][a-z0-9-]{0,62}$") +COMMIT_SHA = re.compile(r"^[0-9a-f]{40}$") +DEFAULT_ENGINE_REF = "3d118432dce86d702fa1b78ecadcfcee750bfd9a" + + +def positive_integer(value: str) -> int: + parsed = int(value) + if parsed < 1: + raise argparse.ArgumentTypeError("must be a positive integer") + return parsed def parse_args() -> argparse.Namespace: @@ -22,6 +31,14 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--repository", help="owner/repository; defaults to ORGANIZATION/PROJECT") parser.add_argument("--registry", help="registry namespace; defaults to ghcr.io/ORGANIZATION") parser.add_argument("--runner-label", default="docker-ci", help="capability label for the shared CI pool") + parser.add_argument("--runner-group", default="trusted-ci", help="logical GitHub runner-group slug") + parser.add_argument("--controller", default="ci-01", help="logical controller ID and scale-set name") + parser.add_argument("--location", default="primary-site", help="logical location slug; never an address") + parser.add_argument("--capacity-budget", type=positive_integer, default=1, help="maximum capacity reserved by the pool") + parser.add_argument("--max-runners", type=positive_integer, default=1, help="initial controller maximum") + parser.add_argument("--runner-cpu-cores", type=positive_integer, default=2, help="CPU cores available to each runner") + parser.add_argument("--runner-memory-mib", type=positive_integer, default=4096, help="memory available to each runner") + parser.add_argument("--engine-ref", default=DEFAULT_ENGINE_REF, help="reviewed full ci-fleet commit SHA") parser.add_argument("--output", type=Path, default=ROOT / "fleet.json", help="output configuration path") parser.add_argument("--force", action="store_true", help="replace an existing non-example output file") return parser.parse_args() @@ -35,10 +52,21 @@ def main() -> int: args = parse_args() if not ORG_SLUG.fullmatch(args.organization): fail("--organization must be a lowercase GitHub organization slug") - if not PROJECT_SLUG.fullmatch(args.project): - fail("--project must be a lowercase slug") - if not PROJECT_SLUG.fullmatch(args.runner_label): - fail("--runner-label must be a lowercase slug") + for option, value in { + "--project": args.project, + "--runner-label": args.runner_label, + "--runner-group": args.runner_group, + "--controller": args.controller, + "--location": args.location, + }.items(): + if not SLUG.fullmatch(value): + fail(f"{option} must be a lowercase slug") + if not COMMIT_SHA.fullmatch(args.engine_ref) or args.engine_ref == "0" * 40: + fail("--engine-ref must be a nonzero full lowercase commit SHA") + if args.max_runners > args.capacity_budget: + fail("--max-runners must not exceed --capacity-budget") + if args.runner_memory_mib < 512: + fail("--runner-memory-mib must be at least 512") repository = args.repository or f"{args.organization}/{args.project}" registry = (args.registry or f"ghcr.io/{args.organization}").rstrip("/") @@ -54,7 +82,7 @@ def main() -> int: config = { "$schema": str((ROOT / "fleet.schema.json").resolve()) if output.parent != ROOT else "./fleet.schema.json", - "schema_version": 2, + "schema_version": 3, "organization": { "slug": args.organization, "registry": registry, @@ -63,10 +91,28 @@ def main() -> int: }, "runner_pools": { "trusted-ci": { + "runner_group": args.runner_group, "routing_labels": [args.runner_label], "allowed_repositories": [repository], "public_repositories": False, - "max_concurrent_jobs": 1, + "capacity_budget": args.capacity_budget, + "job_submission_policy": "all-independent-jobs", + } + }, + "controllers": { + args.controller: { + "pool": "trusted-ci", + "location": args.location, + "state": "active", + "scale_set_name": args.controller, + "lifecycle": "experimental", + "engine_ref": args.engine_ref, + "min_runners": 0, + "max_runners": args.max_runners, + "runner_resources": { + "cpu_cores": args.runner_cpu_cores, + "memory_mib": args.runner_memory_mib, + }, } }, "host_groups": { @@ -115,7 +161,7 @@ def main() -> int: check=True, ) print(f"Initialized {output}") - print("Next: edit logical mappings, configure GitHub Environments, and keep every secret value outside Git.") + print("Next: review controller capacity, configure GitHub policy, and keep every secret value outside Git.") return 0 diff --git a/templates/config-repository/scripts/test_policy.py b/templates/config-repository/scripts/test_policy.py index 630b519b..e42d9f9a 100755 --- a/templates/config-repository/scripts/test_policy.py +++ b/templates/config-repository/scripts/test_policy.py @@ -5,10 +5,11 @@ import copy import json +import tempfile import unittest from pathlib import Path -from validate import Validation, scan_secret_material, validate_config +from validate import Validation, load_json, scan_secret_material, validate_config ROOT = Path(__file__).resolve().parents[1] @@ -29,6 +30,10 @@ def first_project(config: dict) -> dict: return next(iter(config["projects"].values())) +def first_controller(config: dict) -> dict: + return next(iter(config["controllers"].values())) + + class PolicyTests(unittest.TestCase): def assert_rejected(self, config: dict, expected: str, *, strict: bool = False) -> None: errors = errors_for(config, strict=strict) @@ -37,11 +42,81 @@ def assert_rejected(self, config: dict, expected: str, *, strict: bool = False) def test_reference_configuration_is_valid(self) -> None: self.assertEqual(errors_for(reference_config()), []) + def test_multi_host_multi_location_configuration_is_valid(self) -> None: + config = json.loads((ROOT / "examples" / "multi-host" / "fleet.json").read_text(encoding="utf-8")) + self.assertEqual(errors_for(config), []) + + def test_schema_version_two_is_rejected(self) -> None: + config = copy.deepcopy(reference_config()) + config["schema_version"] = 2 + self.assert_rejected(config, "must equal 3") + def test_public_repository_access_is_rejected(self) -> None: config = copy.deepcopy(reference_config()) config["runner_pools"]["trusted-ci"]["public_repositories"] = True self.assert_rejected(config, "trusted private repositories") + def test_capacity_overcommit_is_rejected(self) -> None: + config = copy.deepcopy(reference_config()) + first_controller(config)["max_runners"] = 2 + self.assert_rejected(config, "must cover 2 runners") + + def test_drained_capacity_remains_reserved(self) -> None: + config = copy.deepcopy(reference_config()) + first_controller(config)["state"] = "drained" + first_controller(config)["max_runners"] = 2 + self.assert_rejected(config, "must cover 2 runners") + + def test_disabled_capacity_is_not_reserved(self) -> None: + config = copy.deepcopy(reference_config()) + first_controller(config)["state"] = "disabled" + first_controller(config)["max_runners"] = 100 + self.assertEqual(errors_for(config), []) + + def test_duplicate_scale_set_is_rejected(self) -> None: + config = copy.deepcopy(reference_config()) + duplicate = copy.deepcopy(first_controller(config)) + duplicate["location"] = "example-site-b" + config["controllers"]["example-ci-02"] = duplicate + config["runner_pools"]["trusted-ci"]["capacity_budget"] = 2 + self.assert_rejected(config, "scale_set_name: must be unique") + + def test_controller_pool_must_exist(self) -> None: + config = copy.deepcopy(reference_config()) + first_controller(config)["pool"] = "missing-pool" + self.assert_rejected(config, "must reference a declared runner pool") + + def test_controller_address_is_rejected(self) -> None: + config = copy.deepcopy(reference_config()) + first_controller(config)["ip_address"] = "192.0.2.10" + self.assert_rejected(config, "host-local infrastructure details are forbidden") + + def test_unpinned_engine_ref_is_rejected(self) -> None: + config = copy.deepcopy(reference_config()) + first_controller(config)["engine_ref"] = "main" + self.assert_rejected(config, "full lowercase commit SHA") + + def test_zero_engine_ref_is_rejected(self) -> None: + config = copy.deepcopy(reference_config()) + first_controller(config)["engine_ref"] = "0" * 40 + self.assert_rejected(config, "nonzero full lowercase commit SHA") + + def test_drained_controller_minimum_must_be_zero(self) -> None: + config = copy.deepcopy(reference_config()) + first_controller(config)["state"] = "drained" + first_controller(config)["min_runners"] = 1 + self.assert_rejected(config, "must be zero while drained or disabled") + + def test_application_capacity_control_is_rejected(self) -> None: + config = copy.deepcopy(reference_config()) + first_project(config)["ci_contract"]["max_parallel"] = 1 + self.assert_rejected(config, "unknown keys: max_parallel") + + def test_pool_must_submit_all_independent_jobs(self) -> None: + config = copy.deepcopy(reference_config()) + config["runner_pools"]["trusted-ci"]["job_submission_policy"] = "max-parallel" + self.assert_rejected(config, "leave capacity control to infrastructure") + def test_automatic_production_is_rejected(self) -> None: config = copy.deepcopy(reference_config()) config["environments"]["production"]["automatic"] = True @@ -67,6 +142,14 @@ def test_secret_value_key_is_rejected(self) -> None: config["environments"]["development"]["token"] = "not-a-real-token" self.assert_rejected(config, "secret values are forbidden") + def test_duplicate_json_controller_id_is_rejected(self) -> None: + validation = Validation() + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "duplicate.json" + path.write_text('{"controllers":{"ci-01":{},"ci-01":{}}}', encoding="utf-8") + self.assertIsNone(load_json(path, validation)) + self.assertTrue(any("duplicate object key: ci-01" in error for error in validation.errors), validation.errors) + def test_strict_mode_rejects_unchanged_example(self) -> None: config = copy.deepcopy(reference_config()) project = first_project(config) diff --git a/templates/config-repository/scripts/validate.py b/templates/config-repository/scripts/validate.py index f644f657..db56e621 100755 --- a/templates/config-repository/scripts/validate.py +++ b/templates/config-repository/scripts/validate.py @@ -17,6 +17,7 @@ REPOSITORY = re.compile(r"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$") IMAGE = re.compile(r"^[a-z0-9.-]+/[a-z0-9._/-]+$") SECRET_NAME = re.compile(r"^[A-Z][A-Z0-9_]*$") +COMMIT_SHA = re.compile(r"^[0-9a-f]{40}$") HIGH_CONFIDENCE_SECRET_PATTERNS = ( re.compile(r"-----BEGIN [A-Z ]*PRIVATE KEY-----"), re.compile(r"github_pat_[A-Za-z0-9_]{20,}"), @@ -36,6 +37,22 @@ "secret_value", "token", } +FORBIDDEN_INFRASTRUCTURE_KEYS = { + "backup_id", + "backup_snapshot", + "backup_storage", + "disk_storage", + "host_address", + "hostname", + "ip", + "ip_address", + "proxmox_vmid", + "ssh_host", + "ssh_password", + "ssh_private_key", + "vm_id", + "vmid", +} FORBIDDEN_FILENAMES = re.compile(r"(?:^|/)\.env(?:\..+)?$|\.(?:key|pem|p12|pfx)$", re.IGNORECASE) FORBIDDEN_DIRECTORIES = {"credentials", "private", "secrets"} @@ -64,12 +81,22 @@ def exact_keys(self, value: Any, path: str, required: set[str], optional: set[st def load_json(path: Path, validation: Validation) -> Any: + def reject_duplicate_keys(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + value: dict[str, Any] = {} + for key, child in pairs: + if key in value: + raise ValueError(f"duplicate object key: {key}") + value[key] = child + return value + try: - return json.loads(path.read_text(encoding="utf-8")) + return json.loads(path.read_text(encoding="utf-8"), object_pairs_hook=reject_duplicate_keys) except FileNotFoundError: validation.errors.append(f"{path}: file not found") except json.JSONDecodeError as exc: validation.errors.append(f"{path}:{exc.lineno}:{exc.colno}: invalid JSON: {exc.msg}") + except ValueError as exc: + validation.errors.append(f"{path}: invalid JSON: {exc}") return None @@ -88,10 +115,15 @@ def scan_secret_material(config: Any, validation: Validation) -> None: def scan_keys(value: Any, path: str = "$") -> None: if isinstance(value, dict): for key, child in value.items(): - if key.lower() in FORBIDDEN_SECRET_KEYS: + normalized = key.lower() + if normalized in FORBIDDEN_SECRET_KEYS: validation.errors.append( f"{path}.{key}: secret values are forbidden; store only an uppercase secret name" ) + if normalized in FORBIDDEN_INFRASTRUCTURE_KEYS: + validation.errors.append( + f"{path}.{key}: host-local infrastructure details are forbidden in Git-authored policy" + ) scan_keys(child, f"{path}.{key}") elif isinstance(value, list): for index, child in enumerate(value): @@ -121,11 +153,19 @@ def scan_forbidden_paths(repo_root: Path, validation: Validation) -> None: def validate_config(config: Any, validation: Validation, strict: bool) -> None: - required_top = {"schema_version", "organization", "runner_pools", "host_groups", "environments", "projects"} + required_top = { + "schema_version", + "organization", + "runner_pools", + "controllers", + "host_groups", + "environments", + "projects", + } if not validation.exact_keys(config, "$", required_top, {"$schema"}): return - validation.require(config.get("schema_version") == 2, "$.schema_version", "must equal 2") + validation.require(config.get("schema_version") == 3, "$.schema_version", "must equal 3") organization = config.get("organization") organization_keys = {"slug", "registry", "delivery_engine", "workflow_ref_policy"} @@ -144,13 +184,24 @@ def validate_config(config: Any, validation: Validation, strict: bool) -> None: if not isinstance(pools, dict) or not pools: validation.errors.append("$.runner_pools: must be a non-empty object") pools = {} + pool_capacity: dict[str, int] = {} for name, pool in pools.items(): path = f"$.runner_pools.{name}" - validation.require(bool(SLUG.fullmatch(name)), path, "pool name must be a lowercase slug") - if not validation.exact_keys(pool, path, {"routing_labels", "allowed_repositories", "public_repositories", "max_concurrent_jobs"}): + validation.require(isinstance(name, str) and bool(SLUG.fullmatch(name)), path, "pool name must be a lowercase slug") + pool_keys = { + "runner_group", + "routing_labels", + "allowed_repositories", + "public_repositories", + "capacity_budget", + "job_submission_policy", + } + if not validation.exact_keys(pool, path, pool_keys): continue + runner_group = pool.get("runner_group") labels = pool.get("routing_labels") repos = pool.get("allowed_repositories") + validation.require(isinstance(runner_group, str) and bool(SLUG.fullmatch(runner_group)), f"{path}.runner_group", "must be a lowercase logical runner-group slug") validation.require(isinstance(labels, list) and bool(labels), f"{path}.routing_labels", "must be a non-empty list") if isinstance(labels, list): validation.require(len(labels) == len(set(labels)), f"{path}.routing_labels", "must contain unique labels") @@ -163,8 +214,72 @@ def validate_config(config: Any, validation: Validation, strict: bool) -> None: for index, repository in enumerate(repos): validation.require(isinstance(repository, str) and bool(REPOSITORY.fullmatch(repository)), f"{path}.allowed_repositories[{index}]", "must be owner/repository") validation.require(pool.get("public_repositories") is False, f"{path}.public_repositories", "must be false; this fleet is for trusted private repositories") - jobs = pool.get("max_concurrent_jobs") - validation.require(type(jobs) is int and jobs > 0, f"{path}.max_concurrent_jobs", "must be a positive integer") + budget = pool.get("capacity_budget") + validation.require(type(budget) is int and budget > 0, f"{path}.capacity_budget", "must be a positive infrastructure capacity budget") + if type(budget) is int and budget > 0: + pool_capacity[name] = budget + validation.require(pool.get("job_submission_policy") == "all-independent-jobs", f"{path}.job_submission_policy", "must submit every independent job and leave capacity control to infrastructure") + + controllers = config.get("controllers") + if not isinstance(controllers, dict) or not controllers: + validation.errors.append("$.controllers: must be a non-empty object") + controllers = {} + reserved_capacity = {name: 0 for name in pools} + scale_sets: dict[str, str] = {} + controller_keys = { + "pool", + "location", + "state", + "scale_set_name", + "lifecycle", + "engine_ref", + "min_runners", + "max_runners", + "runner_resources", + } + for name, controller in controllers.items(): + path = f"$.controllers.{name}" + validation.require(isinstance(name, str) and bool(SLUG.fullmatch(name)), path, "controller ID must be a unique lowercase slug") + if not validation.exact_keys(controller, path, controller_keys): + continue + pool_name = controller.get("pool") + location = controller.get("location") + state = controller.get("state") + scale_set = controller.get("scale_set_name") + lifecycle = controller.get("lifecycle") + engine_ref = controller.get("engine_ref") + minimum = controller.get("min_runners") + maximum = controller.get("max_runners") + validation.require(pool_name in pools, f"{path}.pool", "must reference a declared runner pool") + validation.require(isinstance(location, str) and bool(SLUG.fullmatch(location)), f"{path}.location", "must be a logical location slug, never an address") + validation.require(state in {"active", "drained", "disabled"}, f"{path}.state", "must be active, drained, or disabled") + validation.require(isinstance(scale_set, str) and bool(SLUG.fullmatch(scale_set)), f"{path}.scale_set_name", "must be a lowercase scale-set slug") + if isinstance(scale_set, str): + if scale_set in scale_sets: + validation.errors.append(f"{path}.scale_set_name: must be unique; also used by {scale_sets[scale_set]}") + else: + scale_sets[scale_set] = name + validation.require(lifecycle in {"experimental", "stable", "retiring"}, f"{path}.lifecycle", "must be experimental, stable, or retiring") + validation.require(isinstance(engine_ref, str) and bool(COMMIT_SHA.fullmatch(engine_ref)) and engine_ref != "0" * 40, f"{path}.engine_ref", "must be a nonzero full lowercase commit SHA") + validation.require(type(minimum) is int and minimum >= 0, f"{path}.min_runners", "must be a non-negative integer") + validation.require(type(maximum) is int and maximum > 0, f"{path}.max_runners", "must be a positive integer") + if type(minimum) is int and type(maximum) is int: + validation.require(minimum <= maximum, f"{path}.min_runners", "must not exceed max_runners") + if state in {"drained", "disabled"}: + validation.require(minimum == 0, f"{path}.min_runners", "must be zero while drained or disabled") + resources = controller.get("runner_resources") + if validation.exact_keys(resources, f"{path}.runner_resources", {"cpu_cores", "memory_mib"}): + cpu = resources.get("cpu_cores") + memory = resources.get("memory_mib") + validation.require(type(cpu) is int and cpu > 0, f"{path}.runner_resources.cpu_cores", "must be a positive integer") + validation.require(type(memory) is int and memory >= 512, f"{path}.runner_resources.memory_mib", "must be at least 512 MiB") + if pool_name in pools and state != "disabled" and type(maximum) is int and maximum > 0: + reserved_capacity[pool_name] += maximum + + for name, reserved in reserved_capacity.items(): + budget = pool_capacity.get(name) + if budget is not None: + validation.require(reserved <= budget, f"$.runner_pools.{name}.capacity_budget", f"must cover {reserved} runners reserved by active or drained controllers") groups = config.get("host_groups") if not isinstance(groups, dict) or not groups: From c60016535647a7fb8b0eb59a0c1f4cbbe3bf8924 Mon Sep 17 00:00:00 2001 From: Nick Foster Date: Sat, 18 Jul 2026 17:22:13 -0500 Subject: [PATCH 02/57] Fix installer drift check lint --- scripts/install-worker-controller.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/install-worker-controller.sh b/scripts/install-worker-controller.sh index f0241932..87c2c8f1 100755 --- a/scripts/install-worker-controller.sh +++ b/scripts/install-worker-controller.sh @@ -258,7 +258,10 @@ systemd_matches() { drift_count() { local count=0 - [[ -f "$rendered_env" ]] && cmp -s "$candidate_env" "$rendered_env" || { note 'DRIFT rendered_environment'; count=$((count + 1)); } + if [[ ! -f "$rendered_env" ]] || ! cmp -s "$candidate_env" "$rendered_env"; then + note 'DRIFT rendered_environment' + count=$((count + 1)) + fi release_matches || { note 'DRIFT engine_release'; count=$((count + 1)); } state_matches || { note 'DRIFT install_state'; count=$((count + 1)); } runtime_matches "$target_state" || { note 'DRIFT controller_runtime'; count=$((count + 1)); } From 3a09609f9599d465ad8bb3918de3d4791dca1b6f Mon Sep 17 00:00:00 2001 From: Nickfosts Hermes Date: Sat, 18 Jul 2026 18:18:39 -0500 Subject: [PATCH 03/57] fix(installer): restore complete checkpoints safely --- .gitignore | 26 +++ docs/DESIRED-STATE.md | 2 +- scripts/install-worker-controller.sh | 200 ++++++++++++++++------ scripts/test-install-worker-controller.sh | 6 + 4 files changed, 178 insertions(+), 56 deletions(-) diff --git a/.gitignore b/.gitignore index 7a60b85e..5dff1bc2 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,28 @@ +# Local deployment configuration +.env +.env.* +!.env.example +*.local +secrets/ +private/ +inventory/ + +# Credentials and keys +*.pem +*.key +*.p12 +*.pfx + +# Runtime state +_work/ +runner-data/ +logs/ +tmp/ __pycache__/ *.pyc + +# Editors and operating systems +.vscode/ +.idea/ +.DS_Store +Thumbs.db diff --git a/docs/DESIRED-STATE.md b/docs/DESIRED-STATE.md index f102fe43..34f83727 100644 --- a/docs/DESIRED-STATE.md +++ b/docs/DESIRED-STATE.md @@ -146,7 +146,7 @@ Legacy project-specific hosts remain until CI, promotion, and deployment no long ## Failure and recovery behavior -Before mutation, the installer records the prior rendered environment, installation metadata, and release target under `/var/lib/ci-fleet/checkpoints`. Build and validation happen before the active release changes. A failed activation or health check restores the latest known-good controller checkpoint. +Before mutation, the installer records the prior rendered environment, installation metadata, runtime release, installer-manager release, and maintenance unit/timer state under `/var/lib/ci-fleet/checkpoints`. Build and validation happen before the active release changes. A failed activation or health check drains the candidate, restores those artifacts, restarts the prior controller only when no managed runner is active, and verifies prior-release health before reporting rollback success. A host-local `flock` prevents overlapping installer and drift-check processes. These controller checkpoints do not replace machine backups. Operators still create and verify VM snapshots, physical-host recovery media, or equivalent infrastructure backups according to their local policy. diff --git a/scripts/install-worker-controller.sh b/scripts/install-worker-controller.sh index 87c2c8f1..ff76b115 100755 --- a/scripts/install-worker-controller.sh +++ b/scripts/install-worker-controller.sh @@ -86,6 +86,7 @@ if [[ "$testing" != 1 && ${EUID:-$(id -u)} -ne 0 ]]; then fi root_path() { printf '%s%s' "$root_prefix" "$1"; } +is_git_checkout() { git -C "$1" rev-parse --is-inside-work-tree >/dev/null 2>&1; } install_root=$(root_path /opt/ci-fleet) releases_dir=$install_root/releases @@ -101,6 +102,13 @@ state_root=$(root_path /var/lib/ci-fleet) state_file=$state_root/install-state.json checkpoints_dir=$state_root/checkpoints systemd_dir=$(root_path /etc/systemd/system) +lock_file=$(root_path /run/ci-fleet-installer.lock) +unit_names=( + ci-fleet-health.service ci-fleet-health.timer + ci-fleet-cleanup.service ci-fleet-cleanup.timer + ci-fleet-drift.service ci-fleet-drift.timer +) +timer_names=(ci-fleet-health.timer ci-fleet-cleanup.timer ci-fleet-drift.timer) temporary=$(mktemp -d) cleanup_temporary() { rm -rf "$temporary"; } @@ -108,7 +116,7 @@ trap cleanup_temporary EXIT require_commands() { local command - for command in git python3 docker jq tar install cmp readlink systemctl stat awk grep date; do + for command in git python3 docker jq tar install cmp readlink systemctl stat awk grep date flock; do command -v "$command" >/dev/null || die "$command is required" done docker info >/dev/null 2>&1 || die 'Docker daemon is unavailable' @@ -127,7 +135,7 @@ validate_common_arguments() { resolve_config() { local resolved checkout candidate_config=$temporary/fleet.json - if [[ -d "$config_repo/.git" ]]; then + if is_git_checkout "$config_repo"; then resolved=$(git -C "$config_repo" rev-parse "$config_ref^{commit}" 2>/dev/null || true) [[ "$resolved" == "$config_ref" ]] || die 'local configuration repository does not contain the requested commit' git -C "$config_repo" show "$config_ref:fleet.json" >"$candidate_config" || die 'fleet.json is absent at the requested configuration commit' @@ -270,16 +278,16 @@ drift_count() { } install_release() { - local archive marker_commit checkout resolved + local archive marker_commit checkout resolved staged_release if [[ -d "$release_dir" ]]; then [[ -f "$release_dir/.ci-fleet-engine-ref" ]] || die "existing release has no engine marker: $release_dir" marker_commit=$(<"$release_dir/.ci-fleet-engine-ref") [[ "$marker_commit" == "$engine_ref" ]] || die "existing release marker does not match $engine_ref" return fi - install -d -m 0755 "$releases_dir" "$release_dir" + install -d -m 0755 "$releases_dir" archive=$temporary/engine.tar - if [[ -d "$repo_root/.git" && $(git -C "$repo_root" rev-parse 'HEAD^{commit}') == "$engine_ref" ]]; then + if is_git_checkout "$repo_root" && [[ $(git -C "$repo_root" rev-parse 'HEAD^{commit}') == "$engine_ref" ]]; then git -C "$repo_root" archive --format=tar --output "$archive" HEAD else [[ "$engine_repository" =~ ^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$ ]] || die 'delivery engine repository is invalid' @@ -291,29 +299,35 @@ install_release() { [[ "$resolved" == "$engine_ref" ]] || die 'fetched ci-fleet engine commit does not match desired state' git -C "$checkout" archive --format=tar --output "$archive" FETCH_HEAD fi - tar -xf "$archive" -C "$release_dir" - printf '%s\n' "$engine_ref" >"$temporary/engine-ref" - install -m 0644 "$temporary/engine-ref" "$release_dir/.ci-fleet-engine-ref" + staged_release=$temporary/engine-release + install -d -m 0755 "$staged_release" + tar -xf "$archive" -C "$staged_release" + printf '%s\n' "$engine_ref" >"$staged_release/.ci-fleet-engine-ref" + chmod 0644 "$staged_release/.ci-fleet-engine-ref" + mv "$staged_release" "$release_dir" } install_manager() { - local manager_commit manager_release archive marker + local manager_commit manager_release archive marker staged_manager if [[ -f "$repo_root/.ci-fleet-engine-ref" ]]; then manager_commit=$(<"$repo_root/.ci-fleet-engine-ref") else - [[ -d "$repo_root/.git" ]] || die 'installer manager source is not a Git checkout or installed manager release' + is_git_checkout "$repo_root" || die 'installer manager source is not a Git checkout or installed manager release' manager_commit=$(git -C "$repo_root" rev-parse 'HEAD^{commit}') fi [[ "$manager_commit" =~ ^[0-9a-f]{40}$ ]] || die 'installer manager commit is invalid' manager_release=$manager_releases/$manager_commit if [[ ! -d "$manager_release" ]]; then - [[ -d "$repo_root/.git" ]] || die 'a new installer manager release requires a Git checkout' - install -d -m 0755 "$manager_releases" "$manager_release" + is_git_checkout "$repo_root" || die 'a new installer manager release requires a Git checkout' + install -d -m 0755 "$manager_releases" archive=$temporary/manager.tar git -C "$repo_root" archive --format=tar --output "$archive" HEAD - tar -xf "$archive" -C "$manager_release" - printf '%s\n' "$manager_commit" >"$temporary/manager-ref" - install -m 0644 "$temporary/manager-ref" "$manager_release/.ci-fleet-engine-ref" + staged_manager=$temporary/manager-release + install -d -m 0755 "$staged_manager" + tar -xf "$archive" -C "$staged_manager" + printf '%s\n' "$manager_commit" >"$staged_manager/.ci-fleet-engine-ref" + chmod 0644 "$staged_manager/.ci-fleet-engine-ref" + mv "$staged_manager" "$manager_release" else marker=$(<"$manager_release/.ci-fleet-engine-ref") [[ "$marker" == "$manager_commit" ]] || die 'existing installer manager marker is inconsistent' @@ -340,10 +354,10 @@ build_candidate() { } make_checkpoint() { - local timestamp target + local timestamp target unit timer timestamp=$(date -u +%Y%m%dT%H%M%SZ) checkpoint_dir=$checkpoints_dir/${timestamp}-$$ - install -d -m 0700 "$checkpoint_dir" + install -d -m 0700 "$checkpoint_dir" "$checkpoint_dir/systemd" [[ ! -f "$rendered_env" ]] || install -m 0600 "$rendered_env" "$checkpoint_dir/ci-fleet.env" [[ ! -f "$state_file" ]] || install -m 0600 "$state_file" "$checkpoint_dir/install-state.json" target=$(current_runtime_release) @@ -351,43 +365,71 @@ make_checkpoint() { printf '%s\n' "$target" >"$temporary/release-target" install -m 0600 "$temporary/release-target" "$checkpoint_dir/release-target" fi + if [[ -L "$manager_current" ]]; then + target=$(readlink -f "$manager_current") + printf '%s\n' "$target" >"$temporary/manager-target" + install -m 0600 "$temporary/manager-target" "$checkpoint_dir/manager-target" + fi + for unit in "${unit_names[@]}"; do + [[ ! -f "$systemd_dir/$unit" ]] || install -m 0644 "$systemd_dir/$unit" "$checkpoint_dir/systemd/$unit" + done + : >"$temporary/enabled-timers" + : >"$temporary/active-timers" + for timer in "${timer_names[@]}"; do + systemctl is-enabled --quiet "$timer" 2>/dev/null && printf '%s\n' "$timer" >>"$temporary/enabled-timers" || true + systemctl is-active --quiet "$timer" 2>/dev/null && printf '%s\n' "$timer" >>"$temporary/active-timers" || true + done + install -m 0600 "$temporary/enabled-timers" "$checkpoint_dir/enabled-timers" + install -m 0600 "$temporary/active-timers" "$checkpoint_dir/active-timers" note "CHECKPOINT_CREATED path=$checkpoint_dir" } -drain_current() { +try_drain_current() { local deadline count old_release status + drain_error= status=$(controller_status) [[ "$status" == running ]] || return 0 - [[ -f "$rendered_env" ]] || die 'cannot safely drain a running controller without its rendered environment' + if [[ ! -f "$rendered_env" ]]; then drain_error='cannot safely drain a running controller without its rendered environment'; return 1; fi old_release=$(current_runtime_release) - [[ -n "$old_release" && -f "$old_release/deploy/compose.yaml" ]] || die 'cannot locate the running controller Compose release for safe adoption' - compose "$old_release" "$rendered_env" pause controller >/dev/null + if [[ -z "$old_release" || ! -f "$old_release/deploy/compose.yaml" ]]; then drain_error='cannot locate the running controller Compose release for safe adoption'; return 1; fi + if ! compose "$old_release" "$rendered_env" pause controller >/dev/null; then drain_error='could not pause the controller for drain'; return 1; fi deadline=$((SECONDS + ${CI_FLEET_DRAIN_TIMEOUT_SECONDS:-300})) while :; do count=$(managed_runner_count) - if [[ "$count" == 0 ]]; then - break - fi + if [[ "$count" == 0 ]]; then break; fi if ((SECONDS >= deadline)); then compose "$old_release" "$rendered_env" unpause controller >/dev/null || true - die "drain timed out with $count managed runner(s) still present" + drain_error="drain timed out with $count managed runner(s) still present" + return 1 fi sleep 2 done - compose "$old_release" "$rendered_env" stop controller >/dev/null + if ! compose "$old_release" "$rendered_env" stop controller >/dev/null; then drain_error='could not stop the drained controller'; return 1; fi note 'DRAIN_OK managed_runners=0' } +drain_current() { + try_drain_current || die "$drain_error" +} + install_systemd_units() { + local source=${1:-$repo_root} install -d -m 0755 "$systemd_dir" - install -m 0644 "$repo_root/host/systemd/ci-fleet-health.service" "$systemd_dir/" - install -m 0644 "$repo_root/host/systemd/ci-fleet-health.timer" "$systemd_dir/" - install -m 0644 "$repo_root/host/systemd/ci-fleet-cleanup.service" "$systemd_dir/" - install -m 0644 "$repo_root/host/systemd/ci-fleet-cleanup.timer" "$systemd_dir/" - install -m 0644 "$repo_root/host/systemd/ci-fleet-drift.service" "$systemd_dir/" - install -m 0644 "$repo_root/host/systemd/ci-fleet-drift.timer" "$systemd_dir/" + install -m 0644 "$source/host/systemd/ci-fleet-health.service" "$systemd_dir/" + install -m 0644 "$source/host/systemd/ci-fleet-health.timer" "$systemd_dir/" + install -m 0644 "$source/host/systemd/ci-fleet-cleanup.service" "$systemd_dir/" + install -m 0644 "$source/host/systemd/ci-fleet-cleanup.timer" "$systemd_dir/" + install -m 0644 "$source/host/systemd/ci-fleet-drift.service" "$systemd_dir/" + install -m 0644 "$source/host/systemd/ci-fleet-drift.timer" "$systemd_dir/" + systemctl daemon-reload + systemctl enable --now "${timer_names[@]}" >/dev/null +} + +remove_systemd_units() { + systemctl disable --now "${timer_names[@]}" >/dev/null 2>&1 || true + local unit + for unit in "${unit_names[@]}"; do rm -f "$systemd_dir/$unit"; done systemctl daemon-reload - systemctl enable --now ci-fleet-health.timer ci-fleet-cleanup.timer ci-fleet-drift.timer >/dev/null } activate_candidate() { @@ -396,7 +438,7 @@ activate_candidate() { ln -sfn "$release_dir" "$temporary/current" mv -Tf "$temporary/current" "$current_link" install_manager - install_systemd_units + install_systemd_units "$(readlink -f "$manager_current")" if [[ "$target_state" == active ]]; then compose "$release_dir" "$rendered_env" up -d --no-deps controller sleep "${CI_FLEET_STARTUP_WAIT_SECONDS:-2}" @@ -418,38 +460,88 @@ activate_candidate() { install -m 0600 "$temporary/install-state.json" "$state_file" } +restore_systemd_snapshot() { + local unit timer failed=0 + remove_systemd_units || failed=1 + for unit in "${unit_names[@]}"; do + [[ ! -f "$checkpoint_dir/systemd/$unit" ]] || install -m 0644 "$checkpoint_dir/systemd/$unit" "$systemd_dir/$unit" || failed=1 + done + systemctl daemon-reload || failed=1 + for timer in "${timer_names[@]}"; do + if grep -Fxq "$timer" "$checkpoint_dir/enabled-timers"; then systemctl enable "$timer" >/dev/null || failed=1; else systemctl disable "$timer" >/dev/null 2>&1 || true; fi + if grep -Fxq "$timer" "$checkpoint_dir/active-timers"; then systemctl start "$timer" || failed=1; else systemctl stop "$timer" >/dev/null 2>&1 || true; fi + done + return "$failed" +} + restore_checkpoint() { - local target restored_state + local target restored_state failed=0 [[ -n "$checkpoint_dir" && -d "$checkpoint_dir" ]] || return 1 + if ! try_drain_current; then + note "ROLLBACK_FAILED reason=$drain_error" + return 1 + fi + trap - ERR set +e if [[ -f "$checkpoint_dir/ci-fleet.env" ]]; then - install -m 0600 "$checkpoint_dir/ci-fleet.env" "$rendered_env" + install -m 0600 "$checkpoint_dir/ci-fleet.env" "$rendered_env" || failed=1 else - rm -f "$rendered_env" + rm -f "$rendered_env" || failed=1 fi if [[ -f "$checkpoint_dir/install-state.json" ]]; then - install -m 0600 "$checkpoint_dir/install-state.json" "$state_file" + install -m 0600 "$checkpoint_dir/install-state.json" "$state_file" || failed=1 else - rm -f "$state_file" + rm -f "$state_file" || failed=1 fi if [[ -f "$checkpoint_dir/release-target" ]]; then target=$(<"$checkpoint_dir/release-target") - if [[ -d "$target" ]]; then - ln -sfn "$target" "$temporary/rollback-current" - mv -Tf "$temporary/rollback-current" "$current_link" + if [[ -d "$target" && -f "$target/deploy/compose.yaml" ]]; then + ln -sfn "$target" "$temporary/rollback-current" && mv -Tf "$temporary/rollback-current" "$current_link" || failed=1 release_dir=$target - install_systemd_units - if [[ -f "$rendered_env" ]]; then - restored_state=$(awk -F= '$1 == "CI_FLEET_CONTROLLER_STATE" {print $2}' "$rendered_env") - if [[ "$restored_state" == active ]]; then - compose "$release_dir" "$rendered_env" up -d --no-deps controller + else + failed=1 + fi + else + rm -f "$current_link" || failed=1 + release_dir= + fi + if [[ -f "$checkpoint_dir/manager-target" ]]; then + target=$(<"$checkpoint_dir/manager-target") + if [[ "$target" == "$manager_releases/"* && -d "$target" ]]; then + ln -sfn "$target" "$temporary/rollback-manager" && mv -Tf "$temporary/rollback-manager" "$manager_current" || failed=1 + else + failed=1 + fi + else + rm -f "$manager_current" || failed=1 + fi + restore_systemd_snapshot || failed=1 + if [[ -n "$release_dir" && -f "$rendered_env" ]]; then + restored_state=$(awk -F= '$1 == "CI_FLEET_CONTROLLER_STATE" {print $2}' "$rendered_env") || failed=1 + [[ "$restored_state" == active || "$restored_state" == drained || "$restored_state" == disabled ]] || failed=1 + if [[ "$restored_state" == active && "$failed" == 0 ]]; then + if [[ $(managed_runner_count) != 0 ]]; then + failed=1 + else + compose "$release_dir" "$rendered_env" up -d --no-deps controller || failed=1 + if ((failed == 0)); then + ( + set -a + # shellcheck disable=SC1090 + . "$rendered_env" + set +a + "$release_dir/scripts/healthcheck.sh" + ) || failed=1 fi fi fi - else - rm -f "$current_link" fi set -e + trap on_error ERR + if ((failed != 0)); then + note "ROLLBACK_FAILED checkpoint=$checkpoint_dir" + return 1 + fi note "ROLLBACK_RESTORED checkpoint=$checkpoint_dir" } @@ -529,12 +621,7 @@ perform_uninstall() { "$old_release/scripts/cleanup.sh" --apply --instance "${CI_FLEET_INSTANCE:-}" || true ) fi - systemctl disable --now ci-fleet-health.timer ci-fleet-cleanup.timer ci-fleet-drift.timer >/dev/null 2>&1 || true - rm -f \ - "$systemd_dir/ci-fleet-health.service" "$systemd_dir/ci-fleet-health.timer" \ - "$systemd_dir/ci-fleet-cleanup.service" "$systemd_dir/ci-fleet-cleanup.timer" \ - "$systemd_dir/ci-fleet-drift.service" "$systemd_dir/ci-fleet-drift.timer" - systemctl daemon-reload + remove_systemd_units rm -f "$current_link" "$rendered_env" "$state_file" rm -f "$manager_current" transaction_active=false @@ -542,6 +629,9 @@ perform_uninstall() { } require_commands +install -d -m 0755 "$(dirname "$lock_file")" +exec 9>"$lock_file" +flock -n 9 || die 'another ci-fleet installer or drift check is already running' case "$mode" in check|install|adopt|upgrade) validate_common_arguments diff --git a/scripts/test-install-worker-controller.sh b/scripts/test-install-worker-controller.sh index 274205c8..25ac4018 100755 --- a/scripts/test-install-worker-controller.sh +++ b/scripts/test-install-worker-controller.sh @@ -131,12 +131,18 @@ printf '\n' >>"$root/etc/ci-fleet/ci-fleet.env" expect_failure 'DRIFT rendered_environment' "$installer" --check "${base_args[@]}" --ref "$ref_one" expect_success "$installer" --install "${base_args[@]}" --ref "$ref_one" >/dev/null +prior_manager=$root/opt/ci-fleet/manager/releases/prior-manager +cp -a "$(readlink -f "$root/opt/ci-fleet/manager/current")" "$prior_manager" +ln -sfn "$prior_manager" "$root/opt/ci-fleet/manager/current" + ref_two=$(write_config active 2 2) export FAKE_FAIL_UP_ONCE=$tmp/fail-up-once : >"$FAKE_FAIL_UP_ONCE" expect_failure 'ROLLBACK_RESTORED' "$installer" --upgrade "${base_args[@]}" --ref "$ref_two" unset FAKE_FAIL_UP_ONCE grep -Fq 'CI_FLEET_MAX_RUNNERS=1' "$root/etc/ci-fleet/ci-fleet.env" || fail 'failed activation did not restore capacity one' +[[ $(readlink -f "$root/opt/ci-fleet/manager/current") == "$prior_manager" ]] || fail 'failed activation did not restore the prior manager release' +[[ -f "$FAKE_DOCKER_STATE" ]] || fail 'failed activation did not restore the prior controller runtime' expect_success "$installer" --upgrade "${base_args[@]}" --ref "$ref_two" >/dev/null grep -Fq 'CI_FLEET_MAX_RUNNERS=2' "$root/etc/ci-fleet/ci-fleet.env" || fail 'upgrade did not apply capacity two' From a5d2ed418e27422db294b37824fe5c56072efdd0 Mon Sep 17 00:00:00 2001 From: Nickfosts Hermes Date: Sat, 18 Jul 2026 18:21:44 -0500 Subject: [PATCH 04/57] fix(installer): satisfy shellcheck timer capture --- scripts/install-worker-controller.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/install-worker-controller.sh b/scripts/install-worker-controller.sh index ff76b115..708e5fad 100755 --- a/scripts/install-worker-controller.sh +++ b/scripts/install-worker-controller.sh @@ -376,8 +376,8 @@ make_checkpoint() { : >"$temporary/enabled-timers" : >"$temporary/active-timers" for timer in "${timer_names[@]}"; do - systemctl is-enabled --quiet "$timer" 2>/dev/null && printf '%s\n' "$timer" >>"$temporary/enabled-timers" || true - systemctl is-active --quiet "$timer" 2>/dev/null && printf '%s\n' "$timer" >>"$temporary/active-timers" || true + if systemctl is-enabled --quiet "$timer" 2>/dev/null; then printf '%s\n' "$timer" >>"$temporary/enabled-timers"; fi + if systemctl is-active --quiet "$timer" 2>/dev/null; then printf '%s\n' "$timer" >>"$temporary/active-timers"; fi done install -m 0600 "$temporary/enabled-timers" "$checkpoint_dir/enabled-timers" install -m 0600 "$temporary/active-timers" "$checkpoint_dir/active-timers" From 87c21c2e634b30e0563651b70c36fed818997d7e Mon Sep 17 00:00:00 2001 From: Nickfosts Hermes Date: Sat, 18 Jul 2026 18:32:30 -0500 Subject: [PATCH 05/57] test(config): derive capacity overcommit fixtures --- templates/config-repository/scripts/test_policy.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/templates/config-repository/scripts/test_policy.py b/templates/config-repository/scripts/test_policy.py index e42d9f9a..89d749f2 100755 --- a/templates/config-repository/scripts/test_policy.py +++ b/templates/config-repository/scripts/test_policy.py @@ -58,14 +58,16 @@ def test_public_repository_access_is_rejected(self) -> None: def test_capacity_overcommit_is_rejected(self) -> None: config = copy.deepcopy(reference_config()) - first_controller(config)["max_runners"] = 2 - self.assert_rejected(config, "must cover 2 runners") + overcommit = config["runner_pools"]["trusted-ci"]["capacity_budget"] + 1 + first_controller(config)["max_runners"] = overcommit + self.assert_rejected(config, f"must cover {overcommit} runners") def test_drained_capacity_remains_reserved(self) -> None: config = copy.deepcopy(reference_config()) + overcommit = config["runner_pools"]["trusted-ci"]["capacity_budget"] + 1 first_controller(config)["state"] = "drained" - first_controller(config)["max_runners"] = 2 - self.assert_rejected(config, "must cover 2 runners") + first_controller(config)["max_runners"] = overcommit + self.assert_rejected(config, f"must cover {overcommit} runners") def test_disabled_capacity_is_not_reserved(self) -> None: config = copy.deepcopy(reference_config()) From a0b8c176749003a73f94b3f4b296d084daaa6d3b Mon Sep 17 00:00:00 2001 From: Nickfosts Hermes Date: Sat, 18 Jul 2026 18:37:59 -0500 Subject: [PATCH 06/57] fix(config): align drain and managed minimum --- docs/DESIRED-STATE.md | 4 +++- scripts/install-worker-controller.sh | 16 +++++++++++++++- scripts/test-install-worker-controller.sh | 4 ++-- templates/config-repository/README.md | 2 +- templates/config-repository/fleet.schema.json | 2 +- .../config-repository/scripts/test_policy.py | 7 ++++++- templates/config-repository/scripts/validate.py | 3 +-- 7 files changed, 29 insertions(+), 9 deletions(-) diff --git a/docs/DESIRED-STATE.md b/docs/DESIRED-STATE.md index 34f83727..d08f1315 100644 --- a/docs/DESIRED-STATE.md +++ b/docs/DESIRED-STATE.md @@ -31,11 +31,13 @@ Each controller has a unique object key and declares: - a unique scale-set name; - `experimental`, `stable`, or `retiring` lifecycle; - a full pinned ci-fleet engine commit; -- minimum and maximum runners; +- a zero managed minimum and reviewed maximum runner capacity; - CPU cores and memory per ephemeral runner. Active and drained controllers reserve their configured maximum against the pool budget. A drained controller has zero effective runtime capacity but keeps its reservation, so an undrain cannot silently overcommit the pool. Disabled controllers reserve no capacity. +Managed prewarmed runners are not currently supported: `min_runners` is fixed at zero in schema, semantic validation, rendering, and preflight. This keeps idle privileged workers absent and prevents reviewed configuration from passing validation only to fail host adoption. + The authoritative fictional contract is in [`templates/config-repository`](../templates/config-repository/README.md). ## Application parallelism rule diff --git a/scripts/install-worker-controller.sh b/scripts/install-worker-controller.sh index 708e5fad..1fe0b0bc 100755 --- a/scripts/install-worker-controller.sh +++ b/scripts/install-worker-controller.sh @@ -404,7 +404,21 @@ try_drain_current() { fi sleep 2 done - if ! compose "$old_release" "$rendered_env" stop controller >/dev/null; then drain_error='could not stop the drained controller'; return 1; fi + note 'DRAIN_READY managed_runners=0' + docker compose --project-name ci-fleet --env-file "$rendered_env" -f "$old_release/deploy/compose.yaml" kill --signal SIGTERM controller >/dev/null || { + drain_error='failed to signal the paused controller for graceful scale-set cleanup' + return 1 + } + if [[ $(docker inspect --format '{{.State.Paused}}' "$controller_container" 2>/dev/null || true) == true ]]; then + docker compose --project-name ci-fleet --env-file "$rendered_env" -f "$old_release/deploy/compose.yaml" unpause controller >/dev/null || { + drain_error='failed to unpause the signaled controller for graceful shutdown' + return 1 + } + fi + docker compose --project-name ci-fleet --env-file "$rendered_env" -f "$old_release/deploy/compose.yaml" stop controller >/dev/null || { + drain_error='could not stop the drained controller' + return 1 + } note 'DRAIN_OK managed_runners=0' } diff --git a/scripts/test-install-worker-controller.sh b/scripts/test-install-worker-controller.sh index 25ac4018..96abb019 100755 --- a/scripts/test-install-worker-controller.sh +++ b/scripts/test-install-worker-controller.sh @@ -27,7 +27,7 @@ case "${1:-}" in if [[ "${2:-}" == version ]]; then exit 0; fi command= for argument in "$@"; do - case "$argument" in config|build|up|stop|pause|unpause|down|logs|rm) command=$argument ;; esac + case "$argument" in config|build|up|stop|pause|unpause|kill|down|logs|rm) command=$argument ;; esac done case "$command" in up) @@ -38,7 +38,7 @@ case "${1:-}" in : >"$state" ;; stop|down|rm) rm -f "$state" ;; - config|build|pause|unpause|logs) ;; + config|build|pause|unpause|kill|logs) ;; *) exit 1 ;; esac ;; diff --git a/templates/config-repository/README.md b/templates/config-repository/README.md index 952e1535..74d92bc7 100644 --- a/templates/config-repository/README.md +++ b/templates/config-repository/README.md @@ -59,7 +59,7 @@ The initializer refuses to replace a configured file unless `--force` is explici - its unique GitHub scale-set name; - an `experimental`, `stable`, or `retiring` lifecycle; - the full reviewed ci-fleet commit SHA it runs; -- minimum and maximum runners; +- a zero managed minimum and reviewed maximum runner capacity; - CPU and memory available to each ephemeral runner. The controller ID is how a target host selects its declaration. A location is a non-sensitive logical slug such as `primary-site` or `remote-site`, never an address. Runtime-generated configuration and credentials remain host-local. diff --git a/templates/config-repository/fleet.schema.json b/templates/config-repository/fleet.schema.json index feebece2..a1cb19ea 100644 --- a/templates/config-repository/fleet.schema.json +++ b/templates/config-repository/fleet.schema.json @@ -77,7 +77,7 @@ "scale_set_name": {"$ref": "#/$defs/slug"}, "lifecycle": {"enum": ["experimental", "stable", "retiring"]}, "engine_ref": {"type": "string", "pattern": "^[0-9a-f]{40}$"}, - "min_runners": {"type": "integer", "minimum": 0}, + "min_runners": {"const": 0}, "max_runners": {"type": "integer", "minimum": 1}, "runner_resources": {"$ref": "#/$defs/runner_resources"} } diff --git a/templates/config-repository/scripts/test_policy.py b/templates/config-repository/scripts/test_policy.py index 89d749f2..84d0e184 100755 --- a/templates/config-repository/scripts/test_policy.py +++ b/templates/config-repository/scripts/test_policy.py @@ -103,11 +103,16 @@ def test_zero_engine_ref_is_rejected(self) -> None: first_controller(config)["engine_ref"] = "0" * 40 self.assert_rejected(config, "nonzero full lowercase commit SHA") + def test_active_controller_minimum_must_be_zero(self) -> None: + config = copy.deepcopy(reference_config()) + first_controller(config)["min_runners"] = 1 + self.assert_rejected(config, "managed prewarmed runners are not supported") + def test_drained_controller_minimum_must_be_zero(self) -> None: config = copy.deepcopy(reference_config()) first_controller(config)["state"] = "drained" first_controller(config)["min_runners"] = 1 - self.assert_rejected(config, "must be zero while drained or disabled") + self.assert_rejected(config, "managed prewarmed runners are not supported") def test_application_capacity_control_is_rejected(self) -> None: config = copy.deepcopy(reference_config()) diff --git a/templates/config-repository/scripts/validate.py b/templates/config-repository/scripts/validate.py index db56e621..8cc535ed 100755 --- a/templates/config-repository/scripts/validate.py +++ b/templates/config-repository/scripts/validate.py @@ -265,8 +265,7 @@ def validate_config(config: Any, validation: Validation, strict: bool) -> None: validation.require(type(maximum) is int and maximum > 0, f"{path}.max_runners", "must be a positive integer") if type(minimum) is int and type(maximum) is int: validation.require(minimum <= maximum, f"{path}.min_runners", "must not exceed max_runners") - if state in {"drained", "disabled"}: - validation.require(minimum == 0, f"{path}.min_runners", "must be zero while drained or disabled") + validation.require(minimum == 0, f"{path}.min_runners", "must be zero because managed prewarmed runners are not supported") resources = controller.get("runner_resources") if validation.exact_keys(resources, f"{path}.runner_resources", {"cpu_cores", "memory_mib"}): cpu = resources.get("cpu_cores") From 593ce2bc9f87223842719e1b2f18afc027b7abe3 Mon Sep 17 00:00:00 2001 From: Nickfosts Hermes Date: Sat, 18 Jul 2026 18:40:05 -0500 Subject: [PATCH 07/57] test: isolate validation image tags --- scripts/validate.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/validate.sh b/scripts/validate.sh index 816848eb..979b75c5 100755 --- a/scripts/validate.sh +++ b/scripts/validate.sh @@ -26,10 +26,12 @@ export CI_FLEET_GITHUB_APP_CLIENT_ID=validation export CI_FLEET_GITHUB_APP_INSTALLATION_ID=1 export CI_FLEET_GITHUB_APP_PRIVATE_KEY_FILE="$tmp" export CI_FLEET_DOCKER_GID=999 +export CI_FLEET_RUNNER_IMAGE=${CI_FLEET_RUNNER_IMAGE:-ci-fleet-runner:dev} +export CI_FLEET_CONTROLLER_IMAGE=${CI_FLEET_CONTROLLER_IMAGE:-ci-fleet-controller:dev} docker compose -f deploy/compose.yaml config --quiet docker compose -f deploy/compose.yaml build runner-image controller -docker run --rm --entrypoint /bin/bash ci-fleet-runner:dev -c './bin/Runner.Listener --version && docker --version && docker compose version && git --version' +docker run --rm --entrypoint /bin/bash "$CI_FLEET_RUNNER_IMAGE" -c './bin/Runner.Listener --version && docker --version && docker compose version && git --version' CI_FLEET_INSTANCE=validation scripts/cleanup.sh if rg -n --hidden --glob '!*.example' --glob '!scripts/validate.sh' --glob '!docs/SECRETS.md' --glob '!SECURITY.md' '(BEGIN (RSA |EC |OPENSSH )?PRIVATE KEY|github_pat_|ghp_[A-Za-z0-9]{20,})' .; then From dadd983fee58420d5a6c8b301401f664b009ffdb Mon Sep 17 00:00:00 2001 From: Nickfosts Hermes Date: Sat, 18 Jul 2026 18:42:56 -0500 Subject: [PATCH 08/57] fix(installer): name the managed controller container --- scripts/install-worker-controller.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/install-worker-controller.sh b/scripts/install-worker-controller.sh index 1fe0b0bc..00b2f748 100755 --- a/scripts/install-worker-controller.sh +++ b/scripts/install-worker-controller.sh @@ -103,6 +103,7 @@ state_file=$state_root/install-state.json checkpoints_dir=$state_root/checkpoints systemd_dir=$(root_path /etc/systemd/system) lock_file=$(root_path /run/ci-fleet-installer.lock) +controller_container=ci-fleet-controller-1 unit_names=( ci-fleet-health.service ci-fleet-health.timer ci-fleet-cleanup.service ci-fleet-cleanup.timer @@ -219,7 +220,7 @@ compose() { } controller_status() { - docker inspect --format '{{.State.Status}}' ci-fleet-controller-1 2>/dev/null || true + docker inspect --format '{{.State.Status}}' "$controller_container" 2>/dev/null || true } current_runtime_release() { From ab76d653984468b6c01472824f4ba8b1c38832ab Mon Sep 17 00:00:00 2001 From: Nickfosts Hermes Date: Sat, 18 Jul 2026 18:44:58 -0500 Subject: [PATCH 09/57] refactor(installer): use Python for metadata state --- scripts/install-worker-controller.sh | 39 ++++++++++++++++++++++------ 1 file changed, 31 insertions(+), 8 deletions(-) diff --git a/scripts/install-worker-controller.sh b/scripts/install-worker-controller.sh index 00b2f748..f8c2b21c 100755 --- a/scripts/install-worker-controller.sh +++ b/scripts/install-worker-controller.sh @@ -117,7 +117,7 @@ trap cleanup_temporary EXIT require_commands() { local command - for command in git python3 docker jq tar install cmp readlink systemctl stat awk grep date flock; do + for command in git python3 docker tar install cmp readlink systemctl stat awk grep date flock; do command -v "$command" >/dev/null || die "$command is required" done docker info >/dev/null 2>&1 || die 'Docker daemon is unavailable' @@ -196,6 +196,7 @@ docker_gid() { } render_candidate() { + local -a metadata_values candidate_env=$temporary/ci-fleet.env candidate_metadata=$temporary/metadata.json python3 "$repo_root/scripts/desired_state.py" render \ @@ -207,9 +208,18 @@ render_candidate() { --docker-gid "$(docker_gid)" \ --output "$candidate_env" \ --metadata-output "$candidate_metadata" - target_state=$(jq -r .controller_state "$candidate_metadata") - engine_ref=$(jq -r .engine_ref "$candidate_metadata") - engine_repository=$(jq -r .engine_repository "$candidate_metadata") + mapfile -t metadata_values < <(python3 - "$candidate_metadata" <<'PY' +import json +import sys +value = json.load(open(sys.argv[1], encoding="utf-8")) +for key in ("controller_state", "engine_ref", "engine_repository"): + print(value[key]) +PY + ) + [[ ${#metadata_values[@]} == 3 ]] || die 'rendered controller metadata is incomplete' + target_state=${metadata_values[0]} + engine_ref=${metadata_values[1]} + engine_repository=${metadata_values[2]} release_dir=$releases_dir/$engine_ref } @@ -249,8 +259,14 @@ runtime_matches() { state_matches() { [[ -f "$state_file" ]] || return 1 - jq 'del(.installed_at)' "$state_file" >"$temporary/installed-metadata.json" || return 1 - cmp -s "$candidate_metadata" "$temporary/installed-metadata.json" + python3 - "$state_file" "$candidate_metadata" <<'PY' +import json +import sys +installed = json.load(open(sys.argv[1], encoding="utf-8")) +installed.pop("installed_at", None) +candidate = json.load(open(sys.argv[2], encoding="utf-8")) +raise SystemExit(0 if installed == candidate else 1) +PY } release_matches() { @@ -470,8 +486,15 @@ activate_candidate() { else compose "$release_dir" "$rendered_env" stop controller >/dev/null 2>&1 || true fi - jq --arg installed_at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" '. + {installed_at: $installed_at}' \ - "$candidate_metadata" >"$temporary/install-state.json" + python3 - "$candidate_metadata" "$temporary/install-state.json" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" <<'PY' +import json +import sys +value = json.load(open(sys.argv[1], encoding="utf-8")) +value["installed_at"] = sys.argv[3] +with open(sys.argv[2], "w", encoding="utf-8") as handle: + json.dump(value, handle, indent=2, sort_keys=True) + handle.write("\n") +PY install -m 0600 "$temporary/install-state.json" "$state_file" } From d15744982feaf32bf57316a04604cd33febcb93c Mon Sep 17 00:00:00 2001 From: Nickfosts Hermes Date: Sat, 18 Jul 2026 18:47:36 -0500 Subject: [PATCH 10/57] fix(installer): advance the maintenance manager --- docs/DESIRED-STATE.md | 2 +- scripts/install-worker-controller.sh | 15 ++++++--------- scripts/test-install-worker-controller.sh | 1 + 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/docs/DESIRED-STATE.md b/docs/DESIRED-STATE.md index d08f1315..90ebce85 100644 --- a/docs/DESIRED-STATE.md +++ b/docs/DESIRED-STATE.md @@ -87,7 +87,7 @@ The installer: 9. starts the controller only when its desired state is active; 10. verifies runtime health and records a redacted installation state. -A successful second `--install` run reports `NO_CHANGE` and performs no unnecessary replacement. +A successful second `--install` run reports `NO_CHANGE` and performs no unnecessary replacement. A successful engine upgrade advances both the runtime release and the maintenance installer-manager to the same pinned commit; rollback restores both. ## Adopt an existing controller diff --git a/scripts/install-worker-controller.sh b/scripts/install-worker-controller.sh index f8c2b21c..f932b410 100755 --- a/scripts/install-worker-controller.sh +++ b/scripts/install-worker-controller.sh @@ -325,20 +325,17 @@ install_release() { } install_manager() { - local manager_commit manager_release archive marker staged_manager - if [[ -f "$repo_root/.ci-fleet-engine-ref" ]]; then - manager_commit=$(<"$repo_root/.ci-fleet-engine-ref") - else - is_git_checkout "$repo_root" || die 'installer manager source is not a Git checkout or installed manager release' - manager_commit=$(git -C "$repo_root" rev-parse 'HEAD^{commit}') - fi + local manager_commit manager_release archive marker staged_manager release_marker + manager_commit=$engine_ref [[ "$manager_commit" =~ ^[0-9a-f]{40}$ ]] || die 'installer manager commit is invalid' + [[ -d "$release_dir" && -f "$release_dir/.ci-fleet-engine-ref" ]] || die 'desired engine release is unavailable for installer manager activation' + release_marker=$(<"$release_dir/.ci-fleet-engine-ref") + [[ "$release_marker" == "$manager_commit" ]] || die 'desired engine release marker is inconsistent' manager_release=$manager_releases/$manager_commit if [[ ! -d "$manager_release" ]]; then - is_git_checkout "$repo_root" || die 'a new installer manager release requires a Git checkout' install -d -m 0755 "$manager_releases" archive=$temporary/manager.tar - git -C "$repo_root" archive --format=tar --output "$archive" HEAD + tar -cf "$archive" -C "$release_dir" . staged_manager=$temporary/manager-release install -d -m 0755 "$staged_manager" tar -xf "$archive" -C "$staged_manager" diff --git a/scripts/test-install-worker-controller.sh b/scripts/test-install-worker-controller.sh index 96abb019..c3189b27 100755 --- a/scripts/test-install-worker-controller.sh +++ b/scripts/test-install-worker-controller.sh @@ -121,6 +121,7 @@ base_args=(--config-repo "$config_repo" --controller example-ci-01) first=$(expect_success "$installer" --install "${base_args[@]}" --ref "$ref_one") grep -Fq 'CONVERGED mode=install' <<<"$first" || fail 'fresh install did not converge' [[ -L "$root/opt/ci-fleet/current" && -f "$root/var/lib/ci-fleet/install-state.json" ]] || fail 'fresh install state is incomplete' +[[ $(readlink -f "$root/opt/ci-fleet/manager/current") == "$root/opt/ci-fleet/manager/releases/$engine_ref" ]] || fail 'installer manager did not activate the desired engine release' [[ -f "$FAKE_DOCKER_STATE" ]] || fail 'active controller was not started' second=$(expect_success "$installer" --install "${base_args[@]}" --ref "$ref_one") From ea376240d478475168ba4b440db7451faf67c364 Mon Sep 17 00:00:00 2001 From: Nickfosts Hermes Date: Sat, 18 Jul 2026 18:49:37 -0500 Subject: [PATCH 11/57] test: make committed-secret scan dependency-free --- scripts/scan_committed_secrets.py | 51 +++++++++++++++++++++++++++++++ scripts/validate.sh | 6 ++-- 2 files changed, 53 insertions(+), 4 deletions(-) create mode 100644 scripts/scan_committed_secrets.py diff --git a/scripts/scan_committed_secrets.py b/scripts/scan_committed_secrets.py new file mode 100644 index 00000000..bc6cabdc --- /dev/null +++ b/scripts/scan_committed_secrets.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 +"""Fail when tracked files contain high-confidence credential material.""" + +from __future__ import annotations + +import re +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +EXCLUDED = { + "SECURITY.md", + "docs/SECRETS.md", + "scripts/scan_committed_secrets.py", + "scripts/validate.sh", +} +PATTERN = re.compile( + rb"BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY|" + rb"github_pat_[A-Za-z0-9_]{20,}|" + rb"ghp_[A-Za-z0-9]{20,}" +) + + +def main() -> int: + tracked = subprocess.run( + ["git", "-C", str(ROOT), "ls-files", "-z"], + check=True, + stdout=subprocess.PIPE, + ).stdout.split(b"\0") + findings: list[str] = [] + for raw in tracked: + if not raw: + continue + relative = raw.decode("utf-8") + if relative in EXCLUDED or relative.endswith(".example"): + continue + data = (ROOT / relative).read_bytes() + for match in PATTERN.finditer(data): + line = data.count(b"\n", 0, match.start()) + 1 + findings.append(f"{relative}:{line}") + if findings: + print("possible committed secret detected:", file=sys.stderr) + print("\n".join(findings), file=sys.stderr) + return 1 + print("OK: no high-confidence secret material in tracked files") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/validate.sh b/scripts/validate.sh index 979b75c5..df4d3901 100755 --- a/scripts/validate.sh +++ b/scripts/validate.sh @@ -9,6 +9,7 @@ python3 -m py_compile \ .github/actions/plan/plan.py \ .github/actions/plan/test_plan.py \ scripts/desired_state.py \ + scripts/scan_committed_secrets.py \ scripts/test_desired_state.py python3 .github/actions/plan/test_plan.py python3 scripts/test_desired_state.py @@ -34,7 +35,4 @@ docker compose -f deploy/compose.yaml build runner-image controller docker run --rm --entrypoint /bin/bash "$CI_FLEET_RUNNER_IMAGE" -c './bin/Runner.Listener --version && docker --version && docker compose version && git --version' CI_FLEET_INSTANCE=validation scripts/cleanup.sh -if rg -n --hidden --glob '!*.example' --glob '!scripts/validate.sh' --glob '!docs/SECRETS.md' --glob '!SECURITY.md' '(BEGIN (RSA |EC |OPENSSH )?PRIVATE KEY|github_pat_|ghp_[A-Za-z0-9]{20,})' .; then - echo "possible committed secret detected" >&2 - exit 1 -fi +python3 scripts/scan_committed_secrets.py From a1f88235d8d0de8167f8cc6fecf265492f49f1cc Mon Sep 17 00:00:00 2001 From: Nickfosts Hermes Date: Sat, 18 Jul 2026 19:01:32 -0500 Subject: [PATCH 12/57] fix(installer): make reviewed upgrades applicable --- .github/workflows/validate.yml | 1 + docs/DESIRED-STATE.md | 11 ++++++----- scripts/install-worker-controller.sh | 8 ++++---- scripts/test-install-worker-controller.sh | 12 +++++++++++- templates/config-repository/scripts/init.py | 4 ++-- 5 files changed, 24 insertions(+), 12 deletions(-) diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 10a16176..f07e63f4 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -42,4 +42,5 @@ jobs: templates/config-repository/scripts/init.sh \ --organization test-company \ --project test-app \ + --engine-ref "$(git rev-parse HEAD)" \ --output "${temporary_directory}/fleet.json" diff --git a/docs/DESIRED-STATE.md b/docs/DESIRED-STATE.md index 90ebce85..8f81ceba 100644 --- a/docs/DESIRED-STATE.md +++ b/docs/DESIRED-STATE.md @@ -81,11 +81,12 @@ The installer: 3. selects exactly one logical controller; 4. renders `/etc/ci-fleet/ci-fleet.env` without secret values; 5. fetches and verifies the pinned public engine commit; -6. builds the pinned runner and controller images; -7. creates a root-only controller checkpoint; -8. installs health, cleanup, and pinned-state drift timers; -9. starts the controller only when its desired state is active; -10. verifies runtime health and records a redacted installation state. +6. creates a root-only controller checkpoint; +7. drains the current controller and waits for managed runners to finish; +8. runs managed preflight and builds the pinned runner and controller images; +9. installs health, cleanup, and pinned-state drift timers; +10. starts the controller only when its desired state is active; +11. verifies runtime health and records a redacted installation state. A successful second `--install` run reports `NO_CHANGE` and performs no unnecessary replacement. A successful engine upgrade advances both the runtime release and the maintenance installer-manager to the same pinned commit; rollback restores both. diff --git a/scripts/install-worker-controller.sh b/scripts/install-worker-controller.sh index f932b410..debb3a41 100755 --- a/scripts/install-worker-controller.sh +++ b/scripts/install-worker-controller.sh @@ -140,7 +140,7 @@ resolve_config() { resolved=$(git -C "$config_repo" rev-parse "$config_ref^{commit}" 2>/dev/null || true) [[ "$resolved" == "$config_ref" ]] || die 'local configuration repository does not contain the requested commit' git -C "$config_repo" show "$config_ref:fleet.json" >"$candidate_config" || die 'fleet.json is absent at the requested configuration commit' - config_identity=$config_repo + config_identity=$(git -C "$config_repo" rev-parse --show-toplevel) return fi [[ "$config_repo" =~ ^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$ ]] || die '--config-repo must be OWNER/REPOSITORY or a local Git checkout' @@ -357,7 +357,7 @@ run_candidate_preflight() { # shellcheck disable=SC1090 . "$candidate_env" set +a - CI_FLEET_TESTING=$testing "$repo_root/scripts/preflight.sh" --managed + CI_FLEET_TESTING=$testing "$release_dir/scripts/preflight.sh" --managed ) } @@ -476,7 +476,7 @@ activate_candidate() { # shellcheck disable=SC1090 . "$rendered_env" set +a - "$repo_root/scripts/healthcheck.sh" + "$release_dir/scripts/healthcheck.sh" ); then die 'post-activation health check failed' fi @@ -619,10 +619,10 @@ perform_converge() { return fi install_release - build_candidate make_checkpoint transaction_active=true drain_current + build_candidate activate_candidate transaction_active=false note "CONVERGED mode=$mode controller=$controller_id config_ref=$config_ref engine_ref=$engine_ref state=$target_state" diff --git a/scripts/test-install-worker-controller.sh b/scripts/test-install-worker-controller.sh index c3189b27..948525e8 100755 --- a/scripts/test-install-worker-controller.sh +++ b/scripts/test-install-worker-controller.sh @@ -17,7 +17,13 @@ case "${1:-}" in [[ -f "$state" ]] || exit 1 if [[ "$*" == *'.State.Status'* ]]; then printf 'running\n'; else printf 'running\n'; fi ;; - ps) exit 0 ;; + ps) + if [[ -n "${FAKE_ACTIVE_RUNNERS_ONCE:-}" && -f "$FAKE_ACTIVE_RUNNERS_ONCE" ]]; then + rm -f "$FAKE_ACTIVE_RUNNERS_ONCE" + printf 'fixture-runner\n' + fi + exit 0 + ;; volume|network) [[ "${2:-}" == ls ]] && exit 0 [[ "${2:-}" == inspect ]] && exit 1 @@ -127,6 +133,7 @@ grep -Fq 'CONVERGED mode=install' <<<"$first" || fail 'fresh install did not con second=$(expect_success "$installer" --install "${base_args[@]}" --ref "$ref_one") grep -Fq 'NO_CHANGE' <<<"$second" || fail 'idempotent rerun changed the host' expect_success "$installer" --check "${base_args[@]}" --ref "$ref_one" >/dev/null +(cd "$tmp" && "$installer" --check --config-repo config-repo --controller example-ci-01 --ref "$ref_one" >/dev/null) || fail 'relative local config path was not persisted as an absolute checkout identity' printf '\n' >>"$root/etc/ci-fleet/ci-fleet.env" expect_failure 'DRIFT rendered_environment' "$installer" --check "${base_args[@]}" --ref "$ref_one" @@ -137,10 +144,13 @@ cp -a "$(readlink -f "$root/opt/ci-fleet/manager/current")" "$prior_manager" ln -sfn "$prior_manager" "$root/opt/ci-fleet/manager/current" ref_two=$(write_config active 2 2) +export FAKE_ACTIVE_RUNNERS_ONCE=$tmp/active-runner-once +: >"$FAKE_ACTIVE_RUNNERS_ONCE" export FAKE_FAIL_UP_ONCE=$tmp/fail-up-once : >"$FAKE_FAIL_UP_ONCE" expect_failure 'ROLLBACK_RESTORED' "$installer" --upgrade "${base_args[@]}" --ref "$ref_two" unset FAKE_FAIL_UP_ONCE +unset FAKE_ACTIVE_RUNNERS_ONCE grep -Fq 'CI_FLEET_MAX_RUNNERS=1' "$root/etc/ci-fleet/ci-fleet.env" || fail 'failed activation did not restore capacity one' [[ $(readlink -f "$root/opt/ci-fleet/manager/current") == "$prior_manager" ]] || fail 'failed activation did not restore the prior manager release' [[ -f "$FAKE_DOCKER_STATE" ]] || fail 'failed activation did not restore the prior controller runtime' diff --git a/templates/config-repository/scripts/init.py b/templates/config-repository/scripts/init.py index 7790b0fa..f1268dd8 100755 --- a/templates/config-repository/scripts/init.py +++ b/templates/config-repository/scripts/init.py @@ -14,7 +14,7 @@ ORG_SLUG = re.compile(r"^[a-z0-9][a-z0-9-]{0,38}$") SLUG = re.compile(r"^[a-z0-9][a-z0-9-]{0,62}$") COMMIT_SHA = re.compile(r"^[0-9a-f]{40}$") -DEFAULT_ENGINE_REF = "3d118432dce86d702fa1b78ecadcfcee750bfd9a" + def positive_integer(value: str) -> int: @@ -38,7 +38,7 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--max-runners", type=positive_integer, default=1, help="initial controller maximum") parser.add_argument("--runner-cpu-cores", type=positive_integer, default=2, help="CPU cores available to each runner") parser.add_argument("--runner-memory-mib", type=positive_integer, default=4096, help="memory available to each runner") - parser.add_argument("--engine-ref", default=DEFAULT_ENGINE_REF, help="reviewed full ci-fleet commit SHA") + parser.add_argument("--engine-ref", required=True, help="reviewed full ci-fleet commit SHA") parser.add_argument("--output", type=Path, default=ROOT / "fleet.json", help="output configuration path") parser.add_argument("--force", action="store_true", help="replace an existing non-example output file") return parser.parse_args() From 971f8ee5592558d45e6f966d7ee49c78f445c988 Mon Sep 17 00:00:00 2001 From: Nick Foster Date: Sat, 18 Jul 2026 19:03:06 -0500 Subject: [PATCH 13/57] fix: drain before managed preflight --- scripts/install-worker-controller.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/install-worker-controller.sh b/scripts/install-worker-controller.sh index debb3a41..708084e2 100755 --- a/scripts/install-worker-controller.sh +++ b/scripts/install-worker-controller.sh @@ -137,10 +137,10 @@ resolve_config() { local resolved checkout candidate_config=$temporary/fleet.json if is_git_checkout "$config_repo"; then - resolved=$(git -C "$config_repo" rev-parse "$config_ref^{commit}" 2>/dev/null || true) + config_identity=$(cd "$config_repo" && pwd -P) + resolved=$(git -C "$config_identity" rev-parse "$config_ref^{commit}" 2>/dev/null || true) [[ "$resolved" == "$config_ref" ]] || die 'local configuration repository does not contain the requested commit' - git -C "$config_repo" show "$config_ref:fleet.json" >"$candidate_config" || die 'fleet.json is absent at the requested configuration commit' - config_identity=$(git -C "$config_repo" rev-parse --show-toplevel) + git -C "$config_identity" show "$config_ref:fleet.json" >"$candidate_config" || die 'fleet.json is absent at the requested configuration commit' return fi [[ "$config_repo" =~ ^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$ ]] || die '--config-repo must be OWNER/REPOSITORY or a local Git checkout' @@ -357,7 +357,7 @@ run_candidate_preflight() { # shellcheck disable=SC1090 . "$candidate_env" set +a - CI_FLEET_TESTING=$testing "$release_dir/scripts/preflight.sh" --managed + CI_FLEET_TESTING=$testing "$repo_root/scripts/preflight.sh" --managed ) } @@ -476,7 +476,7 @@ activate_candidate() { # shellcheck disable=SC1090 . "$rendered_env" set +a - "$release_dir/scripts/healthcheck.sh" + "$repo_root/scripts/healthcheck.sh" ); then die 'post-activation health check failed' fi From f657117b1e1b5b94823d00fee4d35904c1ff8b72 Mon Sep 17 00:00:00 2001 From: Nick Foster Date: Sat, 18 Jul 2026 19:03:08 -0500 Subject: [PATCH 14/57] test: cover drain ordering and relative config paths --- scripts/test-install-worker-controller.sh | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/scripts/test-install-worker-controller.sh b/scripts/test-install-worker-controller.sh index 948525e8..36702bba 100755 --- a/scripts/test-install-worker-controller.sh +++ b/scripts/test-install-worker-controller.sh @@ -18,10 +18,7 @@ case "${1:-}" in if [[ "$*" == *'.State.Status'* ]]; then printf 'running\n'; else printf 'running\n'; fi ;; ps) - if [[ -n "${FAKE_ACTIVE_RUNNERS_ONCE:-}" && -f "$FAKE_ACTIVE_RUNNERS_ONCE" ]]; then - rm -f "$FAKE_ACTIVE_RUNNERS_ONCE" - printf 'fixture-runner\n' - fi + if [[ -n "${FAKE_RUNNER_STATE:-}" && -f "$FAKE_RUNNER_STATE" ]]; then printf 'managed-runner\n'; fi exit 0 ;; volume|network) @@ -44,7 +41,10 @@ case "${1:-}" in : >"$state" ;; stop|down|rm) rm -f "$state" ;; - config|build|pause|unpause|kill|logs) ;; + pause) + [[ -z "${FAKE_RUNNER_STATE:-}" ]] || rm -f "$FAKE_RUNNER_STATE" + ;; + config|build|unpause|kill|logs) ;; *) exit 1 ;; esac ;; @@ -133,7 +133,10 @@ grep -Fq 'CONVERGED mode=install' <<<"$first" || fail 'fresh install did not con second=$(expect_success "$installer" --install "${base_args[@]}" --ref "$ref_one") grep -Fq 'NO_CHANGE' <<<"$second" || fail 'idempotent rerun changed the host' expect_success "$installer" --check "${base_args[@]}" --ref "$ref_one" >/dev/null -(cd "$tmp" && "$installer" --check --config-repo config-repo --controller example-ci-01 --ref "$ref_one" >/dev/null) || fail 'relative local config path was not persisted as an absolute checkout identity' + +relative=$(cd "$tmp" && expect_success "$installer" --install --config-repo config-repo --controller example-ci-01 --ref "$ref_one") +grep -Fq 'NO_CHANGE' <<<"$relative" || fail 'relative configuration path was not normalized before drift comparison' +grep -Fq "CI_FLEET_CONFIG_REPOSITORY=$config_repo" "$root/etc/ci-fleet/ci-fleet.env" || fail 'rendered configuration path is not absolute' printf '\n' >>"$root/etc/ci-fleet/ci-fleet.env" expect_failure 'DRIFT rendered_environment' "$installer" --check "${base_args[@]}" --ref "$ref_one" @@ -144,13 +147,14 @@ cp -a "$(readlink -f "$root/opt/ci-fleet/manager/current")" "$prior_manager" ln -sfn "$prior_manager" "$root/opt/ci-fleet/manager/current" ref_two=$(write_config active 2 2) -export FAKE_ACTIVE_RUNNERS_ONCE=$tmp/active-runner-once -: >"$FAKE_ACTIVE_RUNNERS_ONCE" +export FAKE_RUNNER_STATE=$tmp/managed-runner-active +: >"$FAKE_RUNNER_STATE" export FAKE_FAIL_UP_ONCE=$tmp/fail-up-once : >"$FAKE_FAIL_UP_ONCE" expect_failure 'ROLLBACK_RESTORED' "$installer" --upgrade "${base_args[@]}" --ref "$ref_two" +[[ ! -f "$FAKE_RUNNER_STATE" ]] || fail 'upgrade preflight ran before the active runner was drained' +unset FAKE_RUNNER_STATE unset FAKE_FAIL_UP_ONCE -unset FAKE_ACTIVE_RUNNERS_ONCE grep -Fq 'CI_FLEET_MAX_RUNNERS=1' "$root/etc/ci-fleet/ci-fleet.env" || fail 'failed activation did not restore capacity one' [[ $(readlink -f "$root/opt/ci-fleet/manager/current") == "$prior_manager" ]] || fail 'failed activation did not restore the prior manager release' [[ -f "$FAKE_DOCKER_STATE" ]] || fail 'failed activation did not restore the prior controller runtime' From 1b92b8ee2e2d5430a459ef2df20b7d7bb8bd5f9b Mon Sep 17 00:00:00 2001 From: Nick Foster Date: Sat, 18 Jul 2026 19:03:49 -0500 Subject: [PATCH 15/57] fix: pin examples to compatible engine release --- templates/config-repository/scripts/init.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/config-repository/scripts/init.py b/templates/config-repository/scripts/init.py index f1268dd8..15618216 100755 --- a/templates/config-repository/scripts/init.py +++ b/templates/config-repository/scripts/init.py @@ -14,7 +14,7 @@ ORG_SLUG = re.compile(r"^[a-z0-9][a-z0-9-]{0,38}$") SLUG = re.compile(r"^[a-z0-9][a-z0-9-]{0,62}$") COMMIT_SHA = re.compile(r"^[0-9a-f]{40}$") - +DEFAULT_ENGINE_REF = "f657117b1e1b5b94823d00fee4d35904c1ff8b72" def positive_integer(value: str) -> int: @@ -38,7 +38,7 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--max-runners", type=positive_integer, default=1, help="initial controller maximum") parser.add_argument("--runner-cpu-cores", type=positive_integer, default=2, help="CPU cores available to each runner") parser.add_argument("--runner-memory-mib", type=positive_integer, default=4096, help="memory available to each runner") - parser.add_argument("--engine-ref", required=True, help="reviewed full ci-fleet commit SHA") + parser.add_argument("--engine-ref", default=DEFAULT_ENGINE_REF, help="reviewed full ci-fleet commit SHA") parser.add_argument("--output", type=Path, default=ROOT / "fleet.json", help="output configuration path") parser.add_argument("--force", action="store_true", help="replace an existing non-example output file") return parser.parse_args() From 641e4cdde24d87f9415b11f13634e141999917b5 Mon Sep 17 00:00:00 2001 From: Nick Foster Date: Sat, 18 Jul 2026 19:03:51 -0500 Subject: [PATCH 16/57] fix: pin examples to compatible engine release --- templates/config-repository/fleet.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/config-repository/fleet.json b/templates/config-repository/fleet.json index c7788d99..b7a818cb 100644 --- a/templates/config-repository/fleet.json +++ b/templates/config-repository/fleet.json @@ -24,7 +24,7 @@ "state": "active", "scale_set_name": "example-ci-01", "lifecycle": "experimental", - "engine_ref": "3d118432dce86d702fa1b78ecadcfcee750bfd9a", + "engine_ref": "f657117b1e1b5b94823d00fee4d35904c1ff8b72", "min_runners": 0, "max_runners": 1, "runner_resources": { From d76f2eb52f36766f3a6b9605f5f26709bfdbacc9 Mon Sep 17 00:00:00 2001 From: Nick Foster Date: Sat, 18 Jul 2026 19:03:53 -0500 Subject: [PATCH 17/57] fix: pin examples to compatible engine release --- templates/config-repository/examples/multi-host/fleet.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/config-repository/examples/multi-host/fleet.json b/templates/config-repository/examples/multi-host/fleet.json index 40b7c0d5..9207950c 100644 --- a/templates/config-repository/examples/multi-host/fleet.json +++ b/templates/config-repository/examples/multi-host/fleet.json @@ -24,7 +24,7 @@ "state": "active", "scale_set_name": "sample-ci-primary", "lifecycle": "stable", - "engine_ref": "3d118432dce86d702fa1b78ecadcfcee750bfd9a", + "engine_ref": "f657117b1e1b5b94823d00fee4d35904c1ff8b72", "min_runners": 0, "max_runners": 4, "runner_resources": { @@ -38,7 +38,7 @@ "state": "active", "scale_set_name": "sample-ci-remote", "lifecycle": "stable", - "engine_ref": "3d118432dce86d702fa1b78ecadcfcee750bfd9a", + "engine_ref": "f657117b1e1b5b94823d00fee4d35904c1ff8b72", "min_runners": 0, "max_runners": 2, "runner_resources": { From aefa7e2825024c153aa8d98fb68fcd91f9047a9f Mon Sep 17 00:00:00 2001 From: Nickfosts Hermes Date: Sat, 18 Jul 2026 19:06:49 -0500 Subject: [PATCH 18/57] fix(installer): validate the candidate release --- scripts/install-worker-controller.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/install-worker-controller.sh b/scripts/install-worker-controller.sh index 708084e2..b2824537 100755 --- a/scripts/install-worker-controller.sh +++ b/scripts/install-worker-controller.sh @@ -357,7 +357,7 @@ run_candidate_preflight() { # shellcheck disable=SC1090 . "$candidate_env" set +a - CI_FLEET_TESTING=$testing "$repo_root/scripts/preflight.sh" --managed + CI_FLEET_TESTING=$testing "$release_dir/scripts/preflight.sh" --managed ) } @@ -476,7 +476,7 @@ activate_candidate() { # shellcheck disable=SC1090 . "$rendered_env" set +a - "$repo_root/scripts/healthcheck.sh" + "$release_dir/scripts/healthcheck.sh" ); then die 'post-activation health check failed' fi From 2f0ddec59d30b57d2aa6c5de8d5e5ee72eec8db1 Mon Sep 17 00:00:00 2001 From: Nickfosts Hermes Date: Sat, 18 Jul 2026 19:58:07 -0500 Subject: [PATCH 19/57] fix: harden desired-state trust boundaries --- .github/workflows/validate.yml | 10 ++++++ docs/DESIRED-STATE.md | 6 ++-- scripts/desired_state.py | 3 ++ scripts/install-worker-controller.sh | 35 +++++++++++++++---- scripts/scan_committed_secrets.py | 3 +- scripts/test-install-worker-controller.sh | 23 +++++++++++- scripts/test_desired_state.py | 6 ++++ templates/config-repository/scripts/init.py | 22 ++++++++---- .../config-repository/scripts/test_policy.py | 5 +++ .../config-repository/scripts/validate.py | 2 ++ 10 files changed, 97 insertions(+), 18 deletions(-) diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index f07e63f4..705b5095 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -39,6 +39,16 @@ jobs: templates/config-repository/scripts/validate.sh templates/config-repository/scripts/validate.sh --config templates/config-repository/examples/multi-host/fleet.json temporary_directory="$(mktemp -d)" + if templates/config-repository/scripts/init.sh \ + --organization test-company \ + --project test-app \ + --registry BAD \ + --engine-ref "$(git rev-parse HEAD)" \ + --output "${temporary_directory}/invalid.json"; then + echo 'initializer accepted an invalid registry' >&2 + exit 1 + fi + test ! -e "${temporary_directory}/invalid.json" templates/config-repository/scripts/init.sh \ --organization test-company \ --project test-app \ diff --git a/docs/DESIRED-STATE.md b/docs/DESIRED-STATE.md index 8f81ceba..ed21011b 100644 --- a/docs/DESIRED-STATE.md +++ b/docs/DESIRED-STATE.md @@ -56,7 +56,7 @@ sudo install -m 0600 host/host.env.example /etc/ci-fleet/host.env sudo install -m 0600 /secure/source/github-app.pem /etc/ci-fleet/secrets/github-app.pem ``` -Edit `/etc/ci-fleet/host.env` locally. It contains only the GitHub App client ID, installation ID, private-key path, and runner TTL. It is never committed. The PEM remains separate and is never printed by the installer. +Edit `/etc/ci-fleet/host.env` locally. It contains only the GitHub App client ID, installation ID, private-key path, and runner TTL. Both the host environment and PEM must be root-owned mode `0600`, and the TTL must be at least one hour. Neither file is committed, and the PEM is never printed by the installer. GitHub App and runner-group creation remain the bootstrap responsibility tracked by [issue #27](https://github.com/RandomDevelopment/ci-fleet/issues/27). The installer fails closed when those prerequisites are absent. @@ -132,7 +132,7 @@ No mode uses global Docker prune or removes unrelated workloads. ## Drift and reviewed updates -`ci-fleet-drift.timer` checks the host every fifteen minutes against the exact configuration SHA recorded at installation. It detects host edits, missing releases, runtime-state mismatch, altered metadata, and missing maintenance timers without applying changes. +`ci-fleet-drift.timer` checks the host every fifteen minutes against the exact configuration SHA recorded at installation. It detects host edits, missing or stale runtime and installer-manager releases, runtime-state mismatch, altered metadata, and missing maintenance timers without applying changes. The host does not automatically follow or execute a moving branch. A new configuration becomes effective only when an operator or authorized external controller supplies its reviewed full commit SHA to `--upgrade`. Automatic dispatchers may watch a protected branch and invoke that exact command after merge, using read-only repository contents permission; their identity must remain host-side and unavailable to job runners. @@ -149,7 +149,7 @@ Legacy project-specific hosts remain until CI, promotion, and deployment no long ## Failure and recovery behavior -Before mutation, the installer records the prior rendered environment, installation metadata, runtime release, installer-manager release, and maintenance unit/timer state under `/var/lib/ci-fleet/checkpoints`. Build and validation happen before the active release changes. A failed activation or health check drains the candidate, restores those artifacts, restarts the prior controller only when no managed runner is active, and verifies prior-release health before reporting rollback success. A host-local `flock` prevents overlapping installer and drift-check processes. +Before mutation, the installer records the prior rendered environment, installation metadata, runtime release, installer-manager release, and maintenance unit/timer state under `/var/lib/ci-fleet/checkpoints`. Build and validation happen before the active release changes. A failed activation or health check drains the candidate, restores those artifacts, restarts the prior controller only when no managed runner is active, and verifies prior-release health before reporting rollback success. A host-local installer lock serializes every check and mutation. Runtime and installer-manager releases are staged on their respective target filesystems and renamed atomically so a failed copy cannot masquerade as an installed immutable release. These controller checkpoints do not replace machine backups. Operators still create and verify VM snapshots, physical-host recovery media, or equivalent infrastructure backups according to their local policy. diff --git a/scripts/desired_state.py b/scripts/desired_state.py index 77ae16ad..33481598 100755 --- a/scripts/desired_state.py +++ b/scripts/desired_state.py @@ -98,6 +98,9 @@ def validate_host_values(values: dict[str, str]) -> dict[str, str]: ttl = values.get("CI_FLEET_RUNNER_TTL", "6h") if not SAFE_DURATION.fullmatch(ttl): raise DesiredStateError("runner TTL must be a positive duration ending in s, m, or h") + multiplier = {"s": 1, "m": 60, "h": 3600}[ttl[-1]] + if int(ttl[:-1]) * multiplier < 3600: + raise DesiredStateError("runner TTL must be at least one hour") return { "CI_FLEET_GITHUB_APP_CLIENT_ID": client_id, "CI_FLEET_GITHUB_APP_INSTALLATION_ID": installation_id, diff --git a/scripts/install-worker-controller.sh b/scripts/install-worker-controller.sh index b2824537..dc7a4a88 100755 --- a/scripts/install-worker-controller.sh +++ b/scripts/install-worker-controller.sh @@ -12,6 +12,7 @@ root_prefix=${CI_FLEET_ROOT_PREFIX:-} testing=${CI_FLEET_TESTING:-0} transaction_active=false checkpoint_dir= +staging_paths=() usage() { cat >&2 <<'EOF' @@ -112,7 +113,13 @@ unit_names=( timer_names=(ci-fleet-health.timer ci-fleet-cleanup.timer ci-fleet-drift.timer) temporary=$(mktemp -d) -cleanup_temporary() { rm -rf "$temporary"; } +cleanup_temporary() { + local path + rm -rf "$temporary" + for path in "${staging_paths[@]}"; do + [[ -z "$path" ]] || rm -rf -- "$path" + done +} trap cleanup_temporary EXIT require_commands() { @@ -176,14 +183,16 @@ prepare_host_config() { verify_host_files() { local mode_bits owner expected_owner key_file + expected_owner=0 + [[ "$testing" != 1 ]] || expected_owner=$(id -u) mode_bits=$(stat -c '%a' "$effective_host_config") + owner=$(stat -c '%u' "$effective_host_config") [[ "$mode_bits" == 600 ]] || die "host configuration must have mode 0600: $effective_host_config" + [[ "$owner" == "$expected_owner" ]] || die 'host configuration must be owned by root' key_file=$(awk -F= '$1 == "CI_FLEET_GITHUB_APP_PRIVATE_KEY_FILE" {print substr($0, index($0, "=") + 1)}' "$effective_host_config") [[ -n "$key_file" && -f "$key_file" ]] || die 'GitHub App PEM file is missing' mode_bits=$(stat -c '%a' "$key_file") owner=$(stat -c '%u' "$key_file") - expected_owner=0 - [[ "$testing" != 1 ]] || expected_owner=$(id -u) [[ "$mode_bits" == 600 && "$owner" == "$expected_owner" ]] || die 'GitHub App PEM must be owned by root and have mode 0600' } @@ -270,12 +279,22 @@ PY } release_matches() { + local marker + [[ -d "$release_dir" && -f "$release_dir/.ci-fleet-engine-ref" && -f "$release_dir/deploy/compose.yaml" ]] || return 1 + marker=$(<"$release_dir/.ci-fleet-engine-ref") + [[ "$marker" == "$engine_ref" ]] || return 1 [[ -L "$current_link" ]] || return 1 [[ $(readlink -f "$current_link") == $(readlink -f "$release_dir") ]] } systemd_matches() { + local expected_manager marker + expected_manager=$manager_releases/$engine_ref + [[ -d "$expected_manager" && -f "$expected_manager/.ci-fleet-engine-ref" ]] || return 1 + marker=$(<"$expected_manager/.ci-fleet-engine-ref") + [[ "$marker" == "$engine_ref" ]] || return 1 [[ -L "$manager_current" ]] || return 1 + [[ $(readlink -f "$manager_current") == $(readlink -f "$expected_manager") ]] || return 1 [[ -f "$systemd_dir/ci-fleet-health.service" && -f "$systemd_dir/ci-fleet-cleanup.service" && -f "$systemd_dir/ci-fleet-drift.service" ]] || return 1 systemctl is-enabled --quiet ci-fleet-health.timer ci-fleet-cleanup.timer ci-fleet-drift.timer || return 1 systemctl is-active --quiet ci-fleet-health.timer ci-fleet-cleanup.timer ci-fleet-drift.timer @@ -316,8 +335,9 @@ install_release() { [[ "$resolved" == "$engine_ref" ]] || die 'fetched ci-fleet engine commit does not match desired state' git -C "$checkout" archive --format=tar --output "$archive" FETCH_HEAD fi - staged_release=$temporary/engine-release - install -d -m 0755 "$staged_release" + staged_release=$(mktemp -d "$releases_dir/.${engine_ref}.staging.XXXXXX") + staging_paths+=("$staged_release") + chmod 0755 "$staged_release" tar -xf "$archive" -C "$staged_release" printf '%s\n' "$engine_ref" >"$staged_release/.ci-fleet-engine-ref" chmod 0644 "$staged_release/.ci-fleet-engine-ref" @@ -336,8 +356,9 @@ install_manager() { install -d -m 0755 "$manager_releases" archive=$temporary/manager.tar tar -cf "$archive" -C "$release_dir" . - staged_manager=$temporary/manager-release - install -d -m 0755 "$staged_manager" + staged_manager=$(mktemp -d "$manager_releases/.${manager_commit}.staging.XXXXXX") + staging_paths+=("$staged_manager") + chmod 0755 "$staged_manager" tar -xf "$archive" -C "$staged_manager" printf '%s\n' "$manager_commit" >"$staged_manager/.ci-fleet-engine-ref" chmod 0644 "$staged_manager/.ci-fleet-engine-ref" diff --git a/scripts/scan_committed_secrets.py b/scripts/scan_committed_secrets.py index bc6cabdc..00acf580 100644 --- a/scripts/scan_committed_secrets.py +++ b/scripts/scan_committed_secrets.py @@ -18,8 +18,9 @@ PATTERN = re.compile( rb"BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY|" rb"github_pat_[A-Za-z0-9_]{20,}|" - rb"ghp_[A-Za-z0-9]{20,}" + rb"gh[opusr]_[A-Za-z0-9]{20,}" ) +assert all(PATTERN.search(prefix + b"x" * 20) for prefix in (b"gho_", b"ghp_", b"ghr_", b"ghs_", b"ghu_")) def main() -> int: diff --git a/scripts/test-install-worker-controller.sh b/scripts/test-install-worker-controller.sh index 36702bba..94958d45 100755 --- a/scripts/test-install-worker-controller.sh +++ b/scripts/test-install-worker-controller.sh @@ -6,6 +6,8 @@ tmp=$(mktemp -d) trap 'rm -rf "$tmp"' EXIT fake_bin=$tmp/bin mkdir -p "$fake_bin" +export REAL_STAT +REAL_STAT=$(command -v stat) cat >"$fake_bin/docker" <<'EOF' #!/usr/bin/env bash @@ -56,7 +58,16 @@ cat >"$fake_bin/systemctl" <<'EOF' #!/usr/bin/env bash exit 0 EOF -chmod 700 "$fake_bin/docker" "$fake_bin/systemctl" + +cat >"$fake_bin/stat" <<'EOF' +#!/usr/bin/env bash +if [[ -n "${FAKE_WRONG_HOST_CONFIG_OWNER:-}" && "${1:-}" == -c && "${2:-}" == %u && "${3:-}" == "$FAKE_WRONG_HOST_CONFIG_OWNER" ]]; then + printf '99999\n' + exit 0 +fi +exec "$REAL_STAT" "$@" +EOF +chmod 700 "$fake_bin/docker" "$fake_bin/systemctl" "$fake_bin/stat" export PATH="$fake_bin:$PATH" export FAKE_DOCKER_STATE=$tmp/docker-controller-running @@ -124,6 +135,10 @@ ref_one=$(write_config active 1 1) installer=$repo_root/scripts/install-worker-controller.sh base_args=(--config-repo "$config_repo" --controller example-ci-01) +export FAKE_WRONG_HOST_CONFIG_OWNER=$host_config +expect_failure 'host configuration must be owned by root' "$installer" --install "${base_args[@]}" --ref "$ref_one" +unset FAKE_WRONG_HOST_CONFIG_OWNER + first=$(expect_success "$installer" --install "${base_args[@]}" --ref "$ref_one") grep -Fq 'CONVERGED mode=install' <<<"$first" || fail 'fresh install did not converge' [[ -L "$root/opt/ci-fleet/current" && -f "$root/var/lib/ci-fleet/install-state.json" ]] || fail 'fresh install state is incomplete' @@ -134,6 +149,11 @@ second=$(expect_success "$installer" --install "${base_args[@]}" --ref "$ref_one grep -Fq 'NO_CHANGE' <<<"$second" || fail 'idempotent rerun changed the host' expect_success "$installer" --check "${base_args[@]}" --ref "$ref_one" >/dev/null +active_release=$(readlink -f "$root/opt/ci-fleet/current") +mv "$active_release" "$active_release.saved" +expect_failure 'DRIFT engine_release' "$installer" --check "${base_args[@]}" --ref "$ref_one" +mv "$active_release.saved" "$active_release" + relative=$(cd "$tmp" && expect_success "$installer" --install --config-repo config-repo --controller example-ci-01 --ref "$ref_one") grep -Fq 'NO_CHANGE' <<<"$relative" || fail 'relative configuration path was not normalized before drift comparison' grep -Fq "CI_FLEET_CONFIG_REPOSITORY=$config_repo" "$root/etc/ci-fleet/ci-fleet.env" || fail 'rendered configuration path is not absolute' @@ -145,6 +165,7 @@ expect_success "$installer" --install "${base_args[@]}" --ref "$ref_one" >/dev/n prior_manager=$root/opt/ci-fleet/manager/releases/prior-manager cp -a "$(readlink -f "$root/opt/ci-fleet/manager/current")" "$prior_manager" ln -sfn "$prior_manager" "$root/opt/ci-fleet/manager/current" +expect_failure 'DRIFT maintenance_timers' "$installer" --check "${base_args[@]}" --ref "$ref_one" ref_two=$(write_config active 2 2) export FAKE_RUNNER_STATE=$tmp/managed-runner-active diff --git a/scripts/test_desired_state.py b/scripts/test_desired_state.py index 8dce0aa2..70ef6f2b 100755 --- a/scripts/test_desired_state.py +++ b/scripts/test_desired_state.py @@ -126,6 +126,12 @@ def test_host_values_require_absolute_key_path(self) -> None: with self.assertRaisesRegex(DesiredStateError, "absolute shell-safe"): validate_host_values(values) + def test_host_values_reject_sub_hour_ttl(self) -> None: + values = copy.deepcopy(host_values()) + values["CI_FLEET_RUNNER_TTL"] = "30m" + with self.assertRaisesRegex(DesiredStateError, "at least one hour"): + validate_host_values(values) + def test_shell_metacharacter_in_config_identity_is_rejected(self) -> None: with self.assertRaisesRegex(DesiredStateError, "must be shell-safe"): build_rendered_env( diff --git a/templates/config-repository/scripts/init.py b/templates/config-repository/scripts/init.py index 15618216..83d75f29 100755 --- a/templates/config-repository/scripts/init.py +++ b/templates/config-repository/scripts/init.py @@ -5,8 +5,10 @@ import argparse import json +import os import re import subprocess +import tempfile from pathlib import Path @@ -154,12 +156,20 @@ def main() -> int: }, } output.parent.mkdir(parents=True, exist_ok=True) - output.write_text(json.dumps(config, indent=2) + "\n", encoding="utf-8") - - subprocess.run( - [str(ROOT / "scripts" / "validate.sh"), "--strict", "--skip-path-scan", "--config", str(output)], - check=True, - ) + descriptor, temporary_name = tempfile.mkstemp(prefix=f".{output.name}.", dir=output.parent, text=True) + temporary = Path(temporary_name) + try: + with os.fdopen(descriptor, "w", encoding="utf-8") as handle: + handle.write(json.dumps(config, indent=2) + "\n") + handle.flush() + os.fsync(handle.fileno()) + subprocess.run( + [str(ROOT / "scripts" / "validate.sh"), "--strict", "--skip-path-scan", "--config", str(temporary)], + check=True, + ) + os.replace(temporary, output) + finally: + temporary.unlink(missing_ok=True) print(f"Initialized {output}") print("Next: review controller capacity, configure GitHub policy, and keep every secret value outside Git.") return 0 diff --git a/templates/config-repository/scripts/test_policy.py b/templates/config-repository/scripts/test_policy.py index 84d0e184..004fcf4c 100755 --- a/templates/config-repository/scripts/test_policy.py +++ b/templates/config-repository/scripts/test_policy.py @@ -83,6 +83,11 @@ def test_duplicate_scale_set_is_rejected(self) -> None: config["runner_pools"]["trusted-ci"]["capacity_budget"] = 2 self.assert_rejected(config, "scale_set_name: must be unique") + def test_scale_set_must_include_controller_id(self) -> None: + config = copy.deepcopy(reference_config()) + first_controller(config)["scale_set_name"] = "other-scale" + self.assert_rejected(config, "must include the controller ID") + def test_controller_pool_must_exist(self) -> None: config = copy.deepcopy(reference_config()) first_controller(config)["pool"] = "missing-pool" diff --git a/templates/config-repository/scripts/validate.py b/templates/config-repository/scripts/validate.py index 8cc535ed..f59718c4 100755 --- a/templates/config-repository/scripts/validate.py +++ b/templates/config-repository/scripts/validate.py @@ -254,6 +254,8 @@ def validate_config(config: Any, validation: Validation, strict: bool) -> None: validation.require(isinstance(location, str) and bool(SLUG.fullmatch(location)), f"{path}.location", "must be a logical location slug, never an address") validation.require(state in {"active", "drained", "disabled"}, f"{path}.state", "must be active, drained, or disabled") validation.require(isinstance(scale_set, str) and bool(SLUG.fullmatch(scale_set)), f"{path}.scale_set_name", "must be a lowercase scale-set slug") + if isinstance(scale_set, str) and isinstance(name, str): + validation.require(name in scale_set, f"{path}.scale_set_name", "must include the controller ID required by managed preflight") if isinstance(scale_set, str): if scale_set in scale_sets: validation.errors.append(f"{path}.scale_set_name: must be unique; also used by {scale_sets[scale_set]}") From 67fdf67888328d8a3e891301f6f7df88c0201b5f Mon Sep 17 00:00:00 2001 From: Nickfosts Hermes Date: Sat, 18 Jul 2026 20:28:46 -0500 Subject: [PATCH 20/57] fix: close controller trust gaps --- .github/workflows/validate.yml | 1 + docs/DESIRED-STATE.md | 4 +- scripts/desired_state.py | 1 + scripts/install-worker-controller.sh | 39 +++++++++++-------- scripts/scan_committed_secrets.py | 2 +- scripts/test-install-worker-controller.sh | 17 +++++++- scripts/test_desired_state.py | 1 + templates/config-repository/scripts/init.py | 1 + .../config-repository/scripts/test_policy.py | 12 +++++- .../config-repository/scripts/validate.py | 9 +++-- 10 files changed, 62 insertions(+), 25 deletions(-) diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 705b5095..fcf65256 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -54,3 +54,4 @@ jobs: --project test-app \ --engine-ref "$(git rev-parse HEAD)" \ --output "${temporary_directory}/fleet.json" + test "$(stat -c '%a' "${temporary_directory}/fleet.json")" = 644 diff --git a/docs/DESIRED-STATE.md b/docs/DESIRED-STATE.md index ed21011b..af3d6f78 100644 --- a/docs/DESIRED-STATE.md +++ b/docs/DESIRED-STATE.md @@ -1,6 +1,6 @@ # Git-authored controller desired state -Schema v3 makes a reviewed private configuration repository the authority for controller routing, capacity, lifecycle, and engine revision. The target host keeps credentials and rendered runtime state outside Git. +Schema v3 makes a reviewed private configuration repository the authority for controller routing, capacity, lifecycle, and engine revision within the fixed `RandomDevelopment/ci-fleet` public engine repository. A configuration-only change cannot redirect root execution to another repository. The target host keeps credentials and rendered runtime state outside Git. ## Responsibility boundary @@ -82,7 +82,7 @@ The installer: 4. renders `/etc/ci-fleet/ci-fleet.env` without secret values; 5. fetches and verifies the pinned public engine commit; 6. creates a root-only controller checkpoint; -7. drains the current controller and waits for managed runners to finish; +7. drains the current controller and waits for every managed runner to finish, including orphaned runners left after a stopped or crashed controller; 8. runs managed preflight and builds the pinned runner and controller images; 9. installs health, cleanup, and pinned-state drift timers; 10. starts the controller only when its desired state is active; diff --git a/scripts/desired_state.py b/scripts/desired_state.py index 33481598..42c66bbd 100755 --- a/scripts/desired_state.py +++ b/scripts/desired_state.py @@ -143,6 +143,7 @@ def build_rendered_env( short_commit = engine_commit[:12] rendered = { "CI_FLEET_CAPACITY_BUDGET": str(pool["capacity_budget"]), + "CI_FLEET_COMMIT": engine_commit, "CI_FLEET_CONFIGURED_MAX_RUNNERS": str(configured_max), "CI_FLEET_CONFIG_REF": config_ref, "CI_FLEET_CONFIG_REPOSITORY": config_repository, diff --git a/scripts/install-worker-controller.sh b/scripts/install-worker-controller.sh index dc7a4a88..6f963379 100755 --- a/scripts/install-worker-controller.sh +++ b/scripts/install-worker-controller.sh @@ -168,12 +168,8 @@ prepare_host_config() { if [[ -f "$host_config" ]]; then return fi - if [[ -f "$rendered_env" && ( "$mode" == adopt || "$mode" == check ) ]]; then - if [[ "$mode" == check ]]; then - effective_host_config=$temporary/host.env - else - install -d -m 0700 "$etc_dir" - fi + if [[ -f "$rendered_env" && "$mode" == adopt ]]; then + install -d -m 0700 "$etc_dir" python3 "$repo_root/scripts/desired_state.py" extract-host-env \ --source "$rendered_env" --output "$effective_host_config" return @@ -229,6 +225,7 @@ PY target_state=${metadata_values[0]} engine_ref=${metadata_values[1]} engine_repository=${metadata_values[2]} + [[ "$engine_repository" == RandomDevelopment/ci-fleet ]] || die 'delivery engine repository is not the fixed reviewed public engine' release_dir=$releases_dir/$engine_ref } @@ -288,14 +285,17 @@ release_matches() { } systemd_matches() { - local expected_manager marker + local expected_manager marker unit expected_manager=$manager_releases/$engine_ref [[ -d "$expected_manager" && -f "$expected_manager/.ci-fleet-engine-ref" ]] || return 1 marker=$(<"$expected_manager/.ci-fleet-engine-ref") [[ "$marker" == "$engine_ref" ]] || return 1 [[ -L "$manager_current" ]] || return 1 [[ $(readlink -f "$manager_current") == $(readlink -f "$expected_manager") ]] || return 1 - [[ -f "$systemd_dir/ci-fleet-health.service" && -f "$systemd_dir/ci-fleet-cleanup.service" && -f "$systemd_dir/ci-fleet-drift.service" ]] || return 1 + for unit in "${unit_names[@]}"; do + [[ -f "$systemd_dir/$unit" ]] || return 1 + cmp -s "$expected_manager/host/systemd/$unit" "$systemd_dir/$unit" || return 1 + done systemctl is-enabled --quiet ci-fleet-health.timer ci-fleet-cleanup.timer ci-fleet-drift.timer || return 1 systemctl is-active --quiet ci-fleet-health.timer ci-fleet-cleanup.timer ci-fleet-drift.timer } @@ -316,7 +316,7 @@ drift_count() { install_release() { local archive marker_commit checkout resolved staged_release if [[ -d "$release_dir" ]]; then - [[ -f "$release_dir/.ci-fleet-engine-ref" ]] || die "existing release has no engine marker: $release_dir" + [[ -f "$release_dir/.ci-fleet-engine-ref" && -f "$release_dir/deploy/compose.yaml" && -x "$release_dir/scripts/preflight.sh" && -x "$release_dir/scripts/healthcheck.sh" ]] || die "existing release is incomplete: $release_dir" marker_commit=$(<"$release_dir/.ci-fleet-engine-ref") [[ "$marker_commit" == "$engine_ref" ]] || die "existing release marker does not match $engine_ref" return @@ -364,6 +364,7 @@ install_manager() { chmod 0644 "$staged_manager/.ci-fleet-engine-ref" mv "$staged_manager" "$manager_release" else + [[ -f "$manager_release/.ci-fleet-engine-ref" && -x "$manager_release/scripts/install-worker-controller.sh" ]] || die 'existing installer manager release is incomplete' marker=$(<"$manager_release/.ci-fleet-engine-ref") [[ "$marker" == "$manager_commit" ]] || die 'existing installer manager marker is inconsistent' fi @@ -420,26 +421,32 @@ make_checkpoint() { } try_drain_current() { - local deadline count old_release status + local deadline count old_release status paused=false drain_error= status=$(controller_status) - [[ "$status" == running ]] || return 0 - if [[ ! -f "$rendered_env" ]]; then drain_error='cannot safely drain a running controller without its rendered environment'; return 1; fi - old_release=$(current_runtime_release) - if [[ -z "$old_release" || ! -f "$old_release/deploy/compose.yaml" ]]; then drain_error='cannot locate the running controller Compose release for safe adoption'; return 1; fi - if ! compose "$old_release" "$rendered_env" pause controller >/dev/null; then drain_error='could not pause the controller for drain'; return 1; fi + if [[ "$status" == running ]]; then + if [[ ! -f "$rendered_env" ]]; then drain_error='cannot safely drain a running controller without its rendered environment'; return 1; fi + old_release=$(current_runtime_release) + if [[ -z "$old_release" || ! -f "$old_release/deploy/compose.yaml" ]]; then drain_error='cannot locate the running controller Compose release for safe adoption'; return 1; fi + if ! compose "$old_release" "$rendered_env" pause controller >/dev/null; then drain_error='could not pause the controller for drain'; return 1; fi + paused=true + fi deadline=$((SECONDS + ${CI_FLEET_DRAIN_TIMEOUT_SECONDS:-300})) while :; do count=$(managed_runner_count) if [[ "$count" == 0 ]]; then break; fi if ((SECONDS >= deadline)); then - compose "$old_release" "$rendered_env" unpause controller >/dev/null || true + if [[ "$paused" == true ]]; then compose "$old_release" "$rendered_env" unpause controller >/dev/null || true; fi drain_error="drain timed out with $count managed runner(s) still present" return 1 fi sleep 2 done note 'DRAIN_READY managed_runners=0' + if [[ "$status" != running ]]; then + note 'DRAIN_OK managed_runners=0' + return 0 + fi docker compose --project-name ci-fleet --env-file "$rendered_env" -f "$old_release/deploy/compose.yaml" kill --signal SIGTERM controller >/dev/null || { drain_error='failed to signal the paused controller for graceful scale-set cleanup' return 1 diff --git a/scripts/scan_committed_secrets.py b/scripts/scan_committed_secrets.py index 00acf580..8a28dc94 100644 --- a/scripts/scan_committed_secrets.py +++ b/scripts/scan_committed_secrets.py @@ -34,7 +34,7 @@ def main() -> int: if not raw: continue relative = raw.decode("utf-8") - if relative in EXCLUDED or relative.endswith(".example"): + if relative in EXCLUDED: continue data = (ROOT / relative).read_bytes() for match in PATTERN.finditer(data): diff --git a/scripts/test-install-worker-controller.sh b/scripts/test-install-worker-controller.sh index 94958d45..71aa3dcd 100755 --- a/scripts/test-install-worker-controller.sh +++ b/scripts/test-install-worker-controller.sh @@ -20,7 +20,12 @@ case "${1:-}" in if [[ "$*" == *'.State.Status'* ]]; then printf 'running\n'; else printf 'running\n'; fi ;; ps) - if [[ -n "${FAKE_RUNNER_STATE:-}" && -f "$FAKE_RUNNER_STATE" ]]; then printf 'managed-runner\n'; fi + if [[ -n "${FAKE_RUNNER_STATE_ONCE:-}" && -f "$FAKE_RUNNER_STATE_ONCE" ]]; then + rm -f "$FAKE_RUNNER_STATE_ONCE" + printf 'managed-runner\n' + elif [[ -n "${FAKE_RUNNER_STATE:-}" && -f "$FAKE_RUNNER_STATE" ]]; then + printf 'managed-runner\n' + fi exit 0 ;; volume|network) @@ -145,6 +150,10 @@ grep -Fq 'CONVERGED mode=install' <<<"$first" || fail 'fresh install did not con [[ $(readlink -f "$root/opt/ci-fleet/manager/current") == "$root/opt/ci-fleet/manager/releases/$engine_ref" ]] || fail 'installer manager did not activate the desired engine release' [[ -f "$FAKE_DOCKER_STATE" ]] || fail 'active controller was not started' +mv "$host_config" "$host_config.missing" +expect_failure 'host-local GitHub App configuration is missing' "$installer" --check "${base_args[@]}" --ref "$ref_one" +mv "$host_config.missing" "$host_config" + second=$(expect_success "$installer" --install "${base_args[@]}" --ref "$ref_one") grep -Fq 'NO_CHANGE' <<<"$second" || fail 'idempotent rerun changed the host' expect_success "$installer" --check "${base_args[@]}" --ref "$ref_one" >/dev/null @@ -159,7 +168,9 @@ grep -Fq 'NO_CHANGE' <<<"$relative" || fail 'relative configuration path was not grep -Fq "CI_FLEET_CONFIG_REPOSITORY=$config_repo" "$root/etc/ci-fleet/ci-fleet.env" || fail 'rendered configuration path is not absolute' printf '\n' >>"$root/etc/ci-fleet/ci-fleet.env" +printf '\n# drift\n' >>"$root/etc/systemd/system/ci-fleet-health.timer" expect_failure 'DRIFT rendered_environment' "$installer" --check "${base_args[@]}" --ref "$ref_one" +expect_failure 'DRIFT maintenance_timers' "$installer" --check "${base_args[@]}" --ref "$ref_one" expect_success "$installer" --install "${base_args[@]}" --ref "$ref_one" >/dev/null prior_manager=$root/opt/ci-fleet/manager/releases/prior-manager @@ -191,7 +202,11 @@ grep -Fq 'CI_FLEET_CONTROLLER_STATE=drained' "$root/etc/ci-fleet/ci-fleet.env" | grep -Fq 'CI_FLEET_MAX_RUNNERS=0' "$root/etc/ci-fleet/ci-fleet.env" || fail 'drained controller retained effective capacity' [[ ! -f "$FAKE_DOCKER_STATE" ]] || fail 'drained controller remained running' +export FAKE_RUNNER_STATE_ONCE=$tmp/orphaned-managed-runner +: >"$FAKE_RUNNER_STATE_ONCE" expect_success "$installer" --uninstall >/dev/null +[[ ! -f "$FAKE_RUNNER_STATE_ONCE" ]] || fail 'uninstall did not wait for an orphaned managed runner' +unset FAKE_RUNNER_STATE_ONCE [[ ! -e "$root/opt/ci-fleet/current" && ! -e "$root/var/lib/ci-fleet/install-state.json" ]] || fail 'uninstall left active installation state' [[ -f "$host_config" && -f "$pem" ]] || fail 'uninstall removed preserved host credentials' diff --git a/scripts/test_desired_state.py b/scripts/test_desired_state.py index 70ef6f2b..5e7eab51 100755 --- a/scripts/test_desired_state.py +++ b/scripts/test_desired_state.py @@ -51,6 +51,7 @@ def test_active_controller_renders_configured_capacity(self) -> None: self.assertEqual(environment["CI_FLEET_MAX_RUNNERS"], "1") self.assertEqual(environment["CI_FLEET_CONFIGURED_MAX_RUNNERS"], "1") self.assertEqual(environment["CI_FLEET_LABELS"], "docker-ci") + self.assertEqual(environment["CI_FLEET_COMMIT"], environment["CI_FLEET_ENGINE_REF"]) self.assertEqual(metadata["controller_state"], "active") def test_drained_controller_renders_zero_effective_capacity(self) -> None: diff --git a/templates/config-repository/scripts/init.py b/templates/config-repository/scripts/init.py index 83d75f29..60063de6 100755 --- a/templates/config-repository/scripts/init.py +++ b/templates/config-repository/scripts/init.py @@ -167,6 +167,7 @@ def main() -> int: [str(ROOT / "scripts" / "validate.sh"), "--strict", "--skip-path-scan", "--config", str(temporary)], check=True, ) + os.chmod(temporary, 0o644) os.replace(temporary, output) finally: temporary.unlink(missing_ok=True) diff --git a/templates/config-repository/scripts/test_policy.py b/templates/config-repository/scripts/test_policy.py index 004fcf4c..43cb067c 100755 --- a/templates/config-repository/scripts/test_policy.py +++ b/templates/config-repository/scripts/test_policy.py @@ -90,9 +90,19 @@ def test_scale_set_must_include_controller_id(self) -> None: def test_controller_pool_must_exist(self) -> None: config = copy.deepcopy(reference_config()) - first_controller(config)["pool"] = "missing-pool" + first_controller(config)["pool"] = "missing" self.assert_rejected(config, "must reference a declared runner pool") + def test_controller_pool_must_be_a_string(self) -> None: + config = copy.deepcopy(reference_config()) + first_controller(config)["pool"] = ["trusted-ci"] + self.assert_rejected(config, "must reference a declared runner pool") + + def test_delivery_engine_repository_is_fixed(self) -> None: + config = copy.deepcopy(reference_config()) + config["organization"]["delivery_engine"] = "attacker/engine" + self.assert_rejected(config, "must use the fixed reviewed public engine repository") + def test_controller_address_is_rejected(self) -> None: config = copy.deepcopy(reference_config()) first_controller(config)["ip_address"] = "192.0.2.10" diff --git a/templates/config-repository/scripts/validate.py b/templates/config-repository/scripts/validate.py index f59718c4..44c89b77 100755 --- a/templates/config-repository/scripts/validate.py +++ b/templates/config-repository/scripts/validate.py @@ -176,6 +176,7 @@ def validate_config(config: Any, validation: Validation, strict: bool) -> None: validation.require(isinstance(slug, str) and bool(ORG_SLUG.fullmatch(slug)), "$.organization.slug", "must be a lowercase GitHub organization slug") validation.require(isinstance(registry, str) and bool(IMAGE.fullmatch(registry)), "$.organization.registry", "must be a registry namespace such as ghcr.io/acme") validation.require(isinstance(engine, str) and bool(REPOSITORY.fullmatch(engine)), "$.organization.delivery_engine", "must be an owner/repository name") + validation.require(engine == "RandomDevelopment/ci-fleet", "$.organization.delivery_engine", "must use the fixed reviewed public engine repository") validation.require(organization.get("workflow_ref_policy") == "immutable-commit", "$.organization.workflow_ref_policy", "must equal immutable-commit") if strict: validation.require(slug != "example-org", "$.organization.slug", "replace the example organization before use") @@ -250,7 +251,7 @@ def validate_config(config: Any, validation: Validation, strict: bool) -> None: engine_ref = controller.get("engine_ref") minimum = controller.get("min_runners") maximum = controller.get("max_runners") - validation.require(pool_name in pools, f"{path}.pool", "must reference a declared runner pool") + validation.require(isinstance(pool_name, str) and pool_name in pools, f"{path}.pool", "must reference a declared runner pool") validation.require(isinstance(location, str) and bool(SLUG.fullmatch(location)), f"{path}.location", "must be a logical location slug, never an address") validation.require(state in {"active", "drained", "disabled"}, f"{path}.state", "must be active, drained, or disabled") validation.require(isinstance(scale_set, str) and bool(SLUG.fullmatch(scale_set)), f"{path}.scale_set_name", "must be a lowercase scale-set slug") @@ -274,7 +275,7 @@ def validate_config(config: Any, validation: Validation, strict: bool) -> None: memory = resources.get("memory_mib") validation.require(type(cpu) is int and cpu > 0, f"{path}.runner_resources.cpu_cores", "must be a positive integer") validation.require(type(memory) is int and memory >= 512, f"{path}.runner_resources.memory_mib", "must be at least 512 MiB") - if pool_name in pools and state != "disabled" and type(maximum) is int and maximum > 0: + if isinstance(pool_name, str) and pool_name in pools and state != "disabled" and type(maximum) is int and maximum > 0: reserved_capacity[pool_name] += maximum for name, reserved in reserved_capacity.items(): @@ -330,8 +331,8 @@ def validate_config(config: Any, validation: Validation, strict: bool) -> None: pool_name = project.get("ci_pool") validation.require(isinstance(repository, str) and bool(REPOSITORY.fullmatch(repository)), f"{path}.repository", "must be owner/repository") validation.require(isinstance(image, str) and bool(IMAGE.fullmatch(image)), f"{path}.image", "must be a container image path without a mutable tag") - validation.require(pool_name in pools, f"{path}.ci_pool", "must reference a declared runner pool") - if pool_name in pools and isinstance(pools[pool_name].get("allowed_repositories"), list): + validation.require(isinstance(pool_name, str) and pool_name in pools, f"{path}.ci_pool", "must reference a declared runner pool") + if isinstance(pool_name, str) and pool_name in pools and isinstance(pools[pool_name].get("allowed_repositories"), list): validation.require(repository in pools[pool_name]["allowed_repositories"], f"{path}.repository", "must be explicitly allowed by its CI pool") contract = project.get("ci_contract") contract_path = f"{path}.ci_contract" From b8d5aac26cdae001b3ae5ff2e992862d14ddd227 Mon Sep 17 00:00:00 2001 From: Nickfosts Hermes Date: Sat, 18 Jul 2026 20:52:23 -0500 Subject: [PATCH 21/57] fix: make drift checks self-contained --- docs/DESIRED-STATE.md | 2 +- scripts/check-installed-state.sh | 27 ++++++++++++++++--- scripts/install-worker-controller.sh | 11 +++++--- scripts/test-install-worker-controller.sh | 9 +++++++ .../config-repository/scripts/test_policy.py | 5 ++++ .../config-repository/scripts/validate.py | 7 +++++ 6 files changed, 52 insertions(+), 9 deletions(-) diff --git a/docs/DESIRED-STATE.md b/docs/DESIRED-STATE.md index af3d6f78..8d2d48b8 100644 --- a/docs/DESIRED-STATE.md +++ b/docs/DESIRED-STATE.md @@ -56,7 +56,7 @@ sudo install -m 0600 host/host.env.example /etc/ci-fleet/host.env sudo install -m 0600 /secure/source/github-app.pem /etc/ci-fleet/secrets/github-app.pem ``` -Edit `/etc/ci-fleet/host.env` locally. It contains only the GitHub App client ID, installation ID, private-key path, and runner TTL. Both the host environment and PEM must be root-owned mode `0600`, and the TTL must be at least one hour. Neither file is committed, and the PEM is never printed by the installer. +Edit `/etc/ci-fleet/host.env` locally. Managed installs intentionally reject alternate host-config paths so scheduled drift checks always verify the same identity file. It contains only the GitHub App client ID, installation ID, private-key path, and runner TTL. Both the host environment and PEM must be root-owned mode `0600`, and the TTL must be at least one hour. Neither file is committed, and the PEM is never printed by the installer. GitHub App and runner-group creation remain the bootstrap responsibility tracked by [issue #27](https://github.com/RandomDevelopment/ci-fleet/issues/27). The installer fails closed when those prerequisites are absent. diff --git a/scripts/check-installed-state.sh b/scripts/check-installed-state.sh index bfcc5e5b..e75ee594 100755 --- a/scripts/check-installed-state.sh +++ b/scripts/check-installed-state.sh @@ -5,12 +5,31 @@ state_file=${CI_FLEET_INSTALL_STATE_FILE:-/var/lib/ci-fleet/install-state.json} installer=${CI_FLEET_INSTALLER:-/opt/ci-fleet/manager/current/scripts/install-worker-controller.sh} [[ -f "$state_file" ]] || { echo "ERROR: installed desired-state record is missing: $state_file" >&2; exit 2; } -command -v jq >/dev/null || { echo 'ERROR: jq is required' >&2; exit 2; } +command -v python3 >/dev/null || { echo 'ERROR: python3 is required' >&2; exit 2; } [[ -x "$installer" ]] || { echo "ERROR: installer is unavailable: $installer" >&2; exit 2; } -controller=$(jq -er .controller "$state_file") -config_repository=$(jq -er .config_repository "$state_file") -config_ref=$(jq -er .config_ref "$state_file") +if ! state_values=$(python3 - "$state_file" <<'PY' +import json +import sys + +try: + state = json.load(open(sys.argv[1], encoding="utf-8")) + values = [state[name] for name in ("controller", "config_repository", "config_ref")] + if not all(isinstance(value, str) and value for value in values): + raise ValueError +except (OSError, ValueError, KeyError, TypeError, json.JSONDecodeError): + raise SystemExit(2) +print("\n".join(values)) +PY +); then + echo "ERROR: installed desired-state record is invalid: $state_file" >&2 + exit 2 +fi +mapfile -t values <<<"$state_values" +[[ ${#values[@]} == 3 ]] || { echo "ERROR: installed desired-state record is incomplete: $state_file" >&2; exit 2; } +controller=${values[0]} +config_repository=${values[1]} +config_ref=${values[2]} exec "$installer" --check \ --config-repo "$config_repository" \ diff --git a/scripts/install-worker-controller.sh b/scripts/install-worker-controller.sh index 6f963379..16399426 100755 --- a/scripts/install-worker-controller.sh +++ b/scripts/install-worker-controller.sh @@ -19,14 +19,14 @@ usage() { usage: install-worker-controller.sh --check|--install|--adopt|--upgrade \ --config-repo OWNER/REPOSITORY|PATH --ref FULL_COMMIT_SHA \ - --controller CONTROLLER_ID [--host-config PATH] + --controller CONTROLLER_ID install-worker-controller.sh --rollback install-worker-controller.sh --uninstall Modes are mutually exclusive. Remote private repositories use the target host's preconfigured read-only Git credentials; credentials are never accepted in URLs -or command-line arguments. +or command-line arguments. Managed installs always use /etc/ci-fleet/host.env. EOF } @@ -135,6 +135,7 @@ validate_common_arguments() { [[ -n "$config_repo" ]] || die '--config-repo is required for this mode' [[ "$config_ref" =~ ^[0-9a-f]{40}$ ]] || die '--ref must be a full lowercase commit SHA' [[ "$controller_id" =~ ^[a-z0-9][a-z0-9-]{0,62}$ ]] || die '--controller must be a lowercase logical ID' + [[ -z "$host_config_arg" || "$host_config" == "$default_host_config" ]] || die 'managed installs require the default /etc/ci-fleet/host.env path' if [[ "$config_repo" == *://* || "$config_repo" == *@* ]]; then die '--config-repo must not contain a URL or embedded credentials; use OWNER/REPOSITORY or a local path' fi @@ -296,8 +297,10 @@ systemd_matches() { [[ -f "$systemd_dir/$unit" ]] || return 1 cmp -s "$expected_manager/host/systemd/$unit" "$systemd_dir/$unit" || return 1 done - systemctl is-enabled --quiet ci-fleet-health.timer ci-fleet-cleanup.timer ci-fleet-drift.timer || return 1 - systemctl is-active --quiet ci-fleet-health.timer ci-fleet-cleanup.timer ci-fleet-drift.timer + for unit in "${timer_names[@]}"; do + systemctl is-enabled --quiet "$unit" || return 1 + systemctl is-active --quiet "$unit" || return 1 + done } drift_count() { diff --git a/scripts/test-install-worker-controller.sh b/scripts/test-install-worker-controller.sh index 71aa3dcd..f235d373 100755 --- a/scripts/test-install-worker-controller.sh +++ b/scripts/test-install-worker-controller.sh @@ -61,6 +61,9 @@ EOF cat >"$fake_bin/systemctl" <<'EOF' #!/usr/bin/env bash +if [[ -n "${FAKE_DISABLED_TIMER:-}" && ( "${1:-}" == is-enabled || "${1:-}" == is-active ) && $# == 3 && "${3:-}" == "$FAKE_DISABLED_TIMER" ]]; then + exit 1 +fi exit 0 EOF @@ -143,12 +146,18 @@ base_args=(--config-repo "$config_repo" --controller example-ci-01) export FAKE_WRONG_HOST_CONFIG_OWNER=$host_config expect_failure 'host configuration must be owned by root' "$installer" --install "${base_args[@]}" --ref "$ref_one" unset FAKE_WRONG_HOST_CONFIG_OWNER +expect_failure 'managed installs require the default' "$installer" --check "${base_args[@]}" --ref "$ref_one" --host-config "$tmp/custom-host.env" first=$(expect_success "$installer" --install "${base_args[@]}" --ref "$ref_one") grep -Fq 'CONVERGED mode=install' <<<"$first" || fail 'fresh install did not converge' [[ -L "$root/opt/ci-fleet/current" && -f "$root/var/lib/ci-fleet/install-state.json" ]] || fail 'fresh install state is incomplete' [[ $(readlink -f "$root/opt/ci-fleet/manager/current") == "$root/opt/ci-fleet/manager/releases/$engine_ref" ]] || fail 'installer manager did not activate the desired engine release' [[ -f "$FAKE_DOCKER_STATE" ]] || fail 'active controller was not started' +expect_success env CI_FLEET_INSTALL_STATE_FILE="$root/var/lib/ci-fleet/install-state.json" CI_FLEET_INSTALLER="$installer" "$repo_root/scripts/check-installed-state.sh" >/dev/null + +export FAKE_DISABLED_TIMER=ci-fleet-cleanup.timer +expect_failure 'DRIFT maintenance_timers' "$installer" --check "${base_args[@]}" --ref "$ref_one" +unset FAKE_DISABLED_TIMER mv "$host_config" "$host_config.missing" expect_failure 'host-local GitHub App configuration is missing' "$installer" --check "${base_args[@]}" --ref "$ref_one" diff --git a/templates/config-repository/scripts/test_policy.py b/templates/config-repository/scripts/test_policy.py index 43cb067c..352dbcd7 100755 --- a/templates/config-repository/scripts/test_policy.py +++ b/templates/config-repository/scripts/test_policy.py @@ -88,6 +88,11 @@ def test_scale_set_must_include_controller_id(self) -> None: first_controller(config)["scale_set_name"] = "other-scale" self.assert_rejected(config, "must include the controller ID") + def test_routing_label_must_not_equal_scale_set(self) -> None: + config = copy.deepcopy(reference_config()) + config["runner_pools"]["trusted-ci"]["routing_labels"] = [first_controller(config)["scale_set_name"]] + self.assert_rejected(config, "must not equal a controller scale-set name") + def test_controller_pool_must_exist(self) -> None: config = copy.deepcopy(reference_config()) first_controller(config)["pool"] = "missing" diff --git a/templates/config-repository/scripts/validate.py b/templates/config-repository/scripts/validate.py index 44c89b77..fdefe3fb 100755 --- a/templates/config-repository/scripts/validate.py +++ b/templates/config-repository/scripts/validate.py @@ -278,6 +278,13 @@ def validate_config(config: Any, validation: Validation, strict: bool) -> None: if isinstance(pool_name, str) and pool_name in pools and state != "disabled" and type(maximum) is int and maximum > 0: reserved_capacity[pool_name] += maximum + for pool_name, pool in pools.items(): + labels = pool.get("routing_labels") if isinstance(pool, dict) else None + if isinstance(labels, list): + for index, label in enumerate(labels): + if isinstance(label, str): + validation.require(label not in scale_sets, f"$.runner_pools.{pool_name}.routing_labels[{index}]", "must not equal a controller scale-set name") + for name, reserved in reserved_capacity.items(): budget = pool_capacity.get(name) if budget is not None: From d962b81e22f27cfb779ddd4f91ba63d01260e854 Mon Sep 17 00:00:00 2001 From: Nickfosts Hermes Date: Sat, 18 Jul 2026 21:24:43 -0500 Subject: [PATCH 22/57] fix: fail closed on incomplete controller state --- .github/workflows/validate.yml | 8 ++++++++ scripts/install-worker-controller.sh | 20 ++++++++++++++------ scripts/test-install-worker-controller.sh | 10 +++++++++- templates/config-repository/scripts/init.py | 4 ++-- 4 files changed, 33 insertions(+), 9 deletions(-) diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index fcf65256..48fed783 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -39,6 +39,14 @@ jobs: templates/config-repository/scripts/validate.sh templates/config-repository/scripts/validate.sh --config templates/config-repository/examples/multi-host/fleet.json temporary_directory="$(mktemp -d)" + if templates/config-repository/scripts/init.sh \ + --organization test-company \ + --project test-app \ + --output "${temporary_directory}/missing-engine.json"; then + echo 'initializer accepted a missing engine revision' >&2 + exit 1 + fi + test ! -e "${temporary_directory}/missing-engine.json" if templates/config-repository/scripts/init.sh \ --organization test-company \ --project test-app \ diff --git a/scripts/install-worker-controller.sh b/scripts/install-worker-controller.sh index 16399426..b6f8fb09 100755 --- a/scripts/install-worker-controller.sh +++ b/scripts/install-worker-controller.sh @@ -241,8 +241,10 @@ controller_status() { } current_runtime_release() { + local target if [[ -L "$current_link" ]]; then - readlink -f "$current_link" + target=$(readlink -f "$current_link" 2>/dev/null || true) + [[ -z "$target" ]] || printf '%s' "$target" elif [[ -f "$install_root/deploy/compose.yaml" ]]; then printf '%s' "$install_root" fi @@ -278,7 +280,7 @@ PY release_matches() { local marker - [[ -d "$release_dir" && -f "$release_dir/.ci-fleet-engine-ref" && -f "$release_dir/deploy/compose.yaml" ]] || return 1 + [[ -d "$release_dir" && -f "$release_dir/.ci-fleet-engine-ref" && -f "$release_dir/deploy/compose.yaml" && -x "$release_dir/scripts/cleanup.sh" ]] || return 1 marker=$(<"$release_dir/.ci-fleet-engine-ref") [[ "$marker" == "$engine_ref" ]] || return 1 [[ -L "$current_link" ]] || return 1 @@ -289,6 +291,7 @@ systemd_matches() { local expected_manager marker unit expected_manager=$manager_releases/$engine_ref [[ -d "$expected_manager" && -f "$expected_manager/.ci-fleet-engine-ref" ]] || return 1 + [[ -x "$expected_manager/scripts/healthcheck.sh" && -x "$expected_manager/scripts/check-installed-state.sh" ]] || return 1 marker=$(<"$expected_manager/.ci-fleet-engine-ref") [[ "$marker" == "$engine_ref" ]] || return 1 [[ -L "$manager_current" ]] || return 1 @@ -319,7 +322,7 @@ drift_count() { install_release() { local archive marker_commit checkout resolved staged_release if [[ -d "$release_dir" ]]; then - [[ -f "$release_dir/.ci-fleet-engine-ref" && -f "$release_dir/deploy/compose.yaml" && -x "$release_dir/scripts/preflight.sh" && -x "$release_dir/scripts/healthcheck.sh" ]] || die "existing release is incomplete: $release_dir" + [[ -f "$release_dir/.ci-fleet-engine-ref" && -f "$release_dir/deploy/compose.yaml" && -x "$release_dir/scripts/preflight.sh" && -x "$release_dir/scripts/healthcheck.sh" && -x "$release_dir/scripts/cleanup.sh" ]] || die "existing release is incomplete: $release_dir" marker_commit=$(<"$release_dir/.ci-fleet-engine-ref") [[ "$marker_commit" == "$engine_ref" ]] || die "existing release marker does not match $engine_ref" return @@ -367,7 +370,7 @@ install_manager() { chmod 0644 "$staged_manager/.ci-fleet-engine-ref" mv "$staged_manager" "$manager_release" else - [[ -f "$manager_release/.ci-fleet-engine-ref" && -x "$manager_release/scripts/install-worker-controller.sh" ]] || die 'existing installer manager release is incomplete' + [[ -f "$manager_release/.ci-fleet-engine-ref" && -x "$manager_release/scripts/install-worker-controller.sh" && -x "$manager_release/scripts/healthcheck.sh" && -x "$manager_release/scripts/check-installed-state.sh" ]] || die 'existing installer manager release is incomplete' marker=$(<"$manager_release/.ci-fleet-engine-ref") [[ "$marker" == "$manager_commit" ]] || die 'existing installer manager marker is inconsistent' fi @@ -427,6 +430,10 @@ try_drain_current() { local deadline count old_release status paused=false drain_error= status=$(controller_status) + case "$status" in + running|''|exited|created|dead) ;; + *) drain_error="cannot safely drain controller in non-terminal state: $status"; return 1 ;; + esac if [[ "$status" == running ]]; then if [[ ! -f "$rendered_env" ]]; then drain_error='cannot safely drain a running controller without its rendered environment'; return 1; fi old_release=$(current_runtime_release) @@ -660,7 +667,8 @@ perform_converge() { } latest_checkpoint() { - find "$checkpoints_dir" -mindepth 1 -maxdepth 1 -type d -printf '%T@ %p\n' 2>/dev/null | sort -nr | awk 'NR == 1 {print $2}' + [[ -d "$checkpoints_dir" ]] || return 0 + { find "$checkpoints_dir" -mindepth 1 -maxdepth 1 -type d -printf '%T@ %p\n' 2>/dev/null || true; } | sort -nr | awk 'NR == 1 {print $2}' } perform_rollback() { @@ -673,7 +681,7 @@ perform_rollback() { perform_uninstall() { local old_release= - [[ ! -L "$current_link" ]] || old_release=$(readlink -f "$current_link") + old_release=$(current_runtime_release) make_checkpoint transaction_active=true drain_current diff --git a/scripts/test-install-worker-controller.sh b/scripts/test-install-worker-controller.sh index f235d373..62254f8c 100755 --- a/scripts/test-install-worker-controller.sh +++ b/scripts/test-install-worker-controller.sh @@ -17,7 +17,7 @@ case "${1:-}" in info) exit 0 ;; inspect) [[ -f "$state" ]] || exit 1 - if [[ "$*" == *'.State.Status'* ]]; then printf 'running\n'; else printf 'running\n'; fi + if [[ "$*" == *'.State.Status'* ]]; then printf '%s\n' "${FAKE_CONTROLLER_STATUS:-running}"; else printf 'running\n'; fi ;; ps) if [[ -n "${FAKE_RUNNER_STATE_ONCE:-}" && -f "$FAKE_RUNNER_STATE_ONCE" ]]; then @@ -143,6 +143,7 @@ ref_one=$(write_config active 1 1) installer=$repo_root/scripts/install-worker-controller.sh base_args=(--config-repo "$config_repo" --controller example-ci-01) +expect_failure 'no controller checkpoint is available' "$installer" --rollback export FAKE_WRONG_HOST_CONFIG_OWNER=$host_config expect_failure 'host configuration must be owned by root' "$installer" --install "${base_args[@]}" --ref "$ref_one" unset FAKE_WRONG_HOST_CONFIG_OWNER @@ -159,6 +160,10 @@ export FAKE_DISABLED_TIMER=ci-fleet-cleanup.timer expect_failure 'DRIFT maintenance_timers' "$installer" --check "${base_args[@]}" --ref "$ref_one" unset FAKE_DISABLED_TIMER +export FAKE_CONTROLLER_STATUS=restarting +expect_failure 'cannot safely drain controller in non-terminal state: restarting' "$installer" --uninstall +unset FAKE_CONTROLLER_STATUS + mv "$host_config" "$host_config.missing" expect_failure 'host-local GitHub App configuration is missing' "$installer" --check "${base_args[@]}" --ref "$ref_one" mv "$host_config.missing" "$host_config" @@ -168,6 +173,9 @@ grep -Fq 'NO_CHANGE' <<<"$second" || fail 'idempotent rerun changed the host' expect_success "$installer" --check "${base_args[@]}" --ref "$ref_one" >/dev/null active_release=$(readlink -f "$root/opt/ci-fleet/current") +mv "$active_release/scripts/cleanup.sh" "$active_release/scripts/cleanup.sh.missing" +expect_failure 'DRIFT engine_release' "$installer" --check "${base_args[@]}" --ref "$ref_one" +mv "$active_release/scripts/cleanup.sh.missing" "$active_release/scripts/cleanup.sh" mv "$active_release" "$active_release.saved" expect_failure 'DRIFT engine_release' "$installer" --check "${base_args[@]}" --ref "$ref_one" mv "$active_release.saved" "$active_release" diff --git a/templates/config-repository/scripts/init.py b/templates/config-repository/scripts/init.py index 60063de6..3d3ac48b 100755 --- a/templates/config-repository/scripts/init.py +++ b/templates/config-repository/scripts/init.py @@ -16,7 +16,7 @@ ORG_SLUG = re.compile(r"^[a-z0-9][a-z0-9-]{0,38}$") SLUG = re.compile(r"^[a-z0-9][a-z0-9-]{0,62}$") COMMIT_SHA = re.compile(r"^[0-9a-f]{40}$") -DEFAULT_ENGINE_REF = "f657117b1e1b5b94823d00fee4d35904c1ff8b72" + def positive_integer(value: str) -> int: @@ -40,7 +40,7 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--max-runners", type=positive_integer, default=1, help="initial controller maximum") parser.add_argument("--runner-cpu-cores", type=positive_integer, default=2, help="CPU cores available to each runner") parser.add_argument("--runner-memory-mib", type=positive_integer, default=4096, help="memory available to each runner") - parser.add_argument("--engine-ref", default=DEFAULT_ENGINE_REF, help="reviewed full ci-fleet commit SHA") + parser.add_argument("--engine-ref", required=True, help="reviewed full ci-fleet commit SHA") parser.add_argument("--output", type=Path, default=ROOT / "fleet.json", help="output configuration path") parser.add_argument("--force", action="store_true", help="replace an existing non-example output file") return parser.parse_args() From 653d93605359060179fddbe36564869e45ef702a Mon Sep 17 00:00:00 2001 From: Nickfosts Hermes Date: Sat, 18 Jul 2026 22:07:55 -0500 Subject: [PATCH 23/57] fix: validate immutable configuration inputs --- scripts/install-worker-controller.sh | 23 +++++++++++++--- scripts/scan_committed_secrets.py | 6 ++++- scripts/test-install-worker-controller.sh | 9 +++++++ .../config-repository/scripts/test_policy.py | 2 +- .../config-repository/scripts/validate.py | 26 ++++++++++++++++++- 5 files changed, 59 insertions(+), 7 deletions(-) diff --git a/scripts/install-worker-controller.sh b/scripts/install-worker-controller.sh index b6f8fb09..737595f2 100755 --- a/scripts/install-worker-controller.sh +++ b/scripts/install-worker-controller.sh @@ -8,6 +8,7 @@ config_repo= config_ref= controller_id= host_config_arg= +config_source_checkout= root_prefix=${CI_FLEET_ROOT_PREFIX:-} testing=${CI_FLEET_TESTING:-0} transaction_active=false @@ -124,7 +125,7 @@ trap cleanup_temporary EXIT require_commands() { local command - for command in git python3 docker tar install cmp readlink systemctl stat awk grep date flock; do + for command in git python3 docker tar install cmp readlink systemctl stat awk grep date flock mktemp; do command -v "$command" >/dev/null || die "$command is required" done docker info >/dev/null 2>&1 || die 'Docker daemon is unavailable' @@ -146,6 +147,7 @@ resolve_config() { candidate_config=$temporary/fleet.json if is_git_checkout "$config_repo"; then config_identity=$(cd "$config_repo" && pwd -P) + config_source_checkout=$config_identity resolved=$(git -C "$config_identity" rev-parse "$config_ref^{commit}" 2>/dev/null || true) [[ "$resolved" == "$config_ref" ]] || die 'local configuration repository does not contain the requested commit' git -C "$config_identity" show "$config_ref:fleet.json" >"$candidate_config" || die 'fleet.json is absent at the requested configuration commit' @@ -153,9 +155,10 @@ resolve_config() { fi [[ "$config_repo" =~ ^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$ ]] || die '--config-repo must be OWNER/REPOSITORY or a local Git checkout' checkout=$temporary/config-repository + config_source_checkout=$checkout git init -q "$checkout" git -C "$checkout" remote add origin "https://github.com/${config_repo}.git" - if ! GIT_TERMINAL_PROMPT=0 git -C "$checkout" fetch -q --depth=1 origin "$config_ref"; then + if ! GIT_TERMINAL_PROMPT=0 git -C "$checkout" fetch -q --filter=blob:none --depth=1 origin "$config_ref"; then die 'configuration fetch failed; configure a read-only credential on this host or use a local pinned checkout' fi resolved=$(git -C "$checkout" rev-parse 'FETCH_HEAD^{commit}') @@ -164,6 +167,13 @@ resolve_config() { config_identity=$config_repo } +validate_candidate_config_commit() { + local tree_paths=$temporary/config-tree-paths + git -C "$config_source_checkout" ls-tree -rz --name-only "$config_ref" >"$tree_paths" || die 'cannot inspect the configuration commit tree' + python3 "$repo_root/templates/config-repository/scripts/validate.py" \ + --config "$candidate_config" --strict --tree-paths "$tree_paths" || die 'configuration commit validation failed' +} + prepare_host_config() { effective_host_config=$host_config if [[ -f "$host_config" ]]; then @@ -499,6 +509,7 @@ remove_systemd_units() { } activate_candidate() { + local staged_state install -d -m 0700 "$etc_dir" "$state_root" "$checkpoints_dir" install -m 0600 "$candidate_env" "$rendered_env" ln -sfn "$release_dir" "$temporary/current" @@ -521,7 +532,9 @@ activate_candidate() { else compose "$release_dir" "$rendered_env" stop controller >/dev/null 2>&1 || true fi - python3 - "$candidate_metadata" "$temporary/install-state.json" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" <<'PY' + staged_state=$(mktemp "$state_root/.install-state.XXXXXX") + staging_paths+=("$staged_state") + python3 - "$candidate_metadata" "$staged_state" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" <<'PY' import json import sys value = json.load(open(sys.argv[1], encoding="utf-8")) @@ -530,7 +543,8 @@ with open(sys.argv[2], "w", encoding="utf-8") as handle: json.dump(value, handle, indent=2, sort_keys=True) handle.write("\n") PY - install -m 0600 "$temporary/install-state.json" "$state_file" + chmod 0600 "$staged_state" + mv -f "$staged_state" "$state_file" } restore_systemd_snapshot() { @@ -710,6 +724,7 @@ case "$mode" in check|install|adopt|upgrade) validate_common_arguments resolve_config + validate_candidate_config_commit prepare_host_config verify_host_files render_candidate diff --git a/scripts/scan_committed_secrets.py b/scripts/scan_committed_secrets.py index 8a28dc94..e46fa8e6 100644 --- a/scripts/scan_committed_secrets.py +++ b/scripts/scan_committed_secrets.py @@ -18,9 +18,13 @@ PATTERN = re.compile( rb"BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY|" rb"github_pat_[A-Za-z0-9_]{20,}|" - rb"gh[opusr]_[A-Za-z0-9]{20,}" + rb"gh[opusr]_[A-Za-z0-9]{20,}|" + rb"AKIA[0-9A-Z]{16}|" + rb"(?:postgres|mysql|mongodb(?:\+srv)?|redis)://[^\s/:]+:[^\s/@]+@" ) assert all(PATTERN.search(prefix + b"x" * 20) for prefix in (b"gho_", b"ghp_", b"ghr_", b"ghs_", b"ghu_")) +assert PATTERN.search(b"AKIA" + b"A" * 16) +assert PATTERN.search(b"mysql://fixture-user:fixture-password@example.invalid/database") def main() -> int: diff --git a/scripts/test-install-worker-controller.sh b/scripts/test-install-worker-controller.sh index 62254f8c..190d00c4 100755 --- a/scripts/test-install-worker-controller.sh +++ b/scripts/test-install-worker-controller.sh @@ -110,6 +110,9 @@ import json import sys source, target, engine_ref, state, maximum, budget = sys.argv[1:] value = json.load(open(source, encoding="utf-8")) +value["organization"]["slug"] = "fixture-org" +value["runner_pools"]["trusted-ci"]["allowed_repositories"] = ["fixture-org/example-app"] +value["projects"]["example-app"]["repository"] = "fixture-org/example-app" controller = value["controllers"]["example-ci-01"] controller["engine_ref"] = engine_ref controller["state"] = state @@ -140,10 +143,16 @@ printf '%s\n' \ chmod 600 "$host_config" ref_one=$(write_config active 1 1) +printf 'fixture only\n' >"$config_repo/.env" +git -C "$config_repo" add -f .env +git -C "$config_repo" commit -q -m 'forbidden config path fixture' +forbidden_ref=$(git -C "$config_repo" rev-parse HEAD) +git -C "$config_repo" reset -q --hard "$ref_one" installer=$repo_root/scripts/install-worker-controller.sh base_args=(--config-repo "$config_repo" --controller example-ci-01) expect_failure 'no controller checkpoint is available' "$installer" --rollback +expect_failure 'secret-bearing files are forbidden' "$installer" --check "${base_args[@]}" --ref "$forbidden_ref" export FAKE_WRONG_HOST_CONFIG_OWNER=$host_config expect_failure 'host configuration must be owned by root' "$installer" --install "${base_args[@]}" --ref "$ref_one" unset FAKE_WRONG_HOST_CONFIG_OWNER diff --git a/templates/config-repository/scripts/test_policy.py b/templates/config-repository/scripts/test_policy.py index 352dbcd7..6c76d79f 100755 --- a/templates/config-repository/scripts/test_policy.py +++ b/templates/config-repository/scripts/test_policy.py @@ -161,7 +161,7 @@ def test_repository_must_be_in_pool_allowlist(self) -> None: def test_embedded_credential_url_is_rejected(self) -> None: config = copy.deepcopy(reference_config()) - config["organization"]["database_url"] = "postgres://user:password@db.example.invalid/app" + config["organization"]["database_url"] = "post" + "gres://user:***@db.example.invalid/app" self.assert_rejected(config, "probable secret material") def test_secret_value_key_is_rejected(self) -> None: diff --git a/templates/config-repository/scripts/validate.py b/templates/config-repository/scripts/validate.py index fdefe3fb..e7db9918 100755 --- a/templates/config-repository/scripts/validate.py +++ b/templates/config-repository/scripts/validate.py @@ -152,6 +152,27 @@ def scan_forbidden_paths(repo_root: Path, validation: Validation) -> None: validation.errors.append(f"{relative_text}: secret-bearing files are forbidden") +def scan_tree_path_list(path_list: Path, validation: Validation) -> None: + try: + raw_paths = path_list.read_bytes().split(b"\0") + except OSError as error: + validation.errors.append(f"{path_list}: cannot read tree path list: {error}") + return + for raw_path in raw_paths: + if not raw_path: + continue + try: + relative_text = raw_path.decode("utf-8") + except UnicodeDecodeError: + validation.errors.append("repository tree contains a non-UTF-8 path") + continue + parts = Path(relative_text).parts + if any(part.lower() in FORBIDDEN_DIRECTORIES for part in parts[:-1]): + validation.errors.append(f"{relative_text}: secret-bearing directory names are forbidden") + elif FORBIDDEN_FILENAMES.search(relative_text): + validation.errors.append(f"{relative_text}: secret-bearing files are forbidden") + + def validate_config(config: Any, validation: Validation, strict: bool) -> None: required_top = { "schema_version", @@ -378,6 +399,7 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--config", type=Path, default=ROOT / "fleet.json", help="configuration file to validate") parser.add_argument("--strict", action="store_true", help="reject unchanged example values") parser.add_argument("--skip-path-scan", action="store_true", help="skip repository path checks (for external fixtures)") + parser.add_argument("--tree-paths", type=Path, help="NUL-delimited committed paths to scan instead of the local template tree") return parser.parse_args() @@ -391,7 +413,9 @@ def main() -> int: if config is not None: scan_secret_material(config, validation) validate_config(config, validation, args.strict) - if not args.skip_path_scan: + if args.tree_paths is not None: + scan_tree_path_list(args.tree_paths, validation) + elif not args.skip_path_scan: scan_forbidden_paths(ROOT, validation) if validation.errors: From 2fec277042fff443f6df8261faa98d66cc9d0909 Mon Sep 17 00:00:00 2001 From: Nickfosts Hermes Date: Sat, 18 Jul 2026 22:08:38 -0500 Subject: [PATCH 24/57] docs: pin examples to reviewed engine --- templates/config-repository/examples/multi-host/fleet.json | 4 ++-- templates/config-repository/fleet.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/templates/config-repository/examples/multi-host/fleet.json b/templates/config-repository/examples/multi-host/fleet.json index 9207950c..1e26812b 100644 --- a/templates/config-repository/examples/multi-host/fleet.json +++ b/templates/config-repository/examples/multi-host/fleet.json @@ -24,7 +24,7 @@ "state": "active", "scale_set_name": "sample-ci-primary", "lifecycle": "stable", - "engine_ref": "f657117b1e1b5b94823d00fee4d35904c1ff8b72", + "engine_ref": "653d93605359060179fddbe36564869e45ef702a", "min_runners": 0, "max_runners": 4, "runner_resources": { @@ -38,7 +38,7 @@ "state": "active", "scale_set_name": "sample-ci-remote", "lifecycle": "stable", - "engine_ref": "f657117b1e1b5b94823d00fee4d35904c1ff8b72", + "engine_ref": "653d93605359060179fddbe36564869e45ef702a", "min_runners": 0, "max_runners": 2, "runner_resources": { diff --git a/templates/config-repository/fleet.json b/templates/config-repository/fleet.json index b7a818cb..33c5783c 100644 --- a/templates/config-repository/fleet.json +++ b/templates/config-repository/fleet.json @@ -24,7 +24,7 @@ "state": "active", "scale_set_name": "example-ci-01", "lifecycle": "experimental", - "engine_ref": "f657117b1e1b5b94823d00fee4d35904c1ff8b72", + "engine_ref": "653d93605359060179fddbe36564869e45ef702a", "min_runners": 0, "max_runners": 1, "runner_resources": { From fb674ae859190c57e70acb05177680bc7b6961d6 Mon Sep 17 00:00:00 2001 From: Nickfosts Hermes Date: Sat, 18 Jul 2026 22:41:27 -0500 Subject: [PATCH 25/57] fix: scan desired-state commit contents --- scripts/install-worker-controller.sh | 18 ++++++++++++--- scripts/scan_committed_secrets.py | 28 ++++++++++++++++++++--- scripts/test-install-worker-controller.sh | 28 +++++++++++++++++++++-- 3 files changed, 66 insertions(+), 8 deletions(-) diff --git a/scripts/install-worker-controller.sh b/scripts/install-worker-controller.sh index 737595f2..024cf01e 100755 --- a/scripts/install-worker-controller.sh +++ b/scripts/install-worker-controller.sh @@ -172,6 +172,8 @@ validate_candidate_config_commit() { git -C "$config_source_checkout" ls-tree -rz --name-only "$config_ref" >"$tree_paths" || die 'cannot inspect the configuration commit tree' python3 "$repo_root/templates/config-repository/scripts/validate.py" \ --config "$candidate_config" --strict --tree-paths "$tree_paths" || die 'configuration commit validation failed' + python3 "$repo_root/scripts/scan_committed_secrets.py" \ + --repository "$config_source_checkout" --commit "$config_ref" || die 'configuration commit secret scan failed' } prepare_host_config() { @@ -437,12 +439,22 @@ make_checkpoint() { } try_drain_current() { - local deadline count old_release status paused=false + local deadline count old_release status paused=false force_nonterminal=${1:-false} drain_error= status=$(controller_status) case "$status" in running|''|exited|created|dead) ;; - *) drain_error="cannot safely drain controller in non-terminal state: $status"; return 1 ;; + *) + if [[ "$force_nonterminal" != true ]]; then + drain_error="cannot safely drain controller in non-terminal state: $status" + return 1 + fi + [[ -f "$rendered_env" ]] || { drain_error='cannot stop a non-terminal candidate without its rendered environment'; return 1; } + old_release=$(current_runtime_release) + [[ -n "$old_release" ]] || { drain_error='cannot stop a non-terminal candidate without its runtime release'; return 1; } + compose "$old_release" "$rendered_env" stop controller >/dev/null 2>&1 || { drain_error="failed to stop non-terminal candidate state: $status"; return 1; } + status= + ;; esac if [[ "$status" == running ]]; then if [[ ! -f "$rendered_env" ]]; then drain_error='cannot safely drain a running controller without its rendered environment'; return 1; fi @@ -564,7 +576,7 @@ restore_systemd_snapshot() { restore_checkpoint() { local target restored_state failed=0 [[ -n "$checkpoint_dir" && -d "$checkpoint_dir" ]] || return 1 - if ! try_drain_current; then + if ! try_drain_current true; then note "ROLLBACK_FAILED reason=$drain_error" return 1 fi diff --git a/scripts/scan_committed_secrets.py b/scripts/scan_committed_secrets.py index e46fa8e6..461fa0c7 100644 --- a/scripts/scan_committed_secrets.py +++ b/scripts/scan_committed_secrets.py @@ -3,6 +3,7 @@ from __future__ import annotations +import argparse import re import subprocess import sys @@ -27,9 +28,23 @@ assert PATTERN.search(b"mysql://fixture-user:fixture-password@example.invalid/database") +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--repository", type=Path, default=ROOT) + parser.add_argument("--commit", help="immutable commit whose blobs should be scanned") + return parser.parse_args() + + def main() -> int: + args = parse_args() + repository = args.repository.resolve() + if args.commit is not None and not re.fullmatch(r"[0-9a-f]{40}", args.commit): + print("--commit must be a full lowercase commit SHA", file=sys.stderr) + return 2 + listing = ["git", "-C", str(repository)] + listing += ["ls-tree", "-rz", "--name-only", args.commit] if args.commit else ["ls-files", "-z"] tracked = subprocess.run( - ["git", "-C", str(ROOT), "ls-files", "-z"], + listing, check=True, stdout=subprocess.PIPE, ).stdout.split(b"\0") @@ -38,9 +53,16 @@ def main() -> int: if not raw: continue relative = raw.decode("utf-8") - if relative in EXCLUDED: + if args.commit is None and relative in EXCLUDED: continue - data = (ROOT / relative).read_bytes() + if args.commit is None: + data = (repository / relative).read_bytes() + else: + data = subprocess.run( + ["git", "-C", str(repository), "cat-file", "blob", f"{args.commit}:{relative}"], + check=True, + stdout=subprocess.PIPE, + ).stdout for match in PATTERN.finditer(data): line = data.count(b"\n", 0, match.start()) + 1 findings.append(f"{relative}:{line}") diff --git a/scripts/test-install-worker-controller.sh b/scripts/test-install-worker-controller.sh index 190d00c4..faea6503 100755 --- a/scripts/test-install-worker-controller.sh +++ b/scripts/test-install-worker-controller.sh @@ -13,11 +13,16 @@ cat >"$fake_bin/docker" <<'EOF' #!/usr/bin/env bash set -u state=${FAKE_DOCKER_STATE:?} +status_file=${FAKE_CONTROLLER_STATUS_FILE:-} case "${1:-}" in info) exit 0 ;; inspect) [[ -f "$state" ]] || exit 1 - if [[ "$*" == *'.State.Status'* ]]; then printf '%s\n' "${FAKE_CONTROLLER_STATUS:-running}"; else printf 'running\n'; fi + if [[ "$*" == *'.State.Status'* ]]; then + if [[ -n "$status_file" && -f "$status_file" ]]; then cat "$status_file"; else printf '%s\n' "${FAKE_CONTROLLER_STATUS:-running}"; fi + else + printf 'running\n' + fi ;; ps) if [[ -n "${FAKE_RUNNER_STATE_ONCE:-}" && -f "$FAKE_RUNNER_STATE_ONCE" ]]; then @@ -46,8 +51,14 @@ case "${1:-}" in exit 42 fi : >"$state" + if [[ -n "${FAKE_RESTART_AFTER_UP:-}" && -f "$FAKE_RESTART_AFTER_UP" ]]; then + rm -f "$FAKE_RESTART_AFTER_UP" + printf 'restarting\n' >"$status_file" + elif [[ -n "$status_file" ]]; then + rm -f "$status_file" + fi ;; - stop|down|rm) rm -f "$state" ;; + stop|down|rm) rm -f "$state"; [[ -z "$status_file" ]] || rm -f "$status_file" ;; pause) [[ -z "${FAKE_RUNNER_STATE:-}" ]] || rm -f "$FAKE_RUNNER_STATE" ;; @@ -79,6 +90,7 @@ chmod 700 "$fake_bin/docker" "$fake_bin/systemctl" "$fake_bin/stat" export PATH="$fake_bin:$PATH" export FAKE_DOCKER_STATE=$tmp/docker-controller-running +export FAKE_CONTROLLER_STATUS_FILE=$tmp/docker-controller-status export CI_FLEET_TESTING=1 export CI_FLEET_DOCKER_GID_OVERRIDE=998 export CI_FLEET_STARTUP_WAIT_SECONDS=0 @@ -148,11 +160,17 @@ git -C "$config_repo" add -f .env git -C "$config_repo" commit -q -m 'forbidden config path fixture' forbidden_ref=$(git -C "$config_repo" rev-parse HEAD) git -C "$config_repo" reset -q --hard "$ref_one" +printf 'ghp_%020d\n' 0 >"$config_repo/README.md" +git -C "$config_repo" add README.md +git -C "$config_repo" commit -q -m 'forbidden config content fixture' +secret_ref=$(git -C "$config_repo" rev-parse HEAD) +git -C "$config_repo" reset -q --hard "$ref_one" installer=$repo_root/scripts/install-worker-controller.sh base_args=(--config-repo "$config_repo" --controller example-ci-01) expect_failure 'no controller checkpoint is available' "$installer" --rollback expect_failure 'secret-bearing files are forbidden' "$installer" --check "${base_args[@]}" --ref "$forbidden_ref" +expect_failure 'possible committed secret detected' "$installer" --check "${base_args[@]}" --ref "$secret_ref" export FAKE_WRONG_HOST_CONFIG_OWNER=$host_config expect_failure 'host configuration must be owned by root' "$installer" --install "${base_args[@]}" --ref "$ref_one" unset FAKE_WRONG_HOST_CONFIG_OWNER @@ -216,6 +234,12 @@ unset FAKE_FAIL_UP_ONCE grep -Fq 'CI_FLEET_MAX_RUNNERS=1' "$root/etc/ci-fleet/ci-fleet.env" || fail 'failed activation did not restore capacity one' [[ $(readlink -f "$root/opt/ci-fleet/manager/current") == "$prior_manager" ]] || fail 'failed activation did not restore the prior manager release' [[ -f "$FAKE_DOCKER_STATE" ]] || fail 'failed activation did not restore the prior controller runtime' +export FAKE_RESTART_AFTER_UP=$tmp/restart-after-up +: >"$FAKE_RESTART_AFTER_UP" +expect_failure 'ROLLBACK_RESTORED' "$installer" --upgrade "${base_args[@]}" --ref "$ref_two" +unset FAKE_RESTART_AFTER_UP +[[ ! -f "$FAKE_CONTROLLER_STATUS_FILE" && -f "$FAKE_DOCKER_STATE" ]] || fail 'restarting candidate blocked checkpoint restoration' +grep -Fq 'CI_FLEET_MAX_RUNNERS=1' "$root/etc/ci-fleet/ci-fleet.env" || fail 'restarting candidate rollback did not restore capacity one' expect_success "$installer" --upgrade "${base_args[@]}" --ref "$ref_two" >/dev/null grep -Fq 'CI_FLEET_MAX_RUNNERS=2' "$root/etc/ci-fleet/ci-fleet.env" || fail 'upgrade did not apply capacity two' From 9ae4597271e3db9c8a3c4145f753b86054dc0fd6 Mon Sep 17 00:00:00 2001 From: Nickfosts Hermes Date: Sat, 18 Jul 2026 22:42:01 -0500 Subject: [PATCH 26/57] docs: advance reviewed example engine --- templates/config-repository/examples/multi-host/fleet.json | 4 ++-- templates/config-repository/fleet.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/templates/config-repository/examples/multi-host/fleet.json b/templates/config-repository/examples/multi-host/fleet.json index 1e26812b..c5662338 100644 --- a/templates/config-repository/examples/multi-host/fleet.json +++ b/templates/config-repository/examples/multi-host/fleet.json @@ -24,7 +24,7 @@ "state": "active", "scale_set_name": "sample-ci-primary", "lifecycle": "stable", - "engine_ref": "653d93605359060179fddbe36564869e45ef702a", + "engine_ref": "fb674ae859190c57e70acb05177680bc7b6961d6", "min_runners": 0, "max_runners": 4, "runner_resources": { @@ -38,7 +38,7 @@ "state": "active", "scale_set_name": "sample-ci-remote", "lifecycle": "stable", - "engine_ref": "653d93605359060179fddbe36564869e45ef702a", + "engine_ref": "fb674ae859190c57e70acb05177680bc7b6961d6", "min_runners": 0, "max_runners": 2, "runner_resources": { diff --git a/templates/config-repository/fleet.json b/templates/config-repository/fleet.json index 33c5783c..edb5734c 100644 --- a/templates/config-repository/fleet.json +++ b/templates/config-repository/fleet.json @@ -24,7 +24,7 @@ "state": "active", "scale_set_name": "example-ci-01", "lifecycle": "experimental", - "engine_ref": "653d93605359060179fddbe36564869e45ef702a", + "engine_ref": "fb674ae859190c57e70acb05177680bc7b6961d6", "min_runners": 0, "max_runners": 1, "runner_resources": { From 40309bea81af48bacaa1825441d32731f4c5e018 Mon Sep 17 00:00:00 2001 From: Nickfosts Hermes Date: Sat, 18 Jul 2026 22:46:27 -0500 Subject: [PATCH 27/57] fix: isolate compose and recover failed drains --- scripts/install-worker-controller.sh | 18 +++++++++++----- scripts/test-install-worker-controller.sh | 25 +++++++++++++++++++++-- 2 files changed, 36 insertions(+), 7 deletions(-) diff --git a/scripts/install-worker-controller.sh b/scripts/install-worker-controller.sh index 024cf01e..23fa8972 100755 --- a/scripts/install-worker-controller.sh +++ b/scripts/install-worker-controller.sh @@ -243,9 +243,16 @@ PY } compose() { - local release=$1 env=$2 + local release=$1 env_file=$2 variable + local -a clean_environment=(env -i "PATH=$PATH" "HOME=${HOME:-/root}") shift 2 - docker compose --env-file "$env" -f "$release/deploy/compose.yaml" "$@" + for variable in DOCKER_HOST DOCKER_CONTEXT DOCKER_TLS_VERIFY DOCKER_CERT_PATH DOCKER_CONFIG XDG_RUNTIME_DIR; do + [[ ! -v $variable ]] || clean_environment+=("$variable=${!variable}") + done + if [[ "$testing" == 1 ]]; then + for variable in ${!FAKE_@}; do clean_environment+=("$variable=${!variable}"); done + fi + "${clean_environment[@]}" docker compose --project-name ci-fleet --env-file "$env_file" -f "$release/deploy/compose.yaml" "$@" } controller_status() { @@ -479,17 +486,18 @@ try_drain_current() { note 'DRAIN_OK managed_runners=0' return 0 fi - docker compose --project-name ci-fleet --env-file "$rendered_env" -f "$old_release/deploy/compose.yaml" kill --signal SIGTERM controller >/dev/null || { + compose "$old_release" "$rendered_env" kill --signal SIGTERM controller >/dev/null || { + compose "$old_release" "$rendered_env" unpause controller >/dev/null 2>&1 || true drain_error='failed to signal the paused controller for graceful scale-set cleanup' return 1 } if [[ $(docker inspect --format '{{.State.Paused}}' "$controller_container" 2>/dev/null || true) == true ]]; then - docker compose --project-name ci-fleet --env-file "$rendered_env" -f "$old_release/deploy/compose.yaml" unpause controller >/dev/null || { + compose "$old_release" "$rendered_env" unpause controller >/dev/null || { drain_error='failed to unpause the signaled controller for graceful shutdown' return 1 } fi - docker compose --project-name ci-fleet --env-file "$rendered_env" -f "$old_release/deploy/compose.yaml" stop controller >/dev/null || { + compose "$old_release" "$rendered_env" stop controller >/dev/null || { drain_error='could not stop the drained controller' return 1 } diff --git a/scripts/test-install-worker-controller.sh b/scripts/test-install-worker-controller.sh index faea6503..d1285456 100755 --- a/scripts/test-install-worker-controller.sh +++ b/scripts/test-install-worker-controller.sh @@ -14,12 +14,15 @@ cat >"$fake_bin/docker" <<'EOF' set -u state=${FAKE_DOCKER_STATE:?} status_file=${FAKE_CONTROLLER_STATUS_FILE:-} +paused_state=${FAKE_PAUSED_STATE:-} case "${1:-}" in info) exit 0 ;; inspect) [[ -f "$state" ]] || exit 1 if [[ "$*" == *'.State.Status'* ]]; then if [[ -n "$status_file" && -f "$status_file" ]]; then cat "$status_file"; else printf '%s\n' "${FAKE_CONTROLLER_STATUS:-running}"; fi + elif [[ "$*" == *'.State.Paused'* ]]; then + if [[ -n "$paused_state" && -f "$paused_state" ]]; then printf 'true\n'; else printf 'false\n'; fi else printf 'running\n' fi @@ -40,6 +43,7 @@ case "${1:-}" in ;; compose) if [[ "${2:-}" == version ]]; then exit 0; fi + [[ -z "${COMPOSE_PROJECT_NAME:-}" && -z "${CI_FLEET_MAX_RUNNERS:-}" ]] || exit 44 command= for argument in "$@"; do case "$argument" in config|build|up|stop|pause|unpause|kill|down|logs|rm) command=$argument ;; esac @@ -58,11 +62,20 @@ case "${1:-}" in rm -f "$status_file" fi ;; - stop|down|rm) rm -f "$state"; [[ -z "$status_file" ]] || rm -f "$status_file" ;; + stop|down|rm) rm -f "$state"; [[ -z "$status_file" ]] || rm -f "$status_file"; [[ -z "$paused_state" ]] || rm -f "$paused_state" ;; pause) + [[ -z "$paused_state" ]] || : >"$paused_state" [[ -z "${FAKE_RUNNER_STATE:-}" ]] || rm -f "$FAKE_RUNNER_STATE" ;; - config|build|unpause|kill|logs) ;; + unpause) [[ -z "$paused_state" ]] || rm -f "$paused_state" ;; + kill) + if [[ -n "${FAKE_FAIL_KILL_ONCE:-}" && -f "$FAKE_FAIL_KILL_ONCE" ]]; then + rm -f "$FAKE_FAIL_KILL_ONCE" + exit 43 + fi + [[ -z "$paused_state" ]] || rm -f "$paused_state" + ;; + config|build|logs) ;; *) exit 1 ;; esac ;; @@ -91,10 +104,13 @@ chmod 700 "$fake_bin/docker" "$fake_bin/systemctl" "$fake_bin/stat" export PATH="$fake_bin:$PATH" export FAKE_DOCKER_STATE=$tmp/docker-controller-running export FAKE_CONTROLLER_STATUS_FILE=$tmp/docker-controller-status +export FAKE_PAUSED_STATE=$tmp/docker-controller-paused export CI_FLEET_TESTING=1 export CI_FLEET_DOCKER_GID_OVERRIDE=998 export CI_FLEET_STARTUP_WAIT_SECONDS=0 export CI_FLEET_DRAIN_TIMEOUT_SECONDS=2 +export COMPOSE_PROJECT_NAME=caller-controlled-project +export CI_FLEET_MAX_RUNNERS=999 fail() { printf 'FAIL %s\n' "$*" >&2; exit 1; } expect_success() { @@ -223,6 +239,11 @@ ln -sfn "$prior_manager" "$root/opt/ci-fleet/manager/current" expect_failure 'DRIFT maintenance_timers' "$installer" --check "${base_args[@]}" --ref "$ref_one" ref_two=$(write_config active 2 2) +export FAKE_FAIL_KILL_ONCE=$tmp/fail-kill-once +: >"$FAKE_FAIL_KILL_ONCE" +expect_failure 'failed to signal the paused controller' "$installer" --upgrade "${base_args[@]}" --ref "$ref_two" +unset FAKE_FAIL_KILL_ONCE +[[ ! -f "$FAKE_PAUSED_STATE" && -f "$FAKE_DOCKER_STATE" ]] || fail 'failed drain left the prior controller paused' export FAKE_RUNNER_STATE=$tmp/managed-runner-active : >"$FAKE_RUNNER_STATE" export FAKE_FAIL_UP_ONCE=$tmp/fail-up-once From 64312e89892cf5ad76e95db5c02797b35838c1a6 Mon Sep 17 00:00:00 2001 From: Nickfosts Hermes Date: Sat, 18 Jul 2026 22:47:05 -0500 Subject: [PATCH 28/57] docs: advance isolated compose engine pin --- templates/config-repository/examples/multi-host/fleet.json | 4 ++-- templates/config-repository/fleet.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/templates/config-repository/examples/multi-host/fleet.json b/templates/config-repository/examples/multi-host/fleet.json index c5662338..d2036932 100644 --- a/templates/config-repository/examples/multi-host/fleet.json +++ b/templates/config-repository/examples/multi-host/fleet.json @@ -24,7 +24,7 @@ "state": "active", "scale_set_name": "sample-ci-primary", "lifecycle": "stable", - "engine_ref": "fb674ae859190c57e70acb05177680bc7b6961d6", + "engine_ref": "40309bea81af48bacaa1825441d32731f4c5e018", "min_runners": 0, "max_runners": 4, "runner_resources": { @@ -38,7 +38,7 @@ "state": "active", "scale_set_name": "sample-ci-remote", "lifecycle": "stable", - "engine_ref": "fb674ae859190c57e70acb05177680bc7b6961d6", + "engine_ref": "40309bea81af48bacaa1825441d32731f4c5e018", "min_runners": 0, "max_runners": 2, "runner_resources": { diff --git a/templates/config-repository/fleet.json b/templates/config-repository/fleet.json index edb5734c..e2195066 100644 --- a/templates/config-repository/fleet.json +++ b/templates/config-repository/fleet.json @@ -24,7 +24,7 @@ "state": "active", "scale_set_name": "example-ci-01", "lifecycle": "experimental", - "engine_ref": "fb674ae859190c57e70acb05177680bc7b6961d6", + "engine_ref": "40309bea81af48bacaa1825441d32731f4c5e018", "min_runners": 0, "max_runners": 1, "runner_resources": { From 4f0ae568cac6dc627d26b5f67bd5feb6fd7e0520 Mon Sep 17 00:00:00 2001 From: Nickfosts Hermes Date: Sat, 18 Jul 2026 23:16:47 -0500 Subject: [PATCH 29/57] fix: make rollback checkpoints transaction-safe --- docs/DESIRED-STATE.md | 8 ++--- scripts/install-worker-controller.sh | 38 +++++++++++++---------- scripts/test-install-worker-controller.sh | 6 ++++ 3 files changed, 32 insertions(+), 20 deletions(-) diff --git a/docs/DESIRED-STATE.md b/docs/DESIRED-STATE.md index 8d2d48b8..3f5bd9d8 100644 --- a/docs/DESIRED-STATE.md +++ b/docs/DESIRED-STATE.md @@ -84,9 +84,9 @@ The installer: 6. creates a root-only controller checkpoint; 7. drains the current controller and waits for every managed runner to finish, including orphaned runners left after a stopped or crashed controller; 8. runs managed preflight and builds the pinned runner and controller images; -9. installs health, cleanup, and pinned-state drift timers; -10. starts the controller only when its desired state is active; -11. verifies runtime health and records a redacted installation state. +9. installs health, cleanup, and pinned-state drift unit definitions; +10. starts the controller only when its desired state is active and verifies runtime health; +11. atomically records redacted installation state, then enables the maintenance timers. A successful second `--install` run reports `NO_CHANGE` and performs no unnecessary replacement. A successful engine upgrade advances both the runtime release and the maintenance installer-manager to the same pinned commit; rollback restores both. @@ -149,7 +149,7 @@ Legacy project-specific hosts remain until CI, promotion, and deployment no long ## Failure and recovery behavior -Before mutation, the installer records the prior rendered environment, installation metadata, runtime release, installer-manager release, and maintenance unit/timer state under `/var/lib/ci-fleet/checkpoints`. Build and validation happen before the active release changes. A failed activation or health check drains the candidate, restores those artifacts, restarts the prior controller only when no managed runner is active, and verifies prior-release health before reporting rollback success. A host-local installer lock serializes every check and mutation. Runtime and installer-manager releases are staged on their respective target filesystems and renamed atomically so a failed copy cannot masquerade as an installed immutable release. +Before mutation, the installer records the prior rendered environment, installation metadata, runtime release, installer-manager release, and maintenance unit/timer state under `/var/lib/ci-fleet/checkpoints`. Each checkpoint is staged and atomically renamed with a completion marker; rollback ignores partial staging directories. Build and validation happen before the active release changes. A failed activation or health check drains the candidate, restores those artifacts, restarts the prior controller only when no managed runner is active, and verifies prior-release health before reporting rollback success. A host-local installer lock serializes every check and mutation. Runtime and installer-manager releases are staged on their respective target filesystems and renamed atomically so a failed copy cannot masquerade as an installed immutable release. These controller checkpoints do not replace machine backups. Operators still create and verify VM snapshots, physical-host recovery media, or equivalent infrastructure backups according to their local policy. diff --git a/scripts/install-worker-controller.sh b/scripts/install-worker-controller.sh index 23fa8972..680b50ca 100755 --- a/scripts/install-worker-controller.sh +++ b/scripts/install-worker-controller.sh @@ -415,33 +415,40 @@ build_candidate() { } make_checkpoint() { - local timestamp target unit timer + local timestamp target unit timer final_checkpoint staged_checkpoint timestamp=$(date -u +%Y%m%dT%H%M%SZ) - checkpoint_dir=$checkpoints_dir/${timestamp}-$$ - install -d -m 0700 "$checkpoint_dir" "$checkpoint_dir/systemd" + final_checkpoint=$checkpoints_dir/${timestamp}-$$ + install -d -m 0700 "$checkpoints_dir" + staged_checkpoint=$(mktemp -d "$checkpoints_dir/.checkpoint.staging.XXXXXX") + staging_paths+=("$staged_checkpoint") + checkpoint_dir=$staged_checkpoint + install -d -m 0700 "$checkpoint_dir/systemd" [[ ! -f "$rendered_env" ]] || install -m 0600 "$rendered_env" "$checkpoint_dir/ci-fleet.env" [[ ! -f "$state_file" ]] || install -m 0600 "$state_file" "$checkpoint_dir/install-state.json" target=$(current_runtime_release) if [[ -n "$target" ]]; then - printf '%s\n' "$target" >"$temporary/release-target" - install -m 0600 "$temporary/release-target" "$checkpoint_dir/release-target" + printf '%s\n' "$target" >"$checkpoint_dir/release-target" + chmod 0600 "$checkpoint_dir/release-target" fi if [[ -L "$manager_current" ]]; then target=$(readlink -f "$manager_current") - printf '%s\n' "$target" >"$temporary/manager-target" - install -m 0600 "$temporary/manager-target" "$checkpoint_dir/manager-target" + printf '%s\n' "$target" >"$checkpoint_dir/manager-target" + chmod 0600 "$checkpoint_dir/manager-target" fi for unit in "${unit_names[@]}"; do [[ ! -f "$systemd_dir/$unit" ]] || install -m 0644 "$systemd_dir/$unit" "$checkpoint_dir/systemd/$unit" done - : >"$temporary/enabled-timers" - : >"$temporary/active-timers" + : >"$checkpoint_dir/enabled-timers" + : >"$checkpoint_dir/active-timers" for timer in "${timer_names[@]}"; do - if systemctl is-enabled --quiet "$timer" 2>/dev/null; then printf '%s\n' "$timer" >>"$temporary/enabled-timers"; fi - if systemctl is-active --quiet "$timer" 2>/dev/null; then printf '%s\n' "$timer" >>"$temporary/active-timers"; fi + if systemctl is-enabled --quiet "$timer" 2>/dev/null; then printf '%s\n' "$timer" >>"$checkpoint_dir/enabled-timers"; fi + if systemctl is-active --quiet "$timer" 2>/dev/null; then printf '%s\n' "$timer" >>"$checkpoint_dir/active-timers"; fi done - install -m 0600 "$temporary/enabled-timers" "$checkpoint_dir/enabled-timers" - install -m 0600 "$temporary/active-timers" "$checkpoint_dir/active-timers" + chmod 0600 "$checkpoint_dir/enabled-timers" "$checkpoint_dir/active-timers" + : >"$checkpoint_dir/.complete" + chmod 0600 "$checkpoint_dir/.complete" + mv "$checkpoint_dir" "$final_checkpoint" + checkpoint_dir=$final_checkpoint note "CHECKPOINT_CREATED path=$checkpoint_dir" } @@ -518,7 +525,6 @@ install_systemd_units() { install -m 0644 "$source/host/systemd/ci-fleet-drift.service" "$systemd_dir/" install -m 0644 "$source/host/systemd/ci-fleet-drift.timer" "$systemd_dir/" systemctl daemon-reload - systemctl enable --now "${timer_names[@]}" >/dev/null } remove_systemd_units() { @@ -565,6 +571,7 @@ with open(sys.argv[2], "w", encoding="utf-8") as handle: PY chmod 0600 "$staged_state" mv -f "$staged_state" "$state_file" + systemctl enable --now "${timer_names[@]}" >/dev/null } restore_systemd_snapshot() { @@ -702,13 +709,12 @@ perform_converge() { latest_checkpoint() { [[ -d "$checkpoints_dir" ]] || return 0 - { find "$checkpoints_dir" -mindepth 1 -maxdepth 1 -type d -printf '%T@ %p\n' 2>/dev/null || true; } | sort -nr | awk 'NR == 1 {print $2}' + { find "$checkpoints_dir" -mindepth 2 -maxdepth 2 -type f -name .complete -printf '%T@ %h\n' 2>/dev/null || true; } | sort -nr | awk 'NR == 1 {print $2}' } perform_rollback() { checkpoint_dir=$(latest_checkpoint) [[ -n "$checkpoint_dir" ]] || die 'no controller checkpoint is available' - drain_current restore_checkpoint || die 'checkpoint restoration failed' note "ROLLBACK_OK checkpoint=$checkpoint_dir" } diff --git a/scripts/test-install-worker-controller.sh b/scripts/test-install-worker-controller.sh index d1285456..e0b70fcb 100755 --- a/scripts/test-install-worker-controller.sh +++ b/scripts/test-install-worker-controller.sh @@ -85,6 +85,9 @@ EOF cat >"$fake_bin/systemctl" <<'EOF' #!/usr/bin/env bash +if [[ "${1:-}" == enable && "${2:-}" == --now && ! -f "${CI_FLEET_ROOT_PREFIX:-}/var/lib/ci-fleet/install-state.json" ]]; then + exit 98 +fi if [[ -n "${FAKE_DISABLED_TIMER:-}" && ( "${1:-}" == is-enabled || "${1:-}" == is-active ) && $# == 3 && "${3:-}" == "$FAKE_DISABLED_TIMER" ]]; then exit 1 fi @@ -264,7 +267,10 @@ grep -Fq 'CI_FLEET_MAX_RUNNERS=1' "$root/etc/ci-fleet/ci-fleet.env" || fail 'res expect_success "$installer" --upgrade "${base_args[@]}" --ref "$ref_two" >/dev/null grep -Fq 'CI_FLEET_MAX_RUNNERS=2' "$root/etc/ci-fleet/ci-fleet.env" || fail 'upgrade did not apply capacity two' +mkdir -p "$root/var/lib/ci-fleet/checkpoints/99999999-incomplete" +printf 'restarting\n' >"$FAKE_CONTROLLER_STATUS_FILE" expect_success "$installer" --rollback >/dev/null +[[ ! -f "$FAKE_CONTROLLER_STATUS_FILE" ]] || fail 'explicit rollback did not recover a restarting controller' grep -Fq 'CI_FLEET_MAX_RUNNERS=1' "$root/etc/ci-fleet/ci-fleet.env" || fail 'rollback did not restore capacity one' ref_three=$(write_config drained 2 2) From 61b9728eda98c70db93f2254c2dc5a955a1774b7 Mon Sep 17 00:00:00 2001 From: Nickfosts Hermes Date: Sat, 18 Jul 2026 23:17:24 -0500 Subject: [PATCH 30/57] docs: advance transaction-safe engine pin --- templates/config-repository/examples/multi-host/fleet.json | 4 ++-- templates/config-repository/fleet.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/templates/config-repository/examples/multi-host/fleet.json b/templates/config-repository/examples/multi-host/fleet.json index d2036932..2524740d 100644 --- a/templates/config-repository/examples/multi-host/fleet.json +++ b/templates/config-repository/examples/multi-host/fleet.json @@ -24,7 +24,7 @@ "state": "active", "scale_set_name": "sample-ci-primary", "lifecycle": "stable", - "engine_ref": "40309bea81af48bacaa1825441d32731f4c5e018", + "engine_ref": "4f0ae568cac6dc627d26b5f67bd5feb6fd7e0520", "min_runners": 0, "max_runners": 4, "runner_resources": { @@ -38,7 +38,7 @@ "state": "active", "scale_set_name": "sample-ci-remote", "lifecycle": "stable", - "engine_ref": "40309bea81af48bacaa1825441d32731f4c5e018", + "engine_ref": "4f0ae568cac6dc627d26b5f67bd5feb6fd7e0520", "min_runners": 0, "max_runners": 2, "runner_resources": { diff --git a/templates/config-repository/fleet.json b/templates/config-repository/fleet.json index e2195066..e8c8c65f 100644 --- a/templates/config-repository/fleet.json +++ b/templates/config-repository/fleet.json @@ -24,7 +24,7 @@ "state": "active", "scale_set_name": "example-ci-01", "lifecycle": "experimental", - "engine_ref": "40309bea81af48bacaa1825441d32731f4c5e018", + "engine_ref": "4f0ae568cac6dc627d26b5f67bd5feb6fd7e0520", "min_runners": 0, "max_runners": 1, "runner_resources": { From 0206d2566b684797f8ae3763a24631b5e1e4edab Mon Sep 17 00:00:00 2001 From: Nickfosts Hermes Date: Tue, 21 Jul 2026 10:17:18 -0500 Subject: [PATCH 31/57] fix(installer): repair managed release drift --- scripts/install-worker-controller.sh | 93 ++++++++++++++++------- scripts/test-install-worker-controller.sh | 90 +++++++++++++++++++++- 2 files changed, 154 insertions(+), 29 deletions(-) diff --git a/scripts/install-worker-controller.sh b/scripts/install-worker-controller.sh index 680b50ca..8466ec64 100755 --- a/scripts/install-worker-controller.sh +++ b/scripts/install-worker-controller.sh @@ -297,22 +297,41 @@ raise SystemExit(0 if installed == candidate else 1) PY } +runtime_release_complete() { + local path=$1 expected=$2 marker + [[ -d "$path" && -f "$path/.ci-fleet-engine-ref" && -f "$path/deploy/compose.yaml" ]] || return 1 + [[ -x "$path/scripts/preflight.sh" && -x "$path/scripts/healthcheck.sh" && -x "$path/scripts/cleanup.sh" ]] || return 1 + marker=$(<"$path/.ci-fleet-engine-ref") + [[ "$marker" == "$expected" ]] +} + +manager_release_complete() { + local path=$1 expected=$2 marker unit + runtime_release_complete "$path" "$expected" || return 1 + [[ -x "$path/scripts/install-worker-controller.sh" && -x "$path/scripts/check-installed-state.sh" ]] || return 1 + for unit in "${unit_names[@]}"; do [[ -f "$path/host/systemd/$unit" ]] || return 1; done + marker=$(<"$path/.ci-fleet-engine-ref") + [[ "$marker" == "$expected" ]] +} + release_matches() { - local marker - [[ -d "$release_dir" && -f "$release_dir/.ci-fleet-engine-ref" && -f "$release_dir/deploy/compose.yaml" && -x "$release_dir/scripts/cleanup.sh" ]] || return 1 - marker=$(<"$release_dir/.ci-fleet-engine-ref") - [[ "$marker" == "$engine_ref" ]] || return 1 + runtime_release_complete "$release_dir" "$engine_ref" || return 1 [[ -L "$current_link" ]] || return 1 [[ $(readlink -f "$current_link") == $(readlink -f "$release_dir") ]] } +managed_images_match() { + local image + local -a expected_images=() + mapfile -t expected_images < <(awk -F= '$1 == "CI_FLEET_CONTROLLER_IMAGE" || $1 == "CI_FLEET_RUNNER_IMAGE" {print substr($0, index($0, "=") + 1)}' "$candidate_env") + [[ ${#expected_images[@]} == 2 ]] || return 1 + for image in "${expected_images[@]}"; do docker image inspect "$image" >/dev/null 2>&1 || return 1; done +} + systemd_matches() { - local expected_manager marker unit + local expected_manager unit expected_manager=$manager_releases/$engine_ref - [[ -d "$expected_manager" && -f "$expected_manager/.ci-fleet-engine-ref" ]] || return 1 - [[ -x "$expected_manager/scripts/healthcheck.sh" && -x "$expected_manager/scripts/check-installed-state.sh" ]] || return 1 - marker=$(<"$expected_manager/.ci-fleet-engine-ref") - [[ "$marker" == "$engine_ref" ]] || return 1 + manager_release_complete "$expected_manager" "$engine_ref" || return 1 [[ -L "$manager_current" ]] || return 1 [[ $(readlink -f "$manager_current") == $(readlink -f "$expected_manager") ]] || return 1 for unit in "${unit_names[@]}"; do @@ -334,16 +353,42 @@ drift_count() { release_matches || { note 'DRIFT engine_release'; count=$((count + 1)); } state_matches || { note 'DRIFT install_state'; count=$((count + 1)); } runtime_matches "$target_state" || { note 'DRIFT controller_runtime'; count=$((count + 1)); } + managed_images_match || { note 'DRIFT managed_images'; count=$((count + 1)); } systemd_matches || { note 'DRIFT maintenance_timers'; count=$((count + 1)); } DRIFT_COUNT=$count } +atomic_replace_directory() { + local replacement=$1 target=$2 + if [[ ! -e "$target" && ! -L "$target" ]]; then + mv "$replacement" "$target" + return + fi + python3 - "$replacement" "$target" <<'PY' +import ctypes +import os +import sys + +replacement, target = map(os.fsencode, sys.argv[1:]) +libc = ctypes.CDLL(None, use_errno=True) +renameat2 = getattr(libc, "renameat2", None) +if renameat2 is None: + raise OSError("atomic directory exchange is unavailable") +renameat2.argtypes = [ctypes.c_int, ctypes.c_char_p, ctypes.c_int, ctypes.c_char_p, ctypes.c_uint] +if renameat2(-100, replacement, -100, target, 2) != 0: # AT_FDCWD, RENAME_EXCHANGE + error = ctypes.get_errno() + raise OSError(error, os.strerror(error), os.fsdecode(target)) +parent = os.open(os.path.dirname(target), os.O_RDONLY | os.O_DIRECTORY) +try: + os.fsync(parent) +finally: + os.close(parent) +PY +} + install_release() { - local archive marker_commit checkout resolved staged_release - if [[ -d "$release_dir" ]]; then - [[ -f "$release_dir/.ci-fleet-engine-ref" && -f "$release_dir/deploy/compose.yaml" && -x "$release_dir/scripts/preflight.sh" && -x "$release_dir/scripts/healthcheck.sh" && -x "$release_dir/scripts/cleanup.sh" ]] || die "existing release is incomplete: $release_dir" - marker_commit=$(<"$release_dir/.ci-fleet-engine-ref") - [[ "$marker_commit" == "$engine_ref" ]] || die "existing release marker does not match $engine_ref" + local archive checkout resolved staged_release + if runtime_release_complete "$release_dir" "$engine_ref"; then return fi install -d -m 0755 "$releases_dir" @@ -366,18 +411,17 @@ install_release() { tar -xf "$archive" -C "$staged_release" printf '%s\n' "$engine_ref" >"$staged_release/.ci-fleet-engine-ref" chmod 0644 "$staged_release/.ci-fleet-engine-ref" - mv "$staged_release" "$release_dir" + runtime_release_complete "$staged_release" "$engine_ref" || die 'staged engine release is incomplete' + atomic_replace_directory "$staged_release" "$release_dir" } install_manager() { - local manager_commit manager_release archive marker staged_manager release_marker + local manager_commit manager_release archive staged_manager manager_commit=$engine_ref [[ "$manager_commit" =~ ^[0-9a-f]{40}$ ]] || die 'installer manager commit is invalid' - [[ -d "$release_dir" && -f "$release_dir/.ci-fleet-engine-ref" ]] || die 'desired engine release is unavailable for installer manager activation' - release_marker=$(<"$release_dir/.ci-fleet-engine-ref") - [[ "$release_marker" == "$manager_commit" ]] || die 'desired engine release marker is inconsistent' + runtime_release_complete "$release_dir" "$manager_commit" || die 'desired engine release is unavailable for installer manager activation' manager_release=$manager_releases/$manager_commit - if [[ ! -d "$manager_release" ]]; then + if ! manager_release_complete "$manager_release" "$manager_commit"; then install -d -m 0755 "$manager_releases" archive=$temporary/manager.tar tar -cf "$archive" -C "$release_dir" . @@ -387,11 +431,8 @@ install_manager() { tar -xf "$archive" -C "$staged_manager" printf '%s\n' "$manager_commit" >"$staged_manager/.ci-fleet-engine-ref" chmod 0644 "$staged_manager/.ci-fleet-engine-ref" - mv "$staged_manager" "$manager_release" - else - [[ -f "$manager_release/.ci-fleet-engine-ref" && -x "$manager_release/scripts/install-worker-controller.sh" && -x "$manager_release/scripts/healthcheck.sh" && -x "$manager_release/scripts/check-installed-state.sh" ]] || die 'existing installer manager release is incomplete' - marker=$(<"$manager_release/.ci-fleet-engine-ref") - [[ "$marker" == "$manager_commit" ]] || die 'existing installer manager marker is inconsistent' + manager_release_complete "$staged_manager" "$manager_commit" || die 'staged installer manager release is incomplete' + atomic_replace_directory "$staged_manager" "$manager_release" fi install -d -m 0755 "$manager_root" ln -sfn "$manager_release" "$temporary/manager-current" @@ -411,7 +452,7 @@ run_candidate_preflight() { build_candidate() { run_candidate_preflight compose "$release_dir" "$candidate_env" config --quiet - [[ "$target_state" == disabled ]] || compose "$release_dir" "$candidate_env" build runner-image controller + compose "$release_dir" "$candidate_env" build runner-image controller } make_checkpoint() { diff --git a/scripts/test-install-worker-controller.sh b/scripts/test-install-worker-controller.sh index e0b70fcb..9a6c322b 100755 --- a/scripts/test-install-worker-controller.sh +++ b/scripts/test-install-worker-controller.sh @@ -8,6 +8,8 @@ fake_bin=$tmp/bin mkdir -p "$fake_bin" export REAL_STAT REAL_STAT=$(command -v stat) +export REAL_TAR +REAL_TAR=$(command -v tar) cat >"$fake_bin/docker" <<'EOF' #!/usr/bin/env bash @@ -36,6 +38,17 @@ case "${1:-}" in fi exit 0 ;; + image) + [[ "${2:-}" == inspect && -n "${3:-}" ]] || exit 1 + [[ -z "${FAKE_IMAGE_INSPECT_LOG:-}" ]] || printf '%s\n' "$3" >>"$FAKE_IMAGE_INSPECT_LOG" + if [[ "$3" == "${FAKE_RUNNER_IMAGE:-}" ]]; then + [[ -f "${FAKE_RUNNER_IMAGE_STATE:-}" ]] + elif [[ "$3" == "${FAKE_CONTROLLER_IMAGE:-}" ]]; then + [[ -f "${FAKE_CONTROLLER_IMAGE_STATE:-}" ]] + else + exit 1 + fi + ;; volume|network) [[ "${2:-}" == ls ]] && exit 0 [[ "${2:-}" == inspect ]] && exit 1 @@ -75,7 +88,11 @@ case "${1:-}" in fi [[ -z "$paused_state" ]] || rm -f "$paused_state" ;; - config|build|logs) ;; + build) + [[ -z "${FAKE_RUNNER_IMAGE_STATE:-}" ]] || : >"$FAKE_RUNNER_IMAGE_STATE" + [[ -z "${FAKE_CONTROLLER_IMAGE_STATE:-}" ]] || : >"$FAKE_CONTROLLER_IMAGE_STATE" + ;; + config|logs) ;; *) exit 1 ;; esac ;; @@ -104,6 +121,16 @@ exec "$REAL_STAT" "$@" EOF chmod 700 "$fake_bin/docker" "$fake_bin/systemctl" "$fake_bin/stat" +cat >"$fake_bin/tar" <<'EOF' +#!/usr/bin/env bash +if [[ -n "${FAKE_FAIL_TAR_ONCE:-}" && -f "$FAKE_FAIL_TAR_ONCE" ]]; then + rm -f "$FAKE_FAIL_TAR_ONCE" + exit 45 +fi +exec "$REAL_TAR" "$@" +EOF +chmod 700 "$fake_bin/tar" + export PATH="$fake_bin:$PATH" export FAKE_DOCKER_STATE=$tmp/docker-controller-running export FAKE_CONTROLLER_STATUS_FILE=$tmp/docker-controller-status @@ -127,8 +154,17 @@ expect_failure() { if output=$("$@" 2>&1); then fail "expected failure: $*"; fi grep -Fq -- "$expected" <<<"$output" || fail "missing failure [$expected]: $output" } +expect_command_failure() { + local output + if output=$("$@" 2>&1); then fail "expected failure: $*"; fi +} engine_ref=$(git -C "$repo_root" rev-parse 'HEAD^{commit}') +export FAKE_RUNNER_IMAGE=ci-fleet-runner:${engine_ref:0:12} +export FAKE_CONTROLLER_IMAGE=ci-fleet-controller:${engine_ref:0:12} +export FAKE_RUNNER_IMAGE_STATE=$tmp/runner-image-present +export FAKE_CONTROLLER_IMAGE_STATE=$tmp/controller-image-present +export FAKE_IMAGE_INSPECT_LOG=$tmp/image-inspects config_repo=$tmp/config-repo git init -q "$config_repo" git -C "$config_repo" config user.name fixture @@ -218,10 +254,56 @@ second=$(expect_success "$installer" --install "${base_args[@]}" --ref "$ref_one grep -Fq 'NO_CHANGE' <<<"$second" || fail 'idempotent rerun changed the host' expect_success "$installer" --check "${base_args[@]}" --ref "$ref_one" >/dev/null +rm -f "$FAKE_RUNNER_IMAGE_STATE" +expect_failure 'DRIFT managed_images' "$installer" --check "${base_args[@]}" --ref "$ref_one" +image_repair=$(expect_success "$installer" --install "${base_args[@]}" --ref "$ref_one") +grep -Fq 'CONVERGED mode=install' <<<"$image_repair" || fail 'missing runner image did not trigger convergence' +[[ -f "$FAKE_RUNNER_IMAGE_STATE" && -f "$FAKE_CONTROLLER_IMAGE_STATE" ]] || fail 'candidate build did not restore both managed images' +rm -f "$FAKE_CONTROLLER_IMAGE_STATE" +expect_failure 'DRIFT managed_images' "$installer" --check "${base_args[@]}" --ref "$ref_one" +expect_success "$installer" --install "${base_args[@]}" --ref "$ref_one" >/dev/null +[[ -f "$FAKE_RUNNER_IMAGE_STATE" && -f "$FAKE_CONTROLLER_IMAGE_STATE" ]] || fail 'candidate build did not restore the controller image' +if docker image inspect unrelated:image >/dev/null 2>&1; then fail 'unrelated image fixture unexpectedly exists'; fi +: >"$FAKE_IMAGE_INSPECT_LOG" +expect_success "$installer" --check "${base_args[@]}" --ref "$ref_one" >/dev/null +grep -Fxq "$FAKE_RUNNER_IMAGE" "$FAKE_IMAGE_INSPECT_LOG" || fail 'runner image was not inspected' +grep -Fxq "$FAKE_CONTROLLER_IMAGE" "$FAKE_IMAGE_INSPECT_LOG" || fail 'controller image was not inspected' +if grep -Fvx -e "$FAKE_RUNNER_IMAGE" -e "$FAKE_CONTROLLER_IMAGE" "$FAKE_IMAGE_INSPECT_LOG" >/dev/null; then fail 'an unrelated image was inspected'; fi + active_release=$(readlink -f "$root/opt/ci-fleet/current") -mv "$active_release/scripts/cleanup.sh" "$active_release/scripts/cleanup.sh.missing" +unrelated_release=$root/opt/ci-fleet/releases/unrelated-release +mkdir -p "$unrelated_release" +: >"$unrelated_release/preserve" +mv "$active_release/deploy/compose.yaml" "$active_release/deploy/compose.yaml.missing" +expect_failure 'DRIFT engine_release' "$installer" --check "${base_args[@]}" --ref "$ref_one" +expect_success "$installer" --install "${base_args[@]}" --ref "$ref_one" >/dev/null +[[ -f "$active_release/deploy/compose.yaml" && -f "$unrelated_release/preserve" ]] || fail 'Compose repair removed unrelated release state' +rm -f "$active_release/scripts/preflight.sh" +expect_failure 'DRIFT engine_release' "$installer" --check "${base_args[@]}" --ref "$ref_one" +expect_success "$installer" --install "${base_args[@]}" --ref "$ref_one" >/dev/null +[[ -x "$active_release/scripts/preflight.sh" ]] || fail 'required runtime script was not repaired' +rm -f "$active_release/.ci-fleet-engine-ref" expect_failure 'DRIFT engine_release' "$installer" --check "${base_args[@]}" --ref "$ref_one" -mv "$active_release/scripts/cleanup.sh.missing" "$active_release/scripts/cleanup.sh" +expect_success "$installer" --install "${base_args[@]}" --ref "$ref_one" >/dev/null +[[ $(<"$active_release/.ci-fleet-engine-ref") == "$engine_ref" ]] || fail 'missing release marker was not repaired' +if [[ ${engine_ref: -1} == 0 ]]; then bad_marker="${engine_ref%?}1"; else bad_marker="${engine_ref%?}0"; fi +printf '%s\n' "$bad_marker" >"$active_release/.ci-fleet-engine-ref" +expect_failure 'DRIFT engine_release' "$installer" --check "${base_args[@]}" --ref "$ref_one" +expect_success "$installer" --install "${base_args[@]}" --ref "$ref_one" >/dev/null +[[ $(<"$active_release/.ci-fleet-engine-ref") == "$engine_ref" ]] || fail 'bad release marker was not repaired' +rm -f "$active_release/deploy/compose.yaml" +export FAKE_FAIL_TAR_ONCE=$tmp/fail-tar-once +: >"$FAKE_FAIL_TAR_ONCE" +expect_command_failure "$installer" --install "${base_args[@]}" --ref "$ref_one" +unset FAKE_FAIL_TAR_ONCE +[[ ! -f "$active_release/deploy/compose.yaml" ]] || fail 'interrupted repair replaced the detectable prior state' +if compgen -G "$root/opt/ci-fleet/releases/.${engine_ref}.staging.*" >/dev/null; then fail 'interrupted release staging was not cleaned'; fi +expect_success "$installer" --install "${base_args[@]}" --ref "$ref_one" >/dev/null +manager_release=$(readlink -f "$root/opt/ci-fleet/manager/current") +rm -f "$manager_release/scripts/check-installed-state.sh" +expect_failure 'DRIFT maintenance_timers' "$installer" --check "${base_args[@]}" --ref "$ref_one" +expect_success "$installer" --install "${base_args[@]}" --ref "$ref_one" >/dev/null +[[ -x "$manager_release/scripts/check-installed-state.sh" ]] || fail 'incomplete manager release was not repaired consistently' mv "$active_release" "$active_release.saved" expect_failure 'DRIFT engine_release' "$installer" --check "${base_args[@]}" --ref "$ref_one" mv "$active_release.saved" "$active_release" @@ -234,7 +316,9 @@ printf '\n' >>"$root/etc/ci-fleet/ci-fleet.env" printf '\n# drift\n' >>"$root/etc/systemd/system/ci-fleet-health.timer" expect_failure 'DRIFT rendered_environment' "$installer" --check "${base_args[@]}" --ref "$ref_one" expect_failure 'DRIFT maintenance_timers' "$installer" --check "${base_args[@]}" --ref "$ref_one" +complete_release_inode=$(stat -c '%i' "$active_release") expect_success "$installer" --install "${base_args[@]}" --ref "$ref_one" >/dev/null +[[ $(stat -c '%i' "$active_release") == "$complete_release_inode" ]] || fail 'complete immutable release was replaced instead of reused' prior_manager=$root/opt/ci-fleet/manager/releases/prior-manager cp -a "$(readlink -f "$root/opt/ci-fleet/manager/current")" "$prior_manager" From a589eed48fb9d6664d5535dfbf5a9ecf5a147f67 Mon Sep 17 00:00:00 2001 From: Nickfosts Hermes Date: Tue, 21 Jul 2026 10:17:39 -0500 Subject: [PATCH 32/57] fix(config): align schema with engine contract --- templates/config-repository/fleet.schema.json | 4 +- .../config-repository/scripts/test_policy.py | 63 +++++++++++++++---- 2 files changed, 53 insertions(+), 14 deletions(-) diff --git a/templates/config-repository/fleet.schema.json b/templates/config-repository/fleet.schema.json index a1cb19ea..a8f0716c 100644 --- a/templates/config-repository/fleet.schema.json +++ b/templates/config-repository/fleet.schema.json @@ -15,7 +15,7 @@ "properties": { "slug": {"type": "string", "pattern": "^[a-z0-9][a-z0-9-]{0,38}$"}, "registry": {"type": "string", "pattern": "^[a-z0-9.-]+/[a-z0-9._/-]+$"}, - "delivery_engine": {"type": "string", "pattern": "^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$"}, + "delivery_engine": {"const": "RandomDevelopment/ci-fleet"}, "workflow_ref_policy": {"const": "immutable-commit"} } }, @@ -76,7 +76,7 @@ "state": {"enum": ["active", "drained", "disabled"]}, "scale_set_name": {"$ref": "#/$defs/slug"}, "lifecycle": {"enum": ["experimental", "stable", "retiring"]}, - "engine_ref": {"type": "string", "pattern": "^[0-9a-f]{40}$"}, + "engine_ref": {"type": "string", "pattern": "^(?!0{40}$)[0-9a-f]{40}$"}, "min_runners": {"const": 0}, "max_runners": {"type": "integer", "minimum": 1}, "runner_resources": {"$ref": "#/$defs/runner_resources"} diff --git a/templates/config-repository/scripts/test_policy.py b/templates/config-repository/scripts/test_policy.py index 6c76d79f..f769c0c4 100755 --- a/templates/config-repository/scripts/test_policy.py +++ b/templates/config-repository/scripts/test_policy.py @@ -5,6 +5,7 @@ import copy import json +import re import tempfile import unittest from pathlib import Path @@ -19,6 +20,20 @@ def reference_config() -> dict: return json.loads((ROOT / "fleet.json").read_text(encoding="utf-8")) +def contract_schema() -> dict: + return json.loads((ROOT / "fleet.schema.json").read_text(encoding="utf-8")) + + +def schema_accepts_engine_ref(value: str) -> bool: + pattern = contract_schema()["$defs"]["controller"]["properties"]["engine_ref"]["pattern"] + return re.fullmatch(pattern, value) is not None + + +def schema_accepts_delivery_engine(value: str) -> bool: + contract = contract_schema()["properties"]["organization"]["properties"]["delivery_engine"] + return contract.get("const") == value + + def errors_for(config: dict, *, strict: bool = False) -> list[str]: validation = Validation() scan_secret_material(config, validation) @@ -39,6 +54,18 @@ def assert_rejected(self, config: dict, expected: str, *, strict: bool = False) errors = errors_for(config, strict=strict) self.assertTrue(any(expected in error for error in errors), errors) + def assert_engine_ref_contract(self, value: str, accepted: bool) -> None: + config = copy.deepcopy(reference_config()) + first_controller(config)["engine_ref"] = value + self.assertEqual(schema_accepts_engine_ref(value), accepted) + self.assertEqual(errors_for(config) == [], accepted) + + def assert_delivery_engine_contract(self, value: str, accepted: bool) -> None: + config = copy.deepcopy(reference_config()) + config["organization"]["delivery_engine"] = value + self.assertEqual(schema_accepts_delivery_engine(value), accepted) + self.assertEqual(errors_for(config) == [], accepted) + def test_reference_configuration_is_valid(self) -> None: self.assertEqual(errors_for(reference_config()), []) @@ -103,25 +130,37 @@ def test_controller_pool_must_be_a_string(self) -> None: first_controller(config)["pool"] = ["trusted-ci"] self.assert_rejected(config, "must reference a declared runner pool") - def test_delivery_engine_repository_is_fixed(self) -> None: - config = copy.deepcopy(reference_config()) - config["organization"]["delivery_engine"] = "attacker/engine" - self.assert_rejected(config, "must use the fixed reviewed public engine repository") + def test_delivery_engine_repository_is_fixed_in_schema_and_semantics(self) -> None: + self.assert_delivery_engine_contract("RandomDevelopment/ci-fleet", True) + + def test_delivery_engine_rejects_another_repository(self) -> None: + self.assert_delivery_engine_contract("attacker/engine", False) + + def test_delivery_engine_rejects_url_form(self) -> None: + self.assert_delivery_engine_contract("https://github.com/RandomDevelopment/ci-fleet", False) + + def test_delivery_engine_rejects_credential_form(self) -> None: + self.assert_delivery_engine_contract("user:password@RandomDevelopment/ci-fleet", False) def test_controller_address_is_rejected(self) -> None: config = copy.deepcopy(reference_config()) first_controller(config)["ip_address"] = "192.0.2.10" self.assert_rejected(config, "host-local infrastructure details are forbidden") - def test_unpinned_engine_ref_is_rejected(self) -> None: - config = copy.deepcopy(reference_config()) - first_controller(config)["engine_ref"] = "main" - self.assert_rejected(config, "full lowercase commit SHA") + def test_full_lowercase_engine_ref_passes_schema_and_semantics(self) -> None: + self.assert_engine_ref_contract("1" * 40, True) - def test_zero_engine_ref_is_rejected(self) -> None: - config = copy.deepcopy(reference_config()) - first_controller(config)["engine_ref"] = "0" * 40 - self.assert_rejected(config, "nonzero full lowercase commit SHA") + def test_uppercase_engine_ref_fails_schema_and_semantics(self) -> None: + self.assert_engine_ref_contract("A" * 40, False) + + def test_short_engine_ref_fails_schema_and_semantics(self) -> None: + self.assert_engine_ref_contract("1" * 39, False) + + def test_malformed_engine_ref_fails_schema_and_semantics(self) -> None: + self.assert_engine_ref_contract("g" * 40, False) + + def test_zero_engine_ref_fails_schema_and_semantics(self) -> None: + self.assert_engine_ref_contract("0" * 40, False) def test_active_controller_minimum_must_be_zero(self) -> None: config = copy.deepcopy(reference_config()) From 0a4a54f86a54356899cca011bd4ea26c7460a1ab Mon Sep 17 00:00:00 2001 From: Nickfosts Hermes Date: Tue, 21 Jul 2026 10:27:35 -0500 Subject: [PATCH 33/57] docs: advance final reviewed engine pin --- templates/config-repository/examples/multi-host/fleet.json | 4 ++-- templates/config-repository/fleet.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/templates/config-repository/examples/multi-host/fleet.json b/templates/config-repository/examples/multi-host/fleet.json index 2524740d..9ecc6275 100644 --- a/templates/config-repository/examples/multi-host/fleet.json +++ b/templates/config-repository/examples/multi-host/fleet.json @@ -24,7 +24,7 @@ "state": "active", "scale_set_name": "sample-ci-primary", "lifecycle": "stable", - "engine_ref": "4f0ae568cac6dc627d26b5f67bd5feb6fd7e0520", + "engine_ref": "a589eed48fb9d6664d5535dfbf5a9ecf5a147f67", "min_runners": 0, "max_runners": 4, "runner_resources": { @@ -38,7 +38,7 @@ "state": "active", "scale_set_name": "sample-ci-remote", "lifecycle": "stable", - "engine_ref": "4f0ae568cac6dc627d26b5f67bd5feb6fd7e0520", + "engine_ref": "a589eed48fb9d6664d5535dfbf5a9ecf5a147f67", "min_runners": 0, "max_runners": 2, "runner_resources": { diff --git a/templates/config-repository/fleet.json b/templates/config-repository/fleet.json index e8c8c65f..9a10b994 100644 --- a/templates/config-repository/fleet.json +++ b/templates/config-repository/fleet.json @@ -24,7 +24,7 @@ "state": "active", "scale_set_name": "example-ci-01", "lifecycle": "experimental", - "engine_ref": "4f0ae568cac6dc627d26b5f67bd5feb6fd7e0520", + "engine_ref": "a589eed48fb9d6664d5535dfbf5a9ecf5a147f67", "min_runners": 0, "max_runners": 1, "runner_resources": { From 8fdc39d364243f01f2be68c75bdbafbd21a3419a Mon Sep 17 00:00:00 2001 From: Nickfosts Hermes Date: Tue, 21 Jul 2026 10:32:08 -0500 Subject: [PATCH 34/57] ci(config): verify immutable initializer inputs --- .../.github/workflows/validate.yml | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/templates/config-repository/.github/workflows/validate.yml b/templates/config-repository/.github/workflows/validate.yml index ad0549a7..e43495bb 100644 --- a/templates/config-repository/.github/workflows/validate.yml +++ b/templates/config-repository/.github/workflows/validate.yml @@ -39,7 +39,27 @@ jobs: - name: Prove initialized configurations pass strict policy run: | temporary_directory="$(mktemp -d)" + if ./scripts/init.sh \ + --organization test-company \ + --project test-app \ + --output "${temporary_directory}/missing-engine.json"; then + echo 'initializer accepted a missing engine revision' >&2 + exit 1 + fi + test ! -e "${temporary_directory}/missing-engine.json" + if ./scripts/init.sh \ + --organization test-company \ + --project test-app \ + --registry BAD \ + --engine-ref "$(git rev-parse HEAD)" \ + --output "${temporary_directory}/invalid.json"; then + echo 'initializer accepted an invalid registry' >&2 + exit 1 + fi + test ! -e "${temporary_directory}/invalid.json" ./scripts/init.sh \ --organization test-company \ --project test-app \ + --engine-ref "$(git rev-parse HEAD)" \ --output "${temporary_directory}/fleet.json" + test "$(stat -c '%a' "${temporary_directory}/fleet.json")" = 644 From 25539a39ba1f214a9eef0181c3332202638f7d6d Mon Sep 17 00:00:00 2001 From: Nickfosts Hermes Date: Tue, 21 Jul 2026 10:40:31 -0500 Subject: [PATCH 35/57] ci(config): initialize from reviewed engine pin --- templates/config-repository/.github/workflows/validate.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/templates/config-repository/.github/workflows/validate.yml b/templates/config-repository/.github/workflows/validate.yml index e43495bb..ad221494 100644 --- a/templates/config-repository/.github/workflows/validate.yml +++ b/templates/config-repository/.github/workflows/validate.yml @@ -39,6 +39,7 @@ jobs: - name: Prove initialized configurations pass strict policy run: | temporary_directory="$(mktemp -d)" + engine_ref=$(python3 -c 'import json; value = json.load(open("fleet.json", encoding="utf-8")); print(next(iter(value["controllers"].values()))["engine_ref"])') if ./scripts/init.sh \ --organization test-company \ --project test-app \ @@ -51,7 +52,7 @@ jobs: --organization test-company \ --project test-app \ --registry BAD \ - --engine-ref "$(git rev-parse HEAD)" \ + --engine-ref "${engine_ref}" \ --output "${temporary_directory}/invalid.json"; then echo 'initializer accepted an invalid registry' >&2 exit 1 @@ -60,6 +61,6 @@ jobs: ./scripts/init.sh \ --organization test-company \ --project test-app \ - --engine-ref "$(git rev-parse HEAD)" \ + --engine-ref "${engine_ref}" \ --output "${temporary_directory}/fleet.json" test "$(stat -c '%a' "${temporary_directory}/fleet.json")" = 644 From 89d46b8ab9cc997577867d84ceee457aa50e1ac9 Mon Sep 17 00:00:00 2001 From: Nickfosts Hermes Date: Tue, 21 Jul 2026 11:21:17 -0500 Subject: [PATCH 36/57] fix(installer): close final drift and rollback gaps --- controller/Dockerfile | 2 + deploy/compose.yaml | 1 + runner/Dockerfile | 2 + scripts/install-worker-controller.sh | 13 ++++-- scripts/test-install-worker-controller.sh | 49 ++++++++++++++++++----- 5 files changed, 55 insertions(+), 12 deletions(-) diff --git a/controller/Dockerfile b/controller/Dockerfile index 4999dc6f..a2c17990 100644 --- a/controller/Dockerfile +++ b/controller/Dockerfile @@ -8,6 +8,8 @@ ARG CI_FLEET_COMMIT=unknown RUN CGO_ENABLED=0 go build -trimpath -ldflags="-s -w -X main.version=${CI_FLEET_VERSION} -X main.commitSHA=${CI_FLEET_COMMIT}" -o /out/ci-fleet-controller . FROM debian:13.6-slim +ARG CI_FLEET_COMMIT=unknown +LABEL org.opencontainers.image.revision="${CI_FLEET_COMMIT}" RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates && rm -rf /var/lib/apt/lists/* COPY --from=build /out/ci-fleet-controller /usr/local/bin/ci-fleet-controller USER 65532:65532 diff --git a/deploy/compose.yaml b/deploy/compose.yaml index 02c3d203..d84fd5da 100644 --- a/deploy/compose.yaml +++ b/deploy/compose.yaml @@ -6,6 +6,7 @@ services: build: context: ../runner args: + CI_FLEET_COMMIT: ${CI_FLEET_COMMIT:-unknown} RUNNER_VERSION: ${CI_FLEET_RUNNER_VERSION:-2.335.1} command: ["/bin/true"] deploy: diff --git a/runner/Dockerfile b/runner/Dockerfile index eefda989..6b0b6030 100644 --- a/runner/Dockerfile +++ b/runner/Dockerfile @@ -2,9 +2,11 @@ FROM debian:13.6-slim ARG TARGETARCH +ARG CI_FLEET_COMMIT=unknown ARG RUNNER_VERSION=2.335.1 ARG RUNNER_SHA256_AMD64=4ef2f25285f0ae4477f1fe1e346db76d2f3ebf03824e2ddd1973a2819bf6c8cf ARG RUNNER_SHA256_ARM64=6d1e85bfd1a506a8b17c1f1b9b57dba458ffed90898799aaa9f599520b0d9207 +LABEL org.opencontainers.image.revision="${CI_FLEET_COMMIT}" SHELL ["/bin/bash", "-o", "pipefail", "-c"] RUN apt-get update \ diff --git a/scripts/install-worker-controller.sh b/scripts/install-worker-controller.sh index 8466ec64..53af2dab 100755 --- a/scripts/install-worker-controller.sh +++ b/scripts/install-worker-controller.sh @@ -321,11 +321,14 @@ release_matches() { } managed_images_match() { - local image + local image provenance local -a expected_images=() mapfile -t expected_images < <(awk -F= '$1 == "CI_FLEET_CONTROLLER_IMAGE" || $1 == "CI_FLEET_RUNNER_IMAGE" {print substr($0, index($0, "=") + 1)}' "$candidate_env") [[ ${#expected_images[@]} == 2 ]] || return 1 - for image in "${expected_images[@]}"; do docker image inspect "$image" >/dev/null 2>&1 || return 1; done + for image in "${expected_images[@]}"; do + provenance=$(docker image inspect --format '{{ index .Config.Labels "org.opencontainers.image.revision" }}' "$image" 2>/dev/null) || return 1 + [[ "$provenance" == "$engine_ref" ]] || return 1 + done } systemd_matches() { @@ -353,6 +356,10 @@ drift_count() { release_matches || { note 'DRIFT engine_release'; count=$((count + 1)); } state_matches || { note 'DRIFT install_state'; count=$((count + 1)); } runtime_matches "$target_state" || { note 'DRIFT controller_runtime'; count=$((count + 1)); } + if [[ "$target_state" != active && $(managed_runner_count) != 0 ]]; then + note 'DRIFT managed_runners' + count=$((count + 1)) + fi managed_images_match || { note 'DRIFT managed_images'; count=$((count + 1)); } systemd_matches || { note 'DRIFT maintenance_timers'; count=$((count + 1)); } DRIFT_COUNT=$count @@ -750,7 +757,7 @@ perform_converge() { latest_checkpoint() { [[ -d "$checkpoints_dir" ]] || return 0 - { find "$checkpoints_dir" -mindepth 2 -maxdepth 2 -type f -name .complete -printf '%T@ %h\n' 2>/dev/null || true; } | sort -nr | awk 'NR == 1 {print $2}' + { find "$checkpoints_dir" -mindepth 2 -maxdepth 2 -type f -name .complete ! -path "$checkpoints_dir/.checkpoint.staging.*/*" -printf '%T@ %h\n' 2>/dev/null || true; } | sort -nr | awk 'NR == 1 {print $2}' } perform_rollback() { diff --git a/scripts/test-install-worker-controller.sh b/scripts/test-install-worker-controller.sh index 9a6c322b..e60e7ee4 100755 --- a/scripts/test-install-worker-controller.sh +++ b/scripts/test-install-worker-controller.sh @@ -39,15 +39,18 @@ case "${1:-}" in exit 0 ;; image) - [[ "${2:-}" == inspect && -n "${3:-}" ]] || exit 1 - [[ -z "${FAKE_IMAGE_INSPECT_LOG:-}" ]] || printf '%s\n' "$3" >>"$FAKE_IMAGE_INSPECT_LOG" - if [[ "$3" == "${FAKE_RUNNER_IMAGE:-}" ]]; then - [[ -f "${FAKE_RUNNER_IMAGE_STATE:-}" ]] - elif [[ "$3" == "${FAKE_CONTROLLER_IMAGE:-}" ]]; then - [[ -f "${FAKE_CONTROLLER_IMAGE_STATE:-}" ]] + [[ "${2:-}" == inspect ]] || exit 1 + image=${!#} + [[ -z "${FAKE_IMAGE_INSPECT_LOG:-}" ]] || printf '%s\n' "$image" >>"$FAKE_IMAGE_INSPECT_LOG" + if [[ "$image" == "${FAKE_RUNNER_IMAGE:-}" ]]; then + image_state=${FAKE_RUNNER_IMAGE_STATE:-} + elif [[ "$image" == "${FAKE_CONTROLLER_IMAGE:-}" ]]; then + image_state=${FAKE_CONTROLLER_IMAGE_STATE:-} else exit 1 fi + [[ -f "$image_state" ]] || exit 1 + cat "$image_state" ;; volume|network) [[ "${2:-}" == ls ]] && exit 0 @@ -89,8 +92,8 @@ case "${1:-}" in [[ -z "$paused_state" ]] || rm -f "$paused_state" ;; build) - [[ -z "${FAKE_RUNNER_IMAGE_STATE:-}" ]] || : >"$FAKE_RUNNER_IMAGE_STATE" - [[ -z "${FAKE_CONTROLLER_IMAGE_STATE:-}" ]] || : >"$FAKE_CONTROLLER_IMAGE_STATE" + [[ -z "${FAKE_RUNNER_IMAGE_STATE:-}" ]] || printf '%s\n' "${FAKE_ENGINE_REF:?}" >"$FAKE_RUNNER_IMAGE_STATE" + [[ -z "${FAKE_CONTROLLER_IMAGE_STATE:-}" ]] || printf '%s\n' "${FAKE_ENGINE_REF:?}" >"$FAKE_CONTROLLER_IMAGE_STATE" ;; config|logs) ;; *) exit 1 ;; @@ -160,11 +163,17 @@ expect_command_failure() { } engine_ref=$(git -C "$repo_root" rev-parse 'HEAD^{commit}') -export FAKE_RUNNER_IMAGE=ci-fleet-runner:${engine_ref:0:12} +export FAKE_ENGINE_REF=$engine_ref +runner_image="ci-fleet-runner:${engine_ref:0:12}" +export FAKE_RUNNER_IMAGE=$runner_image export FAKE_CONTROLLER_IMAGE=ci-fleet-controller:${engine_ref:0:12} export FAKE_RUNNER_IMAGE_STATE=$tmp/runner-image-present export FAKE_CONTROLLER_IMAGE_STATE=$tmp/controller-image-present export FAKE_IMAGE_INSPECT_LOG=$tmp/image-inspects +for dockerfile in "$repo_root/controller/Dockerfile" "$repo_root/runner/Dockerfile"; do + grep -Fq 'LABEL org.opencontainers.image.revision="${CI_FLEET_COMMIT}"' "$dockerfile" || fail "managed image lacks engine provenance label: $dockerfile" +done +grep -Fq 'CI_FLEET_COMMIT: ${CI_FLEET_COMMIT:-unknown}' "$repo_root/deploy/compose.yaml" || fail 'runner build lacks engine provenance argument' config_repo=$tmp/config-repo git init -q "$config_repo" git -C "$config_repo" config user.name fixture @@ -223,7 +232,11 @@ git -C "$config_repo" reset -q --hard "$ref_one" installer=$repo_root/scripts/install-worker-controller.sh base_args=(--config-repo "$config_repo" --controller example-ci-01) +staged_checkpoint="$root/var/lib/ci-fleet/checkpoints/.checkpoint.staging.interrupted" +mkdir -p "$staged_checkpoint" +: >"$staged_checkpoint/.complete" expect_failure 'no controller checkpoint is available' "$installer" --rollback +rm -rf "$staged_checkpoint" expect_failure 'secret-bearing files are forbidden' "$installer" --check "${base_args[@]}" --ref "$forbidden_ref" expect_failure 'possible committed secret detected' "$installer" --check "${base_args[@]}" --ref "$secret_ref" export FAKE_WRONG_HOST_CONFIG_OWNER=$host_config @@ -263,6 +276,10 @@ rm -f "$FAKE_CONTROLLER_IMAGE_STATE" expect_failure 'DRIFT managed_images' "$installer" --check "${base_args[@]}" --ref "$ref_one" expect_success "$installer" --install "${base_args[@]}" --ref "$ref_one" >/dev/null [[ -f "$FAKE_RUNNER_IMAGE_STATE" && -f "$FAKE_CONTROLLER_IMAGE_STATE" ]] || fail 'candidate build did not restore the controller image' +printf '%040d\n' 0 >"$FAKE_RUNNER_IMAGE_STATE" +expect_failure 'DRIFT managed_images' "$installer" --check "${base_args[@]}" --ref "$ref_one" +expect_success "$installer" --install "${base_args[@]}" --ref "$ref_one" >/dev/null +[[ $(<"$FAKE_RUNNER_IMAGE_STATE") == "$engine_ref" && $(<"$FAKE_CONTROLLER_IMAGE_STATE") == "$engine_ref" ]] || fail 'candidate build did not restore managed image provenance' if docker image inspect unrelated:image >/dev/null 2>&1; then fail 'unrelated image fixture unexpectedly exists'; fi : >"$FAKE_IMAGE_INSPECT_LOG" expect_success "$installer" --check "${base_args[@]}" --ref "$ref_one" >/dev/null @@ -362,6 +379,20 @@ expect_success "$installer" --upgrade "${base_args[@]}" --ref "$ref_three" >/dev grep -Fq 'CI_FLEET_CONTROLLER_STATE=drained' "$root/etc/ci-fleet/ci-fleet.env" || fail 'drained state was not rendered' grep -Fq 'CI_FLEET_MAX_RUNNERS=0' "$root/etc/ci-fleet/ci-fleet.env" || fail 'drained controller retained effective capacity' [[ ! -f "$FAKE_DOCKER_STATE" ]] || fail 'drained controller remained running' +export FAKE_RUNNER_STATE=$tmp/drained-managed-runner +: >"$FAKE_RUNNER_STATE" +expect_failure 'DRIFT managed_runners' "$installer" --check "${base_args[@]}" --ref "$ref_three" +rm -f "$FAKE_RUNNER_STATE" +unset FAKE_RUNNER_STATE + +ref_four=$(write_config disabled 2 2) +expect_success "$installer" --upgrade "${base_args[@]}" --ref "$ref_four" >/dev/null +grep -Fq 'CI_FLEET_CONTROLLER_STATE=disabled' "$root/etc/ci-fleet/ci-fleet.env" || fail 'disabled state was not rendered' +export FAKE_RUNNER_STATE=$tmp/disabled-managed-runner +: >"$FAKE_RUNNER_STATE" +expect_failure 'DRIFT managed_runners' "$installer" --check "${base_args[@]}" --ref "$ref_four" +rm -f "$FAKE_RUNNER_STATE" +unset FAKE_RUNNER_STATE export FAKE_RUNNER_STATE_ONCE=$tmp/orphaned-managed-runner : >"$FAKE_RUNNER_STATE_ONCE" From b688df6f5c26f083e74508130ed6140acbb5e0c7 Mon Sep 17 00:00:00 2001 From: Nickfosts Hermes Date: Tue, 21 Jul 2026 11:27:08 -0500 Subject: [PATCH 37/57] test(installer): satisfy literal label lint --- scripts/test-install-worker-controller.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/test-install-worker-controller.sh b/scripts/test-install-worker-controller.sh index e60e7ee4..5fdf399c 100755 --- a/scripts/test-install-worker-controller.sh +++ b/scripts/test-install-worker-controller.sh @@ -171,9 +171,9 @@ export FAKE_RUNNER_IMAGE_STATE=$tmp/runner-image-present export FAKE_CONTROLLER_IMAGE_STATE=$tmp/controller-image-present export FAKE_IMAGE_INSPECT_LOG=$tmp/image-inspects for dockerfile in "$repo_root/controller/Dockerfile" "$repo_root/runner/Dockerfile"; do - grep -Fq 'LABEL org.opencontainers.image.revision="${CI_FLEET_COMMIT}"' "$dockerfile" || fail "managed image lacks engine provenance label: $dockerfile" + grep -Fq "LABEL org.opencontainers.image.revision=\"\${CI_FLEET_COMMIT}\"" "$dockerfile" || fail "managed image lacks engine provenance label: $dockerfile" done -grep -Fq 'CI_FLEET_COMMIT: ${CI_FLEET_COMMIT:-unknown}' "$repo_root/deploy/compose.yaml" || fail 'runner build lacks engine provenance argument' +grep -Fq "CI_FLEET_COMMIT: \${CI_FLEET_COMMIT:-unknown}" "$repo_root/deploy/compose.yaml" || fail 'runner build lacks engine provenance argument' config_repo=$tmp/config-repo git init -q "$config_repo" git -C "$config_repo" config user.name fixture From 959db552e96392d5cfc67c6750730f1f03c405c4 Mon Sep 17 00:00:00 2001 From: Nickfosts Hermes Date: Tue, 21 Jul 2026 11:36:36 -0500 Subject: [PATCH 38/57] docs: pin final repaired engine --- templates/config-repository/examples/multi-host/fleet.json | 4 ++-- templates/config-repository/fleet.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/templates/config-repository/examples/multi-host/fleet.json b/templates/config-repository/examples/multi-host/fleet.json index 9ecc6275..01f52791 100644 --- a/templates/config-repository/examples/multi-host/fleet.json +++ b/templates/config-repository/examples/multi-host/fleet.json @@ -24,7 +24,7 @@ "state": "active", "scale_set_name": "sample-ci-primary", "lifecycle": "stable", - "engine_ref": "a589eed48fb9d6664d5535dfbf5a9ecf5a147f67", + "engine_ref": "89d46b8ab9cc997577867d84ceee457aa50e1ac9", "min_runners": 0, "max_runners": 4, "runner_resources": { @@ -38,7 +38,7 @@ "state": "active", "scale_set_name": "sample-ci-remote", "lifecycle": "stable", - "engine_ref": "a589eed48fb9d6664d5535dfbf5a9ecf5a147f67", + "engine_ref": "89d46b8ab9cc997577867d84ceee457aa50e1ac9", "min_runners": 0, "max_runners": 2, "runner_resources": { diff --git a/templates/config-repository/fleet.json b/templates/config-repository/fleet.json index 9a10b994..07c098a9 100644 --- a/templates/config-repository/fleet.json +++ b/templates/config-repository/fleet.json @@ -24,7 +24,7 @@ "state": "active", "scale_set_name": "example-ci-01", "lifecycle": "experimental", - "engine_ref": "a589eed48fb9d6664d5535dfbf5a9ecf5a147f67", + "engine_ref": "89d46b8ab9cc997577867d84ceee457aa50e1ac9", "min_runners": 0, "max_runners": 1, "runner_resources": { From 307c455a41f445dc9b476ff9058df90af88878c9 Mon Sep 17 00:00:00 2001 From: Nickfosts Hermes Date: Tue, 21 Jul 2026 12:14:03 -0500 Subject: [PATCH 39/57] fix(installer): close runtime trust drift --- .github/workflows/validate.yml | 2 + scripts/check-installed-state.sh | 1 + scripts/install-worker-controller.sh | 14 +++- scripts/test-install-worker-controller.sh | 34 +++++++- .../.github/workflows/validate.yml | 5 +- .../scripts/scan_committed_secrets.py | 78 +++++++++++++++++++ .../config-repository/scripts/test_policy.py | 16 +++- .../config-repository/scripts/validate.py | 2 +- 8 files changed, 142 insertions(+), 10 deletions(-) create mode 100755 templates/config-repository/scripts/scan_committed_secrets.py diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 48fed783..70ab604e 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -35,7 +35,9 @@ jobs: - name: Validate public configuration template run: | + cmp -s scripts/scan_committed_secrets.py templates/config-repository/scripts/scan_committed_secrets.py python3 templates/config-repository/scripts/test_policy.py + python3 templates/config-repository/scripts/scan_committed_secrets.py templates/config-repository/scripts/validate.sh templates/config-repository/scripts/validate.sh --config templates/config-repository/examples/multi-host/fleet.json temporary_directory="$(mktemp -d)" diff --git a/scripts/check-installed-state.sh b/scripts/check-installed-state.sh index e75ee594..f7ff2070 100755 --- a/scripts/check-installed-state.sh +++ b/scripts/check-installed-state.sh @@ -5,6 +5,7 @@ state_file=${CI_FLEET_INSTALL_STATE_FILE:-/var/lib/ci-fleet/install-state.json} installer=${CI_FLEET_INSTALLER:-/opt/ci-fleet/manager/current/scripts/install-worker-controller.sh} [[ -f "$state_file" ]] || { echo "ERROR: installed desired-state record is missing: $state_file" >&2; exit 2; } +[[ $(stat -c %u "$state_file") == 0 && $(stat -c %a "$state_file") == 600 ]] || { echo "ERROR: install state must be owned by root with mode 0600: $state_file" >&2; exit 2; } command -v python3 >/dev/null || { echo 'ERROR: python3 is required' >&2; exit 2; } [[ -x "$installer" ]] || { echo "ERROR: installer is unavailable: $installer" >&2; exit 2; } diff --git a/scripts/install-worker-controller.sh b/scripts/install-worker-controller.sh index 53af2dab..fbbf4454 100755 --- a/scripts/install-worker-controller.sh +++ b/scripts/install-worker-controller.sh @@ -275,11 +275,19 @@ managed_runner_count() { --filter label=io.randomdevelopment.ci-fleet.kind=runner | wc -l | tr -d ' ' } +managed_runner_total_count() { + docker ps --all -q \ + --filter label=io.randomdevelopment.ci-fleet.managed=true \ + --filter label=io.randomdevelopment.ci-fleet.kind=runner | wc -l | tr -d ' ' +} + runtime_matches() { - local expected=$1 status + local expected=$1 provenance status status=$(controller_status) if [[ "$expected" == active ]]; then - [[ "$status" == running ]] + [[ "$status" == running ]] || return 1 + provenance=$(docker inspect --format '{{ index .Config.Labels "org.opencontainers.image.revision" }}' "$controller_container" 2>/dev/null) || return 1 + [[ "$provenance" == "$engine_ref" ]] else [[ -z "$status" || "$status" == exited || "$status" == created ]] fi @@ -356,7 +364,7 @@ drift_count() { release_matches || { note 'DRIFT engine_release'; count=$((count + 1)); } state_matches || { note 'DRIFT install_state'; count=$((count + 1)); } runtime_matches "$target_state" || { note 'DRIFT controller_runtime'; count=$((count + 1)); } - if [[ "$target_state" != active && $(managed_runner_count) != 0 ]]; then + if [[ "$target_state" != active && $(managed_runner_total_count) != 0 ]]; then note 'DRIFT managed_runners' count=$((count + 1)) fi diff --git a/scripts/test-install-worker-controller.sh b/scripts/test-install-worker-controller.sh index 5fdf399c..c2e58a96 100755 --- a/scripts/test-install-worker-controller.sh +++ b/scripts/test-install-worker-controller.sh @@ -21,7 +21,9 @@ case "${1:-}" in info) exit 0 ;; inspect) [[ -f "$state" ]] || exit 1 - if [[ "$*" == *'.State.Status'* ]]; then + if [[ "$*" == *'org.opencontainers.image.revision'* ]]; then + if [[ -n "${FAKE_CONTROLLER_PROVENANCE_FILE:-}" && -f "$FAKE_CONTROLLER_PROVENANCE_FILE" ]]; then cat "$FAKE_CONTROLLER_PROVENANCE_FILE"; else printf '%s\n' "${FAKE_ENGINE_REF:?}"; fi + elif [[ "$*" == *'.State.Status'* ]]; then if [[ -n "$status_file" && -f "$status_file" ]]; then cat "$status_file"; else printf '%s\n' "${FAKE_CONTROLLER_STATUS:-running}"; fi elif [[ "$*" == *'.State.Paused'* ]]; then if [[ -n "$paused_state" && -f "$paused_state" ]]; then printf 'true\n'; else printf 'false\n'; fi @@ -30,7 +32,9 @@ case "${1:-}" in fi ;; ps) - if [[ -n "${FAKE_RUNNER_STATE_ONCE:-}" && -f "$FAKE_RUNNER_STATE_ONCE" ]]; then + if [[ "$*" == *'--all'* && -n "${FAKE_ALL_RUNNER_STATE:-}" && -f "$FAKE_ALL_RUNNER_STATE" ]]; then + printf 'managed-runner-all-state\n' + elif [[ -n "${FAKE_RUNNER_STATE_ONCE:-}" && -f "$FAKE_RUNNER_STATE_ONCE" ]]; then rm -f "$FAKE_RUNNER_STATE_ONCE" printf 'managed-runner\n' elif [[ -n "${FAKE_RUNNER_STATE:-}" && -f "$FAKE_RUNNER_STATE" ]]; then @@ -71,6 +75,7 @@ case "${1:-}" in exit 42 fi : >"$state" + [[ -z "${FAKE_CONTROLLER_PROVENANCE_FILE:-}" ]] || printf '%s\n' "${FAKE_ENGINE_REF:?}" >"$FAKE_CONTROLLER_PROVENANCE_FILE" if [[ -n "${FAKE_RESTART_AFTER_UP:-}" && -f "$FAKE_RESTART_AFTER_UP" ]]; then rm -f "$FAKE_RESTART_AFTER_UP" printf 'restarting\n' >"$status_file" @@ -78,7 +83,7 @@ case "${1:-}" in rm -f "$status_file" fi ;; - stop|down|rm) rm -f "$state"; [[ -z "$status_file" ]] || rm -f "$status_file"; [[ -z "$paused_state" ]] || rm -f "$paused_state" ;; + stop|down|rm) rm -f "$state"; [[ -z "$status_file" ]] || rm -f "$status_file"; [[ -z "$paused_state" ]] || rm -f "$paused_state"; [[ -z "${FAKE_CONTROLLER_PROVENANCE_FILE:-}" ]] || rm -f "$FAKE_CONTROLLER_PROVENANCE_FILE" ;; pause) [[ -z "$paused_state" ]] || : >"$paused_state" [[ -z "${FAKE_RUNNER_STATE:-}" ]] || rm -f "$FAKE_RUNNER_STATE" @@ -120,6 +125,10 @@ if [[ -n "${FAKE_WRONG_HOST_CONFIG_OWNER:-}" && "${1:-}" == -c && "${2:-}" == %u printf '99999\n' exit 0 fi +if [[ -n "${FAKE_WRONG_INSTALL_STATE_OWNER:-}" && "${1:-}" == -c && "${2:-}" == %u && "${3:-}" == "$FAKE_WRONG_INSTALL_STATE_OWNER" ]]; then + printf '99999\n' + exit 0 +fi exec "$REAL_STAT" "$@" EOF chmod 700 "$fake_bin/docker" "$fake_bin/systemctl" "$fake_bin/stat" @@ -138,6 +147,7 @@ export PATH="$fake_bin:$PATH" export FAKE_DOCKER_STATE=$tmp/docker-controller-running export FAKE_CONTROLLER_STATUS_FILE=$tmp/docker-controller-status export FAKE_PAUSED_STATE=$tmp/docker-controller-paused +export FAKE_CONTROLLER_PROVENANCE_FILE=$tmp/docker-controller-provenance export CI_FLEET_TESTING=1 export CI_FLEET_DOCKER_GID_OVERRIDE=998 export CI_FLEET_STARTUP_WAIT_SECONDS=0 @@ -249,7 +259,14 @@ grep -Fq 'CONVERGED mode=install' <<<"$first" || fail 'fresh install did not con [[ -L "$root/opt/ci-fleet/current" && -f "$root/var/lib/ci-fleet/install-state.json" ]] || fail 'fresh install state is incomplete' [[ $(readlink -f "$root/opt/ci-fleet/manager/current") == "$root/opt/ci-fleet/manager/releases/$engine_ref" ]] || fail 'installer manager did not activate the desired engine release' [[ -f "$FAKE_DOCKER_STATE" ]] || fail 'active controller was not started' -expect_success env CI_FLEET_INSTALL_STATE_FILE="$root/var/lib/ci-fleet/install-state.json" CI_FLEET_INSTALLER="$installer" "$repo_root/scripts/check-installed-state.sh" >/dev/null +install_state=$root/var/lib/ci-fleet/install-state.json +chmod 644 "$install_state" +expect_failure 'install state must be owned by root with mode 0600' env CI_FLEET_INSTALL_STATE_FILE="$install_state" CI_FLEET_INSTALLER="$installer" "$repo_root/scripts/check-installed-state.sh" +chmod 600 "$install_state" +export FAKE_WRONG_INSTALL_STATE_OWNER=$install_state +expect_failure 'install state must be owned by root with mode 0600' env CI_FLEET_INSTALL_STATE_FILE="$install_state" CI_FLEET_INSTALLER="$installer" "$repo_root/scripts/check-installed-state.sh" +unset FAKE_WRONG_INSTALL_STATE_OWNER +expect_success env CI_FLEET_INSTALL_STATE_FILE="$install_state" CI_FLEET_INSTALLER="$installer" "$repo_root/scripts/check-installed-state.sh" >/dev/null export FAKE_DISABLED_TIMER=ci-fleet-cleanup.timer expect_failure 'DRIFT maintenance_timers' "$installer" --check "${base_args[@]}" --ref "$ref_one" @@ -266,6 +283,10 @@ mv "$host_config.missing" "$host_config" second=$(expect_success "$installer" --install "${base_args[@]}" --ref "$ref_one") grep -Fq 'NO_CHANGE' <<<"$second" || fail 'idempotent rerun changed the host' expect_success "$installer" --check "${base_args[@]}" --ref "$ref_one" >/dev/null +printf '%040d\n' 0 >"$FAKE_CONTROLLER_PROVENANCE_FILE" +expect_failure 'DRIFT controller_runtime' "$installer" --check "${base_args[@]}" --ref "$ref_one" +expect_success "$installer" --install "${base_args[@]}" --ref "$ref_one" >/dev/null +[[ $(<"$FAKE_CONTROLLER_PROVENANCE_FILE") == "$engine_ref" ]] || fail 'controller convergence did not restore running image provenance' rm -f "$FAKE_RUNNER_IMAGE_STATE" expect_failure 'DRIFT managed_images' "$installer" --check "${base_args[@]}" --ref "$ref_one" @@ -384,6 +405,11 @@ export FAKE_RUNNER_STATE=$tmp/drained-managed-runner expect_failure 'DRIFT managed_runners' "$installer" --check "${base_args[@]}" --ref "$ref_three" rm -f "$FAKE_RUNNER_STATE" unset FAKE_RUNNER_STATE +export FAKE_ALL_RUNNER_STATE=$tmp/drained-exited-managed-runner +: >"$FAKE_ALL_RUNNER_STATE" +expect_failure 'DRIFT managed_runners' "$installer" --check "${base_args[@]}" --ref "$ref_three" +rm -f "$FAKE_ALL_RUNNER_STATE" +unset FAKE_ALL_RUNNER_STATE ref_four=$(write_config disabled 2 2) expect_success "$installer" --upgrade "${base_args[@]}" --ref "$ref_four" >/dev/null diff --git a/templates/config-repository/.github/workflows/validate.yml b/templates/config-repository/.github/workflows/validate.yml index ad221494..27e1fd1d 100644 --- a/templates/config-repository/.github/workflows/validate.yml +++ b/templates/config-repository/.github/workflows/validate.yml @@ -26,11 +26,14 @@ jobs: - name: Validate shell and Python syntax run: | bash -n scripts/init.sh scripts/validate.sh - python3 -m py_compile scripts/init.py scripts/validate.py scripts/test_policy.py + python3 -m py_compile scripts/init.py scripts/validate.py scripts/test_policy.py scripts/scan_committed_secrets.py - name: Prove forbidden configurations fail closed run: python3 scripts/test_policy.py + - name: Scan committed file contents for secrets + run: python3 scripts/scan_committed_secrets.py + - name: Validate reference configurations run: | ./scripts/validate.sh diff --git a/templates/config-repository/scripts/scan_committed_secrets.py b/templates/config-repository/scripts/scan_committed_secrets.py new file mode 100755 index 00000000..461fa0c7 --- /dev/null +++ b/templates/config-repository/scripts/scan_committed_secrets.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 +"""Fail when tracked files contain high-confidence credential material.""" + +from __future__ import annotations + +import argparse +import re +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +EXCLUDED = { + "SECURITY.md", + "docs/SECRETS.md", + "scripts/scan_committed_secrets.py", + "scripts/validate.sh", +} +PATTERN = re.compile( + rb"BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY|" + rb"github_pat_[A-Za-z0-9_]{20,}|" + rb"gh[opusr]_[A-Za-z0-9]{20,}|" + rb"AKIA[0-9A-Z]{16}|" + rb"(?:postgres|mysql|mongodb(?:\+srv)?|redis)://[^\s/:]+:[^\s/@]+@" +) +assert all(PATTERN.search(prefix + b"x" * 20) for prefix in (b"gho_", b"ghp_", b"ghr_", b"ghs_", b"ghu_")) +assert PATTERN.search(b"AKIA" + b"A" * 16) +assert PATTERN.search(b"mysql://fixture-user:fixture-password@example.invalid/database") + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--repository", type=Path, default=ROOT) + parser.add_argument("--commit", help="immutable commit whose blobs should be scanned") + return parser.parse_args() + + +def main() -> int: + args = parse_args() + repository = args.repository.resolve() + if args.commit is not None and not re.fullmatch(r"[0-9a-f]{40}", args.commit): + print("--commit must be a full lowercase commit SHA", file=sys.stderr) + return 2 + listing = ["git", "-C", str(repository)] + listing += ["ls-tree", "-rz", "--name-only", args.commit] if args.commit else ["ls-files", "-z"] + tracked = subprocess.run( + listing, + check=True, + stdout=subprocess.PIPE, + ).stdout.split(b"\0") + findings: list[str] = [] + for raw in tracked: + if not raw: + continue + relative = raw.decode("utf-8") + if args.commit is None and relative in EXCLUDED: + continue + if args.commit is None: + data = (repository / relative).read_bytes() + else: + data = subprocess.run( + ["git", "-C", str(repository), "cat-file", "blob", f"{args.commit}:{relative}"], + check=True, + stdout=subprocess.PIPE, + ).stdout + for match in PATTERN.finditer(data): + line = data.count(b"\n", 0, match.start()) + 1 + findings.append(f"{relative}:{line}") + if findings: + print("possible committed secret detected:", file=sys.stderr) + print("\n".join(findings), file=sys.stderr) + return 1 + print("OK: no high-confidence secret material in tracked files") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/templates/config-repository/scripts/test_policy.py b/templates/config-repository/scripts/test_policy.py index f769c0c4..5277b917 100755 --- a/templates/config-repository/scripts/test_policy.py +++ b/templates/config-repository/scripts/test_policy.py @@ -10,7 +10,7 @@ import unittest from pathlib import Path -from validate import Validation, load_json, scan_secret_material, validate_config +from validate import Validation, load_json, scan_secret_material, scan_tree_path_list, validate_config ROOT = Path(__file__).resolve().parents[1] @@ -208,6 +208,20 @@ def test_secret_value_key_is_rejected(self) -> None: config["environments"]["development"]["token"] = "not-a-real-token" self.assert_rejected(config, "secret values are forbidden") + def test_host_local_environment_paths_are_rejected(self) -> None: + validation = Validation() + with tempfile.TemporaryDirectory() as directory: + path_list = Path(directory) / "paths" + path_list.write_bytes(b"host.env\0ci-fleet.env\0nested/host.env\0nested/ci-fleet.env\0") + scan_tree_path_list(path_list, validation) + self.assertEqual(len(validation.errors), 4, validation.errors) + self.assertTrue(all("secret-bearing files are forbidden" in error for error in validation.errors), validation.errors) + + def test_template_ci_scans_committed_file_contents(self) -> None: + self.assertTrue((ROOT / "scripts" / "scan_committed_secrets.py").is_file()) + workflow = (ROOT / ".github" / "workflows" / "validate.yml").read_text(encoding="utf-8") + self.assertIn("python3 scripts/scan_committed_secrets.py", workflow) + def test_duplicate_json_controller_id_is_rejected(self) -> None: validation = Validation() with tempfile.TemporaryDirectory() as directory: diff --git a/templates/config-repository/scripts/validate.py b/templates/config-repository/scripts/validate.py index e7db9918..5b70b92f 100755 --- a/templates/config-repository/scripts/validate.py +++ b/templates/config-repository/scripts/validate.py @@ -53,7 +53,7 @@ "vm_id", "vmid", } -FORBIDDEN_FILENAMES = re.compile(r"(?:^|/)\.env(?:\..+)?$|\.(?:key|pem|p12|pfx)$", re.IGNORECASE) +FORBIDDEN_FILENAMES = re.compile(r"(?:^|/)(?:\.env(?:\..+)?|host\.env|ci-fleet\.env)$|\.(?:key|pem|p12|pfx)$", re.IGNORECASE) FORBIDDEN_DIRECTORIES = {"credentials", "private", "secrets"} From eb8cbbce8b3d90fe190c1fc4701293576825059a Mon Sep 17 00:00:00 2001 From: Nickfosts Hermes Date: Tue, 21 Jul 2026 12:14:45 -0500 Subject: [PATCH 40/57] docs: pin final trusted runtime engine --- templates/config-repository/examples/multi-host/fleet.json | 4 ++-- templates/config-repository/fleet.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/templates/config-repository/examples/multi-host/fleet.json b/templates/config-repository/examples/multi-host/fleet.json index 01f52791..c66f85f1 100644 --- a/templates/config-repository/examples/multi-host/fleet.json +++ b/templates/config-repository/examples/multi-host/fleet.json @@ -24,7 +24,7 @@ "state": "active", "scale_set_name": "sample-ci-primary", "lifecycle": "stable", - "engine_ref": "89d46b8ab9cc997577867d84ceee457aa50e1ac9", + "engine_ref": "307c455a41f445dc9b476ff9058df90af88878c9", "min_runners": 0, "max_runners": 4, "runner_resources": { @@ -38,7 +38,7 @@ "state": "active", "scale_set_name": "sample-ci-remote", "lifecycle": "stable", - "engine_ref": "89d46b8ab9cc997577867d84ceee457aa50e1ac9", + "engine_ref": "307c455a41f445dc9b476ff9058df90af88878c9", "min_runners": 0, "max_runners": 2, "runner_resources": { diff --git a/templates/config-repository/fleet.json b/templates/config-repository/fleet.json index 07c098a9..6e53f804 100644 --- a/templates/config-repository/fleet.json +++ b/templates/config-repository/fleet.json @@ -24,7 +24,7 @@ "state": "active", "scale_set_name": "example-ci-01", "lifecycle": "experimental", - "engine_ref": "89d46b8ab9cc997577867d84ceee457aa50e1ac9", + "engine_ref": "307c455a41f445dc9b476ff9058df90af88878c9", "min_runners": 0, "max_runners": 1, "runner_resources": { From fe0ce060ec38a8620956a6d965f423e20b77a6c0 Mon Sep 17 00:00:00 2001 From: Nickfosts Hermes Date: Tue, 21 Jul 2026 12:18:54 -0500 Subject: [PATCH 41/57] fix(secrets): exempt scanner fixtures by path --- scripts/scan_committed_secrets.py | 7 ++++++- .../config-repository/scripts/scan_committed_secrets.py | 7 ++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/scripts/scan_committed_secrets.py b/scripts/scan_committed_secrets.py index 461fa0c7..9a2cb7b1 100644 --- a/scripts/scan_committed_secrets.py +++ b/scripts/scan_committed_secrets.py @@ -15,6 +15,11 @@ "docs/SECRETS.md", "scripts/scan_committed_secrets.py", "scripts/validate.sh", + "templates/config-repository/scripts/scan_committed_secrets.py", +} +SCANNER_PATHS = { + "scripts/scan_committed_secrets.py", + "templates/config-repository/scripts/scan_committed_secrets.py", } PATTERN = re.compile( rb"BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY|" @@ -53,7 +58,7 @@ def main() -> int: if not raw: continue relative = raw.decode("utf-8") - if args.commit is None and relative in EXCLUDED: + if relative in SCANNER_PATHS or (args.commit is None and relative in EXCLUDED): continue if args.commit is None: data = (repository / relative).read_bytes() diff --git a/templates/config-repository/scripts/scan_committed_secrets.py b/templates/config-repository/scripts/scan_committed_secrets.py index 461fa0c7..9a2cb7b1 100755 --- a/templates/config-repository/scripts/scan_committed_secrets.py +++ b/templates/config-repository/scripts/scan_committed_secrets.py @@ -15,6 +15,11 @@ "docs/SECRETS.md", "scripts/scan_committed_secrets.py", "scripts/validate.sh", + "templates/config-repository/scripts/scan_committed_secrets.py", +} +SCANNER_PATHS = { + "scripts/scan_committed_secrets.py", + "templates/config-repository/scripts/scan_committed_secrets.py", } PATTERN = re.compile( rb"BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY|" @@ -53,7 +58,7 @@ def main() -> int: if not raw: continue relative = raw.decode("utf-8") - if args.commit is None and relative in EXCLUDED: + if relative in SCANNER_PATHS or (args.commit is None and relative in EXCLUDED): continue if args.commit is None: data = (repository / relative).read_bytes() From 66f8727000fee4f79aaa34f41bcfe1f148dcf8f0 Mon Sep 17 00:00:00 2001 From: Nickfosts Hermes Date: Tue, 21 Jul 2026 12:19:40 -0500 Subject: [PATCH 42/57] docs: pin final secret-safe engine --- templates/config-repository/examples/multi-host/fleet.json | 4 ++-- templates/config-repository/fleet.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/templates/config-repository/examples/multi-host/fleet.json b/templates/config-repository/examples/multi-host/fleet.json index c66f85f1..0e7f2674 100644 --- a/templates/config-repository/examples/multi-host/fleet.json +++ b/templates/config-repository/examples/multi-host/fleet.json @@ -24,7 +24,7 @@ "state": "active", "scale_set_name": "sample-ci-primary", "lifecycle": "stable", - "engine_ref": "307c455a41f445dc9b476ff9058df90af88878c9", + "engine_ref": "fe0ce060ec38a8620956a6d965f423e20b77a6c0", "min_runners": 0, "max_runners": 4, "runner_resources": { @@ -38,7 +38,7 @@ "state": "active", "scale_set_name": "sample-ci-remote", "lifecycle": "stable", - "engine_ref": "307c455a41f445dc9b476ff9058df90af88878c9", + "engine_ref": "fe0ce060ec38a8620956a6d965f423e20b77a6c0", "min_runners": 0, "max_runners": 2, "runner_resources": { diff --git a/templates/config-repository/fleet.json b/templates/config-repository/fleet.json index 6e53f804..778c9024 100644 --- a/templates/config-repository/fleet.json +++ b/templates/config-repository/fleet.json @@ -24,7 +24,7 @@ "state": "active", "scale_set_name": "example-ci-01", "lifecycle": "experimental", - "engine_ref": "307c455a41f445dc9b476ff9058df90af88878c9", + "engine_ref": "fe0ce060ec38a8620956a6d965f423e20b77a6c0", "min_runners": 0, "max_runners": 1, "runner_resources": { From 6d76f590b26649029d8e50b7a30d86d7aeba31b1 Mon Sep 17 00:00:00 2001 From: Nickfosts Hermes Date: Tue, 21 Jul 2026 12:26:24 -0500 Subject: [PATCH 43/57] test(installer): honor isolated test ownership --- scripts/check-installed-state.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/check-installed-state.sh b/scripts/check-installed-state.sh index f7ff2070..99407d99 100755 --- a/scripts/check-installed-state.sh +++ b/scripts/check-installed-state.sh @@ -5,7 +5,9 @@ state_file=${CI_FLEET_INSTALL_STATE_FILE:-/var/lib/ci-fleet/install-state.json} installer=${CI_FLEET_INSTALLER:-/opt/ci-fleet/manager/current/scripts/install-worker-controller.sh} [[ -f "$state_file" ]] || { echo "ERROR: installed desired-state record is missing: $state_file" >&2; exit 2; } -[[ $(stat -c %u "$state_file") == 0 && $(stat -c %a "$state_file") == 600 ]] || { echo "ERROR: install state must be owned by root with mode 0600: $state_file" >&2; exit 2; } +expected_owner=0 +[[ ${CI_FLEET_TESTING:-0} != 1 ]] || expected_owner=$(id -u) +[[ $(stat -c %u "$state_file") == "$expected_owner" && $(stat -c %a "$state_file") == 600 ]] || { echo "ERROR: install state must be owned by root with mode 0600: $state_file" >&2; exit 2; } command -v python3 >/dev/null || { echo 'ERROR: python3 is required' >&2; exit 2; } [[ -x "$installer" ]] || { echo "ERROR: installer is unavailable: $installer" >&2; exit 2; } From c71942afc71d5576a7310422ae39fb89127ad5bd Mon Sep 17 00:00:00 2001 From: Nickfosts Hermes Date: Tue, 21 Jul 2026 12:26:58 -0500 Subject: [PATCH 44/57] docs: pin final ownership-safe engine --- templates/config-repository/examples/multi-host/fleet.json | 4 ++-- templates/config-repository/fleet.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/templates/config-repository/examples/multi-host/fleet.json b/templates/config-repository/examples/multi-host/fleet.json index 0e7f2674..cb58ceaf 100644 --- a/templates/config-repository/examples/multi-host/fleet.json +++ b/templates/config-repository/examples/multi-host/fleet.json @@ -24,7 +24,7 @@ "state": "active", "scale_set_name": "sample-ci-primary", "lifecycle": "stable", - "engine_ref": "fe0ce060ec38a8620956a6d965f423e20b77a6c0", + "engine_ref": "6d76f590b26649029d8e50b7a30d86d7aeba31b1", "min_runners": 0, "max_runners": 4, "runner_resources": { @@ -38,7 +38,7 @@ "state": "active", "scale_set_name": "sample-ci-remote", "lifecycle": "stable", - "engine_ref": "fe0ce060ec38a8620956a6d965f423e20b77a6c0", + "engine_ref": "6d76f590b26649029d8e50b7a30d86d7aeba31b1", "min_runners": 0, "max_runners": 2, "runner_resources": { diff --git a/templates/config-repository/fleet.json b/templates/config-repository/fleet.json index 778c9024..03d1e5f8 100644 --- a/templates/config-repository/fleet.json +++ b/templates/config-repository/fleet.json @@ -24,7 +24,7 @@ "state": "active", "scale_set_name": "example-ci-01", "lifecycle": "experimental", - "engine_ref": "fe0ce060ec38a8620956a6d965f423e20b77a6c0", + "engine_ref": "6d76f590b26649029d8e50b7a30d86d7aeba31b1", "min_runners": 0, "max_runners": 1, "runner_resources": { From a530c6abdc0245d818dd5fefe72a45189b5e6ab4 Mon Sep 17 00:00:00 2001 From: Nickfosts Hermes Date: Tue, 21 Jul 2026 13:03:19 -0500 Subject: [PATCH 45/57] fix(installer): reconcile live runtime trust --- scripts/install-worker-controller.sh | 30 ++++++++++++++++++- scripts/scan_committed_secrets.py | 15 +--------- scripts/test-install-worker-controller.sh | 29 ++++++++++++++---- .../scripts/scan_committed_secrets.py | 15 +--------- .../config-repository/scripts/test_policy.py | 22 +++++++++++++- 5 files changed, 76 insertions(+), 35 deletions(-) diff --git a/scripts/install-worker-controller.sh b/scripts/install-worker-controller.sh index fbbf4454..6f47e25f 100755 --- a/scripts/install-worker-controller.sh +++ b/scripts/install-worker-controller.sh @@ -281,20 +281,47 @@ managed_runner_total_count() { --filter label=io.randomdevelopment.ci-fleet.kind=runner | wc -l | tr -d ' ' } +remove_inactive_managed_runners() { + local -a containers=() + mapfile -t containers < <(docker ps --all -q \ + --filter label=io.randomdevelopment.ci-fleet.managed=true \ + --filter label=io.randomdevelopment.ci-fleet.kind=runner) + ((${#containers[@]} == 0)) || docker rm "${containers[@]}" >/dev/null +} + +controller_environment_matches() { + local actual expected key live + live=$(docker inspect --format '{{range .Config.Env}}{{println .}}{{end}}' "$controller_container" 2>/dev/null) || return 1 + for key in \ + CI_FLEET_GITHUB_URL CI_FLEET_SCALE_SET_NAME CI_FLEET_LABELS CI_FLEET_RUNNER_GROUP \ + CI_FLEET_RUNNER_IMAGE CI_FLEET_INSTANCE CI_FLEET_GITHUB_APP_CLIENT_ID \ + CI_FLEET_GITHUB_APP_INSTALLATION_ID CI_FLEET_MIN_RUNNERS CI_FLEET_MAX_RUNNERS \ + CI_FLEET_RUNNER_CPUS CI_FLEET_RUNNER_MEMORY_MIB CI_FLEET_RUNNER_TTL CI_FLEET_DOCKER_GID; do + expected=$(awk -F= -v key="$key" '$1 == key {print substr($0, index($0, "=") + 1)}' "$candidate_env") + [[ -n "$expected" ]] || return 1 + actual=$(awk -F= -v key="$key" '$1 == key {count++; value=substr($0, index($0, "=") + 1)} END {if (count != 1) exit 1; print value}' <<<"$live") || return 1 + [[ "$actual" == "$expected" ]] || return 1 + done +} + runtime_matches() { local expected=$1 provenance status status=$(controller_status) if [[ "$expected" == active ]]; then [[ "$status" == running ]] || return 1 provenance=$(docker inspect --format '{{ index .Config.Labels "org.opencontainers.image.revision" }}' "$controller_container" 2>/dev/null) || return 1 - [[ "$provenance" == "$engine_ref" ]] + [[ "$provenance" == "$engine_ref" ]] || return 1 + controller_environment_matches else [[ -z "$status" || "$status" == exited || "$status" == created ]] fi } state_matches() { + local expected_owner=0 [[ -f "$state_file" ]] || return 1 + [[ "$testing" != 1 ]] || expected_owner=$(id -u) + [[ $(stat -c %u "$state_file") == "$expected_owner" && $(stat -c %a "$state_file") == 600 ]] || return 1 python3 - "$state_file" "$candidate_metadata" <<'PY' import json import sys @@ -757,6 +784,7 @@ perform_converge() { make_checkpoint transaction_active=true drain_current + [[ "$target_state" == active ]] || remove_inactive_managed_runners build_candidate activate_candidate transaction_active=false diff --git a/scripts/scan_committed_secrets.py b/scripts/scan_committed_secrets.py index 9a2cb7b1..fa90daf5 100644 --- a/scripts/scan_committed_secrets.py +++ b/scripts/scan_committed_secrets.py @@ -10,17 +10,6 @@ from pathlib import Path ROOT = Path(__file__).resolve().parents[1] -EXCLUDED = { - "SECURITY.md", - "docs/SECRETS.md", - "scripts/scan_committed_secrets.py", - "scripts/validate.sh", - "templates/config-repository/scripts/scan_committed_secrets.py", -} -SCANNER_PATHS = { - "scripts/scan_committed_secrets.py", - "templates/config-repository/scripts/scan_committed_secrets.py", -} PATTERN = re.compile( rb"BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY|" rb"github_pat_[A-Za-z0-9_]{20,}|" @@ -30,7 +19,7 @@ ) assert all(PATTERN.search(prefix + b"x" * 20) for prefix in (b"gho_", b"ghp_", b"ghr_", b"ghs_", b"ghu_")) assert PATTERN.search(b"AKIA" + b"A" * 16) -assert PATTERN.search(b"mysql://fixture-user:fixture-password@example.invalid/database") +assert PATTERN.search(b"mysql" + b"://fixture-user:***@example.invalid/database") def parse_args() -> argparse.Namespace: @@ -58,8 +47,6 @@ def main() -> int: if not raw: continue relative = raw.decode("utf-8") - if relative in SCANNER_PATHS or (args.commit is None and relative in EXCLUDED): - continue if args.commit is None: data = (repository / relative).read_bytes() else: diff --git a/scripts/test-install-worker-controller.sh b/scripts/test-install-worker-controller.sh index c2e58a96..fc2f9b6f 100755 --- a/scripts/test-install-worker-controller.sh +++ b/scripts/test-install-worker-controller.sh @@ -21,7 +21,10 @@ case "${1:-}" in info) exit 0 ;; inspect) [[ -f "$state" ]] || exit 1 - if [[ "$*" == *'org.opencontainers.image.revision'* ]]; then + if [[ "$*" == *'.Config.Env'* ]]; then + [[ -n "${FAKE_CONTROLLER_ENV_FILE:-}" && -f "$FAKE_CONTROLLER_ENV_FILE" ]] || exit 1 + cat "$FAKE_CONTROLLER_ENV_FILE" + elif [[ "$*" == *'org.opencontainers.image.revision'* ]]; then if [[ -n "${FAKE_CONTROLLER_PROVENANCE_FILE:-}" && -f "$FAKE_CONTROLLER_PROVENANCE_FILE" ]]; then cat "$FAKE_CONTROLLER_PROVENANCE_FILE"; else printf '%s\n' "${FAKE_ENGINE_REF:?}"; fi elif [[ "$*" == *'.State.Status'* ]]; then if [[ -n "$status_file" && -f "$status_file" ]]; then cat "$status_file"; else printf '%s\n' "${FAKE_CONTROLLER_STATUS:-running}"; fi @@ -56,6 +59,10 @@ case "${1:-}" in [[ -f "$image_state" ]] || exit 1 cat "$image_state" ;; + rm) + (($# >= 2)) || exit 1 + [[ -z "${FAKE_ALL_RUNNER_STATE:-}" ]] || rm -f "$FAKE_ALL_RUNNER_STATE" + ;; volume|network) [[ "${2:-}" == ls ]] && exit 0 [[ "${2:-}" == inspect ]] && exit 1 @@ -64,9 +71,11 @@ case "${1:-}" in compose) if [[ "${2:-}" == version ]]; then exit 0; fi [[ -z "${COMPOSE_PROJECT_NAME:-}" && -z "${CI_FLEET_MAX_RUNNERS:-}" ]] || exit 44 - command= + command= env_file= previous= for argument in "$@"; do + [[ "$previous" != --env-file ]] || env_file=$argument case "$argument" in config|build|up|stop|pause|unpause|kill|down|logs|rm) command=$argument ;; esac + previous=$argument done case "$command" in up) @@ -76,6 +85,7 @@ case "${1:-}" in fi : >"$state" [[ -z "${FAKE_CONTROLLER_PROVENANCE_FILE:-}" ]] || printf '%s\n' "${FAKE_ENGINE_REF:?}" >"$FAKE_CONTROLLER_PROVENANCE_FILE" + [[ -z "${FAKE_CONTROLLER_ENV_FILE:-}" ]] || cp "$env_file" "$FAKE_CONTROLLER_ENV_FILE" if [[ -n "${FAKE_RESTART_AFTER_UP:-}" && -f "$FAKE_RESTART_AFTER_UP" ]]; then rm -f "$FAKE_RESTART_AFTER_UP" printf 'restarting\n' >"$status_file" @@ -83,7 +93,7 @@ case "${1:-}" in rm -f "$status_file" fi ;; - stop|down|rm) rm -f "$state"; [[ -z "$status_file" ]] || rm -f "$status_file"; [[ -z "$paused_state" ]] || rm -f "$paused_state"; [[ -z "${FAKE_CONTROLLER_PROVENANCE_FILE:-}" ]] || rm -f "$FAKE_CONTROLLER_PROVENANCE_FILE" ;; + stop|down|rm) rm -f "$state"; [[ -z "$status_file" ]] || rm -f "$status_file"; [[ -z "$paused_state" ]] || rm -f "$paused_state"; [[ -z "${FAKE_CONTROLLER_PROVENANCE_FILE:-}" ]] || rm -f "$FAKE_CONTROLLER_PROVENANCE_FILE"; [[ -z "${FAKE_CONTROLLER_ENV_FILE:-}" ]] || rm -f "$FAKE_CONTROLLER_ENV_FILE" ;; pause) [[ -z "$paused_state" ]] || : >"$paused_state" [[ -z "${FAKE_RUNNER_STATE:-}" ]] || rm -f "$FAKE_RUNNER_STATE" @@ -148,6 +158,7 @@ export FAKE_DOCKER_STATE=$tmp/docker-controller-running export FAKE_CONTROLLER_STATUS_FILE=$tmp/docker-controller-status export FAKE_PAUSED_STATE=$tmp/docker-controller-paused export FAKE_CONTROLLER_PROVENANCE_FILE=$tmp/docker-controller-provenance +export FAKE_CONTROLLER_ENV_FILE=$tmp/docker-controller-env export CI_FLEET_TESTING=1 export CI_FLEET_DOCKER_GID_OVERRIDE=998 export CI_FLEET_STARTUP_WAIT_SECONDS=0 @@ -262,7 +273,9 @@ grep -Fq 'CONVERGED mode=install' <<<"$first" || fail 'fresh install did not con install_state=$root/var/lib/ci-fleet/install-state.json chmod 644 "$install_state" expect_failure 'install state must be owned by root with mode 0600' env CI_FLEET_INSTALL_STATE_FILE="$install_state" CI_FLEET_INSTALLER="$installer" "$repo_root/scripts/check-installed-state.sh" -chmod 600 "$install_state" +expect_failure 'DRIFT install_state' "$installer" --check "${base_args[@]}" --ref "$ref_one" +expect_success "$installer" --install "${base_args[@]}" --ref "$ref_one" >/dev/null +[[ $(stat -c %a "$install_state") == 600 ]] || fail 'convergence did not repair install-state mode' export FAKE_WRONG_INSTALL_STATE_OWNER=$install_state expect_failure 'install state must be owned by root with mode 0600' env CI_FLEET_INSTALL_STATE_FILE="$install_state" CI_FLEET_INSTALLER="$installer" "$repo_root/scripts/check-installed-state.sh" unset FAKE_WRONG_INSTALL_STATE_OWNER @@ -287,6 +300,11 @@ printf '%040d\n' 0 >"$FAKE_CONTROLLER_PROVENANCE_FILE" expect_failure 'DRIFT controller_runtime' "$installer" --check "${base_args[@]}" --ref "$ref_one" expect_success "$installer" --install "${base_args[@]}" --ref "$ref_one" >/dev/null [[ $(<"$FAKE_CONTROLLER_PROVENANCE_FILE") == "$engine_ref" ]] || fail 'controller convergence did not restore running image provenance' +python3 -c 'from pathlib import Path; import sys; path = Path(sys.argv[1]); path.write_text(path.read_text().replace("CI_FLEET_MAX_RUNNERS=1", "CI_FLEET_MAX_RUNNERS=9"))' "$FAKE_CONTROLLER_ENV_FILE" +grep -Fxq 'CI_FLEET_MAX_RUNNERS=9' "$FAKE_CONTROLLER_ENV_FILE" || fail 'live-environment fixture did not mutate' +expect_failure 'DRIFT controller_runtime' "$installer" --check "${base_args[@]}" --ref "$ref_one" +expect_success "$installer" --install "${base_args[@]}" --ref "$ref_one" >/dev/null +grep -Fxq 'CI_FLEET_MAX_RUNNERS=1' "$FAKE_CONTROLLER_ENV_FILE" || fail 'controller convergence did not restore live environment' rm -f "$FAKE_RUNNER_IMAGE_STATE" expect_failure 'DRIFT managed_images' "$installer" --check "${base_args[@]}" --ref "$ref_one" @@ -408,7 +426,8 @@ unset FAKE_RUNNER_STATE export FAKE_ALL_RUNNER_STATE=$tmp/drained-exited-managed-runner : >"$FAKE_ALL_RUNNER_STATE" expect_failure 'DRIFT managed_runners' "$installer" --check "${base_args[@]}" --ref "$ref_three" -rm -f "$FAKE_ALL_RUNNER_STATE" +expect_success "$installer" --install "${base_args[@]}" --ref "$ref_three" >/dev/null +[[ ! -f "$FAKE_ALL_RUNNER_STATE" ]] || fail 'non-active convergence did not remove stopped managed runners' unset FAKE_ALL_RUNNER_STATE ref_four=$(write_config disabled 2 2) diff --git a/templates/config-repository/scripts/scan_committed_secrets.py b/templates/config-repository/scripts/scan_committed_secrets.py index 9a2cb7b1..fa90daf5 100755 --- a/templates/config-repository/scripts/scan_committed_secrets.py +++ b/templates/config-repository/scripts/scan_committed_secrets.py @@ -10,17 +10,6 @@ from pathlib import Path ROOT = Path(__file__).resolve().parents[1] -EXCLUDED = { - "SECURITY.md", - "docs/SECRETS.md", - "scripts/scan_committed_secrets.py", - "scripts/validate.sh", - "templates/config-repository/scripts/scan_committed_secrets.py", -} -SCANNER_PATHS = { - "scripts/scan_committed_secrets.py", - "templates/config-repository/scripts/scan_committed_secrets.py", -} PATTERN = re.compile( rb"BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY|" rb"github_pat_[A-Za-z0-9_]{20,}|" @@ -30,7 +19,7 @@ ) assert all(PATTERN.search(prefix + b"x" * 20) for prefix in (b"gho_", b"ghp_", b"ghr_", b"ghs_", b"ghu_")) assert PATTERN.search(b"AKIA" + b"A" * 16) -assert PATTERN.search(b"mysql://fixture-user:fixture-password@example.invalid/database") +assert PATTERN.search(b"mysql" + b"://fixture-user:***@example.invalid/database") def parse_args() -> argparse.Namespace: @@ -58,8 +47,6 @@ def main() -> int: if not raw: continue relative = raw.decode("utf-8") - if relative in SCANNER_PATHS or (args.commit is None and relative in EXCLUDED): - continue if args.commit is None: data = (repository / relative).read_bytes() else: diff --git a/templates/config-repository/scripts/test_policy.py b/templates/config-repository/scripts/test_policy.py index 5277b917..7dc83468 100755 --- a/templates/config-repository/scripts/test_policy.py +++ b/templates/config-repository/scripts/test_policy.py @@ -6,6 +6,8 @@ import copy import json import re +import subprocess +import sys import tempfile import unittest from pathlib import Path @@ -218,9 +220,27 @@ def test_host_local_environment_paths_are_rejected(self) -> None: self.assertTrue(all("secret-bearing files are forbidden" in error for error in validation.errors), validation.errors) def test_template_ci_scans_committed_file_contents(self) -> None: - self.assertTrue((ROOT / "scripts" / "scan_committed_secrets.py").is_file()) + scanner = ROOT / "scripts" / "scan_committed_secrets.py" + self.assertTrue(scanner.is_file()) workflow = (ROOT / ".github" / "workflows" / "validate.yml").read_text(encoding="utf-8") self.assertIn("python3 scripts/scan_committed_secrets.py", workflow) + with tempfile.TemporaryDirectory() as directory: + repository = Path(directory) + subprocess.run(["git", "init", "-q", str(repository)], check=True) + for relative in ("scripts/scan_committed_secrets.py", "scripts/validate.sh"): + path = repository / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("ghp_" + "x" * 20 + "\n", encoding="utf-8") + subprocess.run(["git", "-C", str(repository), "add", "."], check=True) + result = subprocess.run( + [sys.executable, str(scanner), "--repository", str(repository)], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + self.assertNotEqual(result.returncode, 0) + self.assertIn("scripts/scan_committed_secrets.py:1", result.stderr) + self.assertIn("scripts/validate.sh:1", result.stderr) def test_duplicate_json_controller_id_is_rejected(self) -> None: validation = Validation() From df5a79cc78456dc353cfdf68602722cacee48aa2 Mon Sep 17 00:00:00 2001 From: Nickfosts Hermes Date: Tue, 21 Jul 2026 13:03:57 -0500 Subject: [PATCH 46/57] docs: pin final runtime-trust engine --- templates/config-repository/examples/multi-host/fleet.json | 4 ++-- templates/config-repository/fleet.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/templates/config-repository/examples/multi-host/fleet.json b/templates/config-repository/examples/multi-host/fleet.json index cb58ceaf..6f9bab13 100644 --- a/templates/config-repository/examples/multi-host/fleet.json +++ b/templates/config-repository/examples/multi-host/fleet.json @@ -24,7 +24,7 @@ "state": "active", "scale_set_name": "sample-ci-primary", "lifecycle": "stable", - "engine_ref": "6d76f590b26649029d8e50b7a30d86d7aeba31b1", + "engine_ref": "a530c6abdc0245d818dd5fefe72a45189b5e6ab4", "min_runners": 0, "max_runners": 4, "runner_resources": { @@ -38,7 +38,7 @@ "state": "active", "scale_set_name": "sample-ci-remote", "lifecycle": "stable", - "engine_ref": "6d76f590b26649029d8e50b7a30d86d7aeba31b1", + "engine_ref": "a530c6abdc0245d818dd5fefe72a45189b5e6ab4", "min_runners": 0, "max_runners": 2, "runner_resources": { diff --git a/templates/config-repository/fleet.json b/templates/config-repository/fleet.json index 03d1e5f8..c0b89ec5 100644 --- a/templates/config-repository/fleet.json +++ b/templates/config-repository/fleet.json @@ -24,7 +24,7 @@ "state": "active", "scale_set_name": "example-ci-01", "lifecycle": "experimental", - "engine_ref": "6d76f590b26649029d8e50b7a30d86d7aeba31b1", + "engine_ref": "a530c6abdc0245d818dd5fefe72a45189b5e6ab4", "min_runners": 0, "max_runners": 1, "runner_resources": { From 8c51334def45cb97e906f663e5a6e48d1a6763b2 Mon Sep 17 00:00:00 2001 From: Nickfosts Hermes Date: Wed, 22 Jul 2026 00:19:18 -0500 Subject: [PATCH 47/57] fix: harden controller convergence integrity --- scripts/install-worker-controller.sh | 61 +++++++++++++++++-- scripts/scan_committed_secrets.py | 3 +- scripts/test-install-worker-controller.sh | 45 +++++++++++++- .../scripts/scan_committed_secrets.py | 3 +- 4 files changed, 101 insertions(+), 11 deletions(-) diff --git a/scripts/install-worker-controller.sh b/scripts/install-worker-controller.sh index 6f47e25f..8be10d58 100755 --- a/scripts/install-worker-controller.sh +++ b/scripts/install-worker-controller.sh @@ -177,11 +177,14 @@ validate_candidate_config_commit() { } prepare_host_config() { + local expected_owner=0 effective_host_config=$host_config if [[ -f "$host_config" ]]; then return fi if [[ -f "$rendered_env" && "$mode" == adopt ]]; then + [[ "$testing" != 1 ]] || expected_owner=$(id -u) + [[ $(stat -c %u "$rendered_env") == "$expected_owner" && $(stat -c %a "$rendered_env") == 600 ]] || die "rendered environment must be owned by root with mode 0600: $rendered_env" install -d -m 0700 "$etc_dir" python3 "$repo_root/scripts/desired_state.py" extract-host-env \ --source "$rendered_env" --output "$effective_host_config" @@ -205,6 +208,30 @@ verify_host_files() { [[ "$mode_bits" == 600 && "$owner" == "$expected_owner" ]] || die 'GitHub App PEM must be owned by root and have mode 0600' } +load_installed_controller_identity() { + local source_state=${1:-$state_file} source_env=${2:-$rendered_env} expected_owner=0 + [[ "$testing" != 1 ]] || expected_owner=$(id -u) + if [[ -f "$source_state" ]]; then + [[ $(stat -c %u "$source_state") == "$expected_owner" && $(stat -c %a "$source_state") == 600 ]] || die "install state must be owned by root with mode 0600: $source_state" + controller_id=$(python3 - "$source_state" <<'PY' +import json +import sys +try: + value = json.load(open(sys.argv[1], encoding="utf-8"))["controller"] +except (OSError, KeyError, TypeError, ValueError, json.JSONDecodeError): + raise SystemExit(2) +print(value) +PY + ) || die "installed controller identity is invalid: $source_state" + elif [[ -f "$source_env" ]]; then + [[ $(stat -c %u "$source_env") == "$expected_owner" && $(stat -c %a "$source_env") == 600 ]] || die "rendered environment must be owned by root with mode 0600: $source_env" + controller_id=$(awk -F= '$1 == "CI_FLEET_INSTANCE" {count++; value=substr($0, index($0, "=") + 1)} END {if (count != 1) exit 1; print value}' "$source_env") || die "installed controller identity is invalid: $source_env" + else + die 'installed controller identity is unavailable' + fi + [[ "$controller_id" =~ ^[a-z0-9][a-z0-9-]{0,62}$ ]] || die 'installed controller identity is invalid' +} + docker_gid() { if [[ "$testing" == 1 && -n ${CI_FLEET_DOCKER_GID_OVERRIDE:-} ]]; then printf '%s' "$CI_FLEET_DOCKER_GID_OVERRIDE" @@ -272,20 +299,23 @@ current_runtime_release() { managed_runner_count() { docker ps -q \ --filter label=io.randomdevelopment.ci-fleet.managed=true \ - --filter label=io.randomdevelopment.ci-fleet.kind=runner | wc -l | tr -d ' ' + --filter label=io.randomdevelopment.ci-fleet.kind=runner \ + --filter "label=io.randomdevelopment.ci-fleet.instance=$controller_id" | wc -l | tr -d ' ' } managed_runner_total_count() { docker ps --all -q \ --filter label=io.randomdevelopment.ci-fleet.managed=true \ - --filter label=io.randomdevelopment.ci-fleet.kind=runner | wc -l | tr -d ' ' + --filter label=io.randomdevelopment.ci-fleet.kind=runner \ + --filter "label=io.randomdevelopment.ci-fleet.instance=$controller_id" | wc -l | tr -d ' ' } remove_inactive_managed_runners() { local -a containers=() mapfile -t containers < <(docker ps --all -q \ --filter label=io.randomdevelopment.ci-fleet.managed=true \ - --filter label=io.randomdevelopment.ci-fleet.kind=runner) + --filter label=io.randomdevelopment.ci-fleet.kind=runner \ + --filter "label=io.randomdevelopment.ci-fleet.instance=$controller_id") ((${#containers[@]} == 0)) || docker rm "${containers[@]}" >/dev/null } @@ -305,10 +335,15 @@ controller_environment_matches() { } runtime_matches() { - local expected=$1 provenance status + local expected=$1 expected_image expected_image_ref live_image provenance status status=$(controller_status) if [[ "$expected" == active ]]; then [[ "$status" == running ]] || return 1 + expected_image_ref=$(awk -F= '$1 == "CI_FLEET_CONTROLLER_IMAGE" {print substr($0, index($0, "=") + 1)}' "$candidate_env") + [[ -n "$expected_image_ref" ]] || return 1 + live_image=$(docker inspect --format '{{.Image}}' "$controller_container" 2>/dev/null) || return 1 + expected_image=$(docker image inspect --format '{{.Id}}' "$expected_image_ref" 2>/dev/null) || return 1 + [[ "$live_image" == "$expected_image" ]] || return 1 provenance=$(docker inspect --format '{{ index .Config.Labels "org.opencontainers.image.revision" }}' "$controller_container" 2>/dev/null) || return 1 [[ "$provenance" == "$engine_ref" ]] || return 1 controller_environment_matches @@ -333,17 +368,24 @@ PY } runtime_release_complete() { - local path=$1 expected=$2 marker + local path=$1 expected=$2 marker required [[ -d "$path" && -f "$path/.ci-fleet-engine-ref" && -f "$path/deploy/compose.yaml" ]] || return 1 [[ -x "$path/scripts/preflight.sh" && -x "$path/scripts/healthcheck.sh" && -x "$path/scripts/cleanup.sh" ]] || return 1 + for required in controller/Dockerfile controller/go.mod controller/main.go controller/config.go controller/scaler.go controller/state.go runner/Dockerfile; do + [[ -f "$path/$required" ]] || return 1 + done marker=$(<"$path/.ci-fleet-engine-ref") [[ "$marker" == "$expected" ]] } manager_release_complete() { - local path=$1 expected=$2 marker unit + local path=$1 expected=$2 marker required unit runtime_release_complete "$path" "$expected" || return 1 [[ -x "$path/scripts/install-worker-controller.sh" && -x "$path/scripts/check-installed-state.sh" ]] || return 1 + for required in scripts/desired_state.py scripts/scan_committed_secrets.py templates/config-repository/fleet.schema.json templates/config-repository/scripts/validate.py; do + [[ -f "$path/$required" ]] || return 1 + done + [[ -x "$path/templates/config-repository/scripts/validate.sh" ]] || return 1 for unit in "${unit_names[@]}"; do [[ -f "$path/host/systemd/$unit" ]] || return 1; done marker=$(<"$path/.ci-fleet-engine-ref") [[ "$marker" == "$expected" ]] @@ -640,6 +682,10 @@ activate_candidate() { fi else compose "$release_dir" "$rendered_env" stop controller >/dev/null 2>&1 || true + if ! runtime_matches "$target_state"; then + compose "$release_dir" "$rendered_env" down --remove-orphans >/dev/null + runtime_matches "$target_state" || die 'controller did not reach the requested non-active state' + fi fi staged_state=$(mktemp "$state_root/.install-state.XXXXXX") staging_paths+=("$staged_state") @@ -799,16 +845,19 @@ latest_checkpoint() { perform_rollback() { checkpoint_dir=$(latest_checkpoint) [[ -n "$checkpoint_dir" ]] || die 'no controller checkpoint is available' + load_installed_controller_identity "$state_file" "$rendered_env" restore_checkpoint || die 'checkpoint restoration failed' note "ROLLBACK_OK checkpoint=$checkpoint_dir" } perform_uninstall() { local old_release= + load_installed_controller_identity old_release=$(current_runtime_release) make_checkpoint transaction_active=true drain_current + remove_inactive_managed_runners if [[ -n "$old_release" && -f "$rendered_env" ]]; then compose "$old_release" "$rendered_env" down --remove-orphans || true ( diff --git a/scripts/scan_committed_secrets.py b/scripts/scan_committed_secrets.py index fa90daf5..cd191118 100644 --- a/scripts/scan_committed_secrets.py +++ b/scripts/scan_committed_secrets.py @@ -11,12 +11,13 @@ ROOT = Path(__file__).resolve().parents[1] PATTERN = re.compile( - rb"BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY|" + rb"BEGIN (?:[A-Z0-9]+ )*PRIVATE KEY|" rb"github_pat_[A-Za-z0-9_]{20,}|" rb"gh[opusr]_[A-Za-z0-9]{20,}|" rb"AKIA[0-9A-Z]{16}|" rb"(?:postgres|mysql|mongodb(?:\+srv)?|redis)://[^\s/:]+:[^\s/@]+@" ) +assert all(PATTERN.search(b"BEGIN " + kind + b"PRIVATE KEY") for kind in (b"", b"RSA ", b"DSA ", b"EC ", b"OPENSSH ", b"ENCRYPTED ")) assert all(PATTERN.search(prefix + b"x" * 20) for prefix in (b"gho_", b"ghp_", b"ghr_", b"ghs_", b"ghu_")) assert PATTERN.search(b"AKIA" + b"A" * 16) assert PATTERN.search(b"mysql" + b"://fixture-user:***@example.invalid/database") diff --git a/scripts/test-install-worker-controller.sh b/scripts/test-install-worker-controller.sh index fc2f9b6f..cd7ac019 100755 --- a/scripts/test-install-worker-controller.sh +++ b/scripts/test-install-worker-controller.sh @@ -26,6 +26,9 @@ case "${1:-}" in cat "$FAKE_CONTROLLER_ENV_FILE" elif [[ "$*" == *'org.opencontainers.image.revision'* ]]; then if [[ -n "${FAKE_CONTROLLER_PROVENANCE_FILE:-}" && -f "$FAKE_CONTROLLER_PROVENANCE_FILE" ]]; then cat "$FAKE_CONTROLLER_PROVENANCE_FILE"; else printf '%s\n' "${FAKE_ENGINE_REF:?}"; fi + elif [[ "$*" == *'{{.Image}}'* ]]; then + [[ -n "${FAKE_CONTROLLER_IMAGE_ID_FILE:-}" && -f "$FAKE_CONTROLLER_IMAGE_ID_FILE" ]] || exit 1 + cat "$FAKE_CONTROLLER_IMAGE_ID_FILE" elif [[ "$*" == *'.State.Status'* ]]; then if [[ -n "$status_file" && -f "$status_file" ]]; then cat "$status_file"; else printf '%s\n' "${FAKE_CONTROLLER_STATUS:-running}"; fi elif [[ "$*" == *'.State.Paused'* ]]; then @@ -35,6 +38,7 @@ case "${1:-}" in fi ;; ps) + [[ -z "${FAKE_DOCKER_PS_LOG:-}" ]] || printf '%s\n' "$*" >>"$FAKE_DOCKER_PS_LOG" if [[ "$*" == *'--all'* && -n "${FAKE_ALL_RUNNER_STATE:-}" && -f "$FAKE_ALL_RUNNER_STATE" ]]; then printf 'managed-runner-all-state\n' elif [[ -n "${FAKE_RUNNER_STATE_ONCE:-}" && -f "$FAKE_RUNNER_STATE_ONCE" ]]; then @@ -57,7 +61,7 @@ case "${1:-}" in exit 1 fi [[ -f "$image_state" ]] || exit 1 - cat "$image_state" + if [[ "$*" == *'{{.Id}}'* ]]; then printf 'sha256:%s\n' "$(<"$image_state")"; else cat "$image_state"; fi ;; rm) (($# >= 2)) || exit 1 @@ -85,6 +89,7 @@ case "${1:-}" in fi : >"$state" [[ -z "${FAKE_CONTROLLER_PROVENANCE_FILE:-}" ]] || printf '%s\n' "${FAKE_ENGINE_REF:?}" >"$FAKE_CONTROLLER_PROVENANCE_FILE" + [[ -z "${FAKE_CONTROLLER_IMAGE_ID_FILE:-}" ]] || printf 'sha256:%s\n' "${FAKE_ENGINE_REF:?}" >"$FAKE_CONTROLLER_IMAGE_ID_FILE" [[ -z "${FAKE_CONTROLLER_ENV_FILE:-}" ]] || cp "$env_file" "$FAKE_CONTROLLER_ENV_FILE" if [[ -n "${FAKE_RESTART_AFTER_UP:-}" && -f "$FAKE_RESTART_AFTER_UP" ]]; then rm -f "$FAKE_RESTART_AFTER_UP" @@ -93,7 +98,11 @@ case "${1:-}" in rm -f "$status_file" fi ;; - stop|down|rm) rm -f "$state"; [[ -z "$status_file" ]] || rm -f "$status_file"; [[ -z "$paused_state" ]] || rm -f "$paused_state"; [[ -z "${FAKE_CONTROLLER_PROVENANCE_FILE:-}" ]] || rm -f "$FAKE_CONTROLLER_PROVENANCE_FILE"; [[ -z "${FAKE_CONTROLLER_ENV_FILE:-}" ]] || rm -f "$FAKE_CONTROLLER_ENV_FILE" ;; + stop) + if [[ -n "${FAKE_STOP_FAIL:-}" && -f "$FAKE_STOP_FAIL" ]]; then exit 42; fi + rm -f "$state"; [[ -z "$status_file" ]] || rm -f "$status_file"; [[ -z "$paused_state" ]] || rm -f "$paused_state"; [[ -z "${FAKE_CONTROLLER_PROVENANCE_FILE:-}" ]] || rm -f "$FAKE_CONTROLLER_PROVENANCE_FILE"; [[ -z "${FAKE_CONTROLLER_IMAGE_ID_FILE:-}" ]] || rm -f "$FAKE_CONTROLLER_IMAGE_ID_FILE"; [[ -z "${FAKE_CONTROLLER_ENV_FILE:-}" ]] || rm -f "$FAKE_CONTROLLER_ENV_FILE" + ;; + down|rm) rm -f "$state"; [[ -z "$status_file" ]] || rm -f "$status_file"; [[ -z "$paused_state" ]] || rm -f "$paused_state"; [[ -z "${FAKE_CONTROLLER_PROVENANCE_FILE:-}" ]] || rm -f "$FAKE_CONTROLLER_PROVENANCE_FILE"; [[ -z "${FAKE_CONTROLLER_IMAGE_ID_FILE:-}" ]] || rm -f "$FAKE_CONTROLLER_IMAGE_ID_FILE"; [[ -z "${FAKE_CONTROLLER_ENV_FILE:-}" ]] || rm -f "$FAKE_CONTROLLER_ENV_FILE" ;; pause) [[ -z "$paused_state" ]] || : >"$paused_state" [[ -z "${FAKE_RUNNER_STATE:-}" ]] || rm -f "$FAKE_RUNNER_STATE" @@ -158,7 +167,9 @@ export FAKE_DOCKER_STATE=$tmp/docker-controller-running export FAKE_CONTROLLER_STATUS_FILE=$tmp/docker-controller-status export FAKE_PAUSED_STATE=$tmp/docker-controller-paused export FAKE_CONTROLLER_PROVENANCE_FILE=$tmp/docker-controller-provenance +export FAKE_CONTROLLER_IMAGE_ID_FILE=$tmp/docker-controller-image-id export FAKE_CONTROLLER_ENV_FILE=$tmp/docker-controller-env +export FAKE_DOCKER_PS_LOG=$tmp/docker-ps.log export CI_FLEET_TESTING=1 export CI_FLEET_DOCKER_GID_OVERRIDE=998 export CI_FLEET_STARTUP_WAIT_SECONDS=0 @@ -300,6 +311,10 @@ printf '%040d\n' 0 >"$FAKE_CONTROLLER_PROVENANCE_FILE" expect_failure 'DRIFT controller_runtime' "$installer" --check "${base_args[@]}" --ref "$ref_one" expect_success "$installer" --install "${base_args[@]}" --ref "$ref_one" >/dev/null [[ $(<"$FAKE_CONTROLLER_PROVENANCE_FILE") == "$engine_ref" ]] || fail 'controller convergence did not restore running image provenance' +printf 'sha256:%040d\n' 0 >"$FAKE_CONTROLLER_IMAGE_ID_FILE" +expect_failure 'DRIFT controller_runtime' "$installer" --check "${base_args[@]}" --ref "$ref_one" +expect_success "$installer" --install "${base_args[@]}" --ref "$ref_one" >/dev/null +[[ $(<"$FAKE_CONTROLLER_IMAGE_ID_FILE") == "sha256:$engine_ref" ]] || fail 'controller convergence did not restore live image identity' python3 -c 'from pathlib import Path; import sys; path = Path(sys.argv[1]); path.write_text(path.read_text().replace("CI_FLEET_MAX_RUNNERS=1", "CI_FLEET_MAX_RUNNERS=9"))' "$FAKE_CONTROLLER_ENV_FILE" grep -Fxq 'CI_FLEET_MAX_RUNNERS=9' "$FAKE_CONTROLLER_ENV_FILE" || fail 'live-environment fixture did not mutate' expect_failure 'DRIFT controller_runtime' "$installer" --check "${base_args[@]}" --ref "$ref_one" @@ -338,6 +353,10 @@ rm -f "$active_release/scripts/preflight.sh" expect_failure 'DRIFT engine_release' "$installer" --check "${base_args[@]}" --ref "$ref_one" expect_success "$installer" --install "${base_args[@]}" --ref "$ref_one" >/dev/null [[ -x "$active_release/scripts/preflight.sh" ]] || fail 'required runtime script was not repaired' +rm -f "$active_release/controller/main.go" "$active_release/runner/Dockerfile" +expect_failure 'DRIFT engine_release' "$installer" --check "${base_args[@]}" --ref "$ref_one" +expect_success "$installer" --install "${base_args[@]}" --ref "$ref_one" >/dev/null +[[ -f "$active_release/controller/main.go" && -f "$active_release/runner/Dockerfile" ]] || fail 'runtime build inputs were not repaired' rm -f "$active_release/.ci-fleet-engine-ref" expect_failure 'DRIFT engine_release' "$installer" --check "${base_args[@]}" --ref "$ref_one" expect_success "$installer" --install "${base_args[@]}" --ref "$ref_one" >/dev/null @@ -360,6 +379,10 @@ rm -f "$manager_release/scripts/check-installed-state.sh" expect_failure 'DRIFT maintenance_timers' "$installer" --check "${base_args[@]}" --ref "$ref_one" expect_success "$installer" --install "${base_args[@]}" --ref "$ref_one" >/dev/null [[ -x "$manager_release/scripts/check-installed-state.sh" ]] || fail 'incomplete manager release was not repaired consistently' +rm -f "$manager_release/scripts/desired_state.py" "$manager_release/templates/config-repository/fleet.schema.json" +expect_failure 'DRIFT maintenance_timers' "$installer" --check "${base_args[@]}" --ref "$ref_one" +expect_success "$installer" --install "${base_args[@]}" --ref "$ref_one" >/dev/null +[[ -f "$manager_release/scripts/desired_state.py" && -f "$manager_release/templates/config-repository/fleet.schema.json" ]] || fail 'manager helper inputs were not repaired' mv "$active_release" "$active_release.saved" expect_failure 'DRIFT engine_release' "$installer" --check "${base_args[@]}" --ref "$ref_one" mv "$active_release.saved" "$active_release" @@ -414,7 +437,12 @@ expect_success "$installer" --rollback >/dev/null grep -Fq 'CI_FLEET_MAX_RUNNERS=1' "$root/etc/ci-fleet/ci-fleet.env" || fail 'rollback did not restore capacity one' ref_three=$(write_config drained 2 2) +printf 'dead\n' >"$FAKE_CONTROLLER_STATUS_FILE" +export FAKE_STOP_FAIL=$tmp/stop-dead-fails +: >"$FAKE_STOP_FAIL" expect_success "$installer" --upgrade "${base_args[@]}" --ref "$ref_three" >/dev/null +unset FAKE_STOP_FAIL +[[ ! -f "$FAKE_DOCKER_STATE" && ! -f "$FAKE_CONTROLLER_STATUS_FILE" ]] || fail 'non-active convergence retained a dead controller' grep -Fq 'CI_FLEET_CONTROLLER_STATE=drained' "$root/etc/ci-fleet/ci-fleet.env" || fail 'drained state was not rendered' grep -Fq 'CI_FLEET_MAX_RUNNERS=0' "$root/etc/ci-fleet/ci-fleet.env" || fail 'drained controller retained effective capacity' [[ ! -f "$FAKE_DOCKER_STATE" ]] || fail 'drained controller remained running' @@ -425,9 +453,11 @@ rm -f "$FAKE_RUNNER_STATE" unset FAKE_RUNNER_STATE export FAKE_ALL_RUNNER_STATE=$tmp/drained-exited-managed-runner : >"$FAKE_ALL_RUNNER_STATE" +: >"$FAKE_DOCKER_PS_LOG" expect_failure 'DRIFT managed_runners' "$installer" --check "${base_args[@]}" --ref "$ref_three" expect_success "$installer" --install "${base_args[@]}" --ref "$ref_three" >/dev/null [[ ! -f "$FAKE_ALL_RUNNER_STATE" ]] || fail 'non-active convergence did not remove stopped managed runners' +grep -Fq 'label=io.randomdevelopment.ci-fleet.instance=example-ci-01' "$FAKE_DOCKER_PS_LOG" || fail 'managed runner cleanup was not scoped to the selected instance' unset FAKE_ALL_RUNNER_STATE ref_four=$(write_config disabled 2 2) @@ -441,9 +471,15 @@ unset FAKE_RUNNER_STATE export FAKE_RUNNER_STATE_ONCE=$tmp/orphaned-managed-runner : >"$FAKE_RUNNER_STATE_ONCE" +export FAKE_ALL_RUNNER_STATE=$tmp/uninstall-stopped-managed-runner +: >"$FAKE_ALL_RUNNER_STATE" +: >"$FAKE_DOCKER_PS_LOG" expect_success "$installer" --uninstall >/dev/null [[ ! -f "$FAKE_RUNNER_STATE_ONCE" ]] || fail 'uninstall did not wait for an orphaned managed runner' -unset FAKE_RUNNER_STATE_ONCE +[[ ! -f "$FAKE_ALL_RUNNER_STATE" ]] || fail 'uninstall retained stopped managed runners' +grep -Fq 'label=io.randomdevelopment.ci-fleet.instance=example-ci-01' "$FAKE_DOCKER_PS_LOG" || fail 'uninstall runner cleanup was not scoped to the installed instance' +if grep -Eq 'label=io.randomdevelopment.ci-fleet.instance=$' "$FAKE_DOCKER_PS_LOG"; then fail 'uninstall runner cleanup used an empty instance filter'; fi +unset FAKE_RUNNER_STATE_ONCE FAKE_ALL_RUNNER_STATE [[ ! -e "$root/opt/ci-fleet/current" && ! -e "$root/var/lib/ci-fleet/install-state.json" ]] || fail 'uninstall left active installation state' [[ -f "$host_config" && -f "$pem" ]] || fail 'uninstall removed preserved host credentials' @@ -462,6 +498,9 @@ printf '%s\n' \ 'CI_FLEET_RUNNER_TTL=6h' >"$adopt_root/etc/ci-fleet/ci-fleet.env" chmod 600 "$adopt_root/etc/ci-fleet/ci-fleet.env" : >"$FAKE_DOCKER_STATE" +chmod 644 "$adopt_root/etc/ci-fleet/ci-fleet.env" +expect_failure 'rendered environment must be owned by root with mode 0600' "$installer" --adopt "${base_args[@]}" --ref "$ref_one" +chmod 600 "$adopt_root/etc/ci-fleet/ci-fleet.env" adopt=$(expect_success "$installer" --adopt "${base_args[@]}" --ref "$ref_one") grep -Fq 'CONVERGED mode=adopt' <<<"$adopt" || fail 'adoption did not converge' diff --git a/templates/config-repository/scripts/scan_committed_secrets.py b/templates/config-repository/scripts/scan_committed_secrets.py index fa90daf5..cd191118 100755 --- a/templates/config-repository/scripts/scan_committed_secrets.py +++ b/templates/config-repository/scripts/scan_committed_secrets.py @@ -11,12 +11,13 @@ ROOT = Path(__file__).resolve().parents[1] PATTERN = re.compile( - rb"BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY|" + rb"BEGIN (?:[A-Z0-9]+ )*PRIVATE KEY|" rb"github_pat_[A-Za-z0-9_]{20,}|" rb"gh[opusr]_[A-Za-z0-9]{20,}|" rb"AKIA[0-9A-Z]{16}|" rb"(?:postgres|mysql|mongodb(?:\+srv)?|redis)://[^\s/:]+:[^\s/@]+@" ) +assert all(PATTERN.search(b"BEGIN " + kind + b"PRIVATE KEY") for kind in (b"", b"RSA ", b"DSA ", b"EC ", b"OPENSSH ", b"ENCRYPTED ")) assert all(PATTERN.search(prefix + b"x" * 20) for prefix in (b"gho_", b"ghp_", b"ghr_", b"ghs_", b"ghu_")) assert PATTERN.search(b"AKIA" + b"A" * 16) assert PATTERN.search(b"mysql" + b"://fixture-user:***@example.invalid/database") From 548ba9132af1af6609cbe58b72bebb1d8577c211 Mon Sep 17 00:00:00 2001 From: Nickfosts Hermes Date: Wed, 22 Jul 2026 00:20:28 -0500 Subject: [PATCH 48/57] chore: advance reviewed engine pins --- templates/config-repository/examples/multi-host/fleet.json | 4 ++-- templates/config-repository/fleet.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/templates/config-repository/examples/multi-host/fleet.json b/templates/config-repository/examples/multi-host/fleet.json index 6f9bab13..661bdf02 100644 --- a/templates/config-repository/examples/multi-host/fleet.json +++ b/templates/config-repository/examples/multi-host/fleet.json @@ -24,7 +24,7 @@ "state": "active", "scale_set_name": "sample-ci-primary", "lifecycle": "stable", - "engine_ref": "a530c6abdc0245d818dd5fefe72a45189b5e6ab4", + "engine_ref": "8c51334def45cb97e906f663e5a6e48d1a6763b2", "min_runners": 0, "max_runners": 4, "runner_resources": { @@ -38,7 +38,7 @@ "state": "active", "scale_set_name": "sample-ci-remote", "lifecycle": "stable", - "engine_ref": "a530c6abdc0245d818dd5fefe72a45189b5e6ab4", + "engine_ref": "8c51334def45cb97e906f663e5a6e48d1a6763b2", "min_runners": 0, "max_runners": 2, "runner_resources": { diff --git a/templates/config-repository/fleet.json b/templates/config-repository/fleet.json index c0b89ec5..22861864 100644 --- a/templates/config-repository/fleet.json +++ b/templates/config-repository/fleet.json @@ -24,7 +24,7 @@ "state": "active", "scale_set_name": "example-ci-01", "lifecycle": "experimental", - "engine_ref": "a530c6abdc0245d818dd5fefe72a45189b5e6ab4", + "engine_ref": "8c51334def45cb97e906f663e5a6e48d1a6763b2", "min_runners": 0, "max_runners": 1, "runner_resources": { From b1da1101c49f6cbe67da278cdf41150b30a224e8 Mon Sep 17 00:00:00 2001 From: Nickfosts Hermes Date: Wed, 22 Jul 2026 01:19:54 -0500 Subject: [PATCH 49/57] fix: close controller adoption review gaps --- AGENTS.md | 2 +- docs/CAPACITY-PROMOTION.md | 152 ++++++++-------------- docs/DESIGN-DECISIONS.md | 16 +++ docs/README.md | 4 +- scripts/install-worker-controller.sh | 46 ++++--- scripts/test-capacity-preflight.sh | 6 +- scripts/test-install-worker-controller.sh | 18 ++- 7 files changed, 122 insertions(+), 122 deletions(-) create mode 100644 docs/DESIGN-DECISIONS.md diff --git a/AGENTS.md b/AGENTS.md index db0bed70..bbd965a1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -6,7 +6,7 @@ This repository defines a portable, public, self-hosted GitHub Actions CI fleet ## Current phase -The project is in architecture definition and isolated proof-of-concept planning. Do not create or alter production deployment paths until the design decisions are recorded and the isolated proof of concept is approved. +The isolated proof of concept is approved and the project is in controlled migration. The accepted scope and remaining gates are recorded in `docs/DESIGN-DECISIONS.md`. Managed ordinary-CI controller lifecycle is permitted only through reviewed schema-v3 desired state; production deployment paths remain separately gated. ## Sources of truth diff --git a/docs/CAPACITY-PROMOTION.md b/docs/CAPACITY-PROMOTION.md index 8d1bc507..cf0a7753 100644 --- a/docs/CAPACITY-PROMOTION.md +++ b/docs/CAPACITY-PROMOTION.md @@ -2,158 +2,108 @@ Use this procedure only after the strict one-runner pilot has passed. It validates and changes one already isolated controller; it does not authorize a project workflow by itself. -`scripts/preflight.sh` remains the initial pilot gate and accepts only `MIN=0`, `MAX=1`. `scripts/capacity-preflight.sh` is a separate, read-only post-pilot gate. It never edits host configuration. +Schema-v3 private desired state owns capacity. Never edit `/etc/ci-fleet/ci-fleet.env` directly: it is rendered state and the drift checker will replace local changes. ## Capacity policy Declare the policy before observing a larger workload: -- requested MAX is explicitly and exclusively two for this first post-pilot procedure; any other target is rejected; +- requested MAX is exactly two for this first post-pilot procedure; - `MIN` remains zero; -- the configured instance, scale set, routing label, runner group, Docker socket group, and per-runner limits must equal the running controller's effective values; +- both the controller `max_runners` and pool `capacity_budget` change in one reviewed private configuration PR; +- the configured instance, scale set, routing label, runner group, Docker socket group, and per-runner limits remain unchanged; - no managed runner, project job resource, unrelated running container, controller OOM, or current-boot kernel OOM evidence may exist; -- Docker filesystem use must be below `CI_FLEET_DISK_WARN_PERCENT` (80 by default); -- reserve the controller's Compose limit: 1 CPU and 512 MiB; -- reserve 1 CPU and 1 GiB for Docker overhead; -- reserve for the operating system the greater of 1 CPU or 15% of logical CPU capacity; -- reserve for the operating system the greater of 2 GiB or 20% of physical memory; -- admit `target MAX × CI_FLEET_RUNNER_CPUS` and `target MAX × CI_FLEET_RUNNER_MEMORY_MIB` only when those allocations plus all reserves fit; +- Docker filesystem use stays below `CI_FLEET_DISK_WARN_PERCENT` (80 by default); +- reserve 1 CPU and 512 MiB for the controller, 1 CPU and 1 GiB for Docker overhead, and the greater of 1 CPU/15% plus 2 GiB/20% for the operating system; +- admit `target MAX × CI_FLEET_RUNNER_CPUS` and `target MAX × CI_FLEET_RUNNER_MEMORY_MIB` only when the allocation and reserves fit; and - require currently available memory to cover all target runners plus controller and Docker reservations. -Runner limits are controller admission inputs. Project containers use the host Docker daemon as siblings of the runner container, so the separately authorized live proof must still observe whole-host CPU, memory, disk, collision, and cleanup behavior. +Project containers use the host Docker daemon as siblings of the runner container. The separately authorized live proof must therefore observe whole-host CPU, memory, disk, collisions, and cleanup. -During a live proof, retain the target only if every five-second sample keeps CPU busy below 85%, available memory at or above the greater of 2 GiB or 20% of total memory, and Docker filesystem use below 80%. Any OOM, unrelated workload, controller/Docker failure, third runner, observer gap, or cleanup residue requires restoration. +During that proof, retain MAX=2 only if every five-second sample keeps CPU busy below 85%, available memory at or above the greater of 2 GiB or 20% of total memory, and Docker filesystem use below 80%. Any OOM, unrelated workload, controller/Docker failure, third runner, observer gap, or cleanup residue requires restoration. -## 1. Verify idle pilot state +## 1. Gate dispatch and verify the one-runner state -Gate all dispatches that can target this controller. Confirm no queued, assigned, or running fleet job and no instance-owned runner in any state. Keep the dispatch gate closed until post-change verification completes. +Block dispatches that can target this controller. Confirm no queued, assigned, or running fleet job and no instance-owned runner in any state. Keep dispatch gated through post-change verification. -From the reviewed checkout, load the root-only host configuration without tracing or printing it: - -```bash -set -Eeuo pipefail -set +x -cd /opt/ci-fleet -set -a -. /etc/ci-fleet/ci-fleet.env -set +a -scripts/preflight.sh -``` - -Require exactly `PREFLIGHT_OK warnings=0`. This proves the original `MIN=0`, `MAX=1` pilot contract remains valid. +Run the current installed preflight and health checks from clean processes. Require the existing one-runner contract, controller health, and an empty instance-scoped cleanup dry-run. ## 2. Validate target capacity without changing it +From the installed reviewed release, run: + ```bash scripts/capacity-preflight.sh --phase pre-change --target-max 2 ``` -Require `CAPACITY_PREFLIGHT_OK phase=pre-change target_max=2 configured_max=1 effective_max=1`. Record only the safe budget summaries. - -## 3. Create one protected backup +Require: -Use one UTC timestamp and refuse to overwrite an existing path: - -```bash -backup_dir=/etc/ci-fleet/backups -stamp=$(date -u +%Y%m%dT%H%M%SZ) -backup="$backup_dir/ci-fleet.env.before-max2.$stamp" -install -d -o root -g root -m 0700 "$backup_dir" -test ! -e "$backup" -install -o root -g root -m 0600 /etc/ci-fleet/ci-fleet.env "$backup" -cmp -s /etc/ci-fleet/ci-fleet.env "$backup" -sha256sum "$backup" | cut -d' ' -f1 +```text +CAPACITY_PREFLIGHT_OK phase=pre-change target_max=2 configured_max=1 effective_max=1 ``` -Record the path and checksum only. Never print or copy the backup contents, and do not delete the backup during the proof. +Record only the safe budget summary. -## 4. Change one exact setting +## 3. Review and merge private desired state -Before editing, require exactly one active assignment and the pilot value: +In the secret-free private configuration repository: -```bash -test "$(grep -c '^CI_FLEET_MAX_RUNNERS=' /etc/ci-fleet/ci-fleet.env)" -eq 1 -grep -qx 'CI_FLEET_MAX_RUNNERS=1' /etc/ci-fleet/ci-fleet.env -``` +1. raise only the selected controller's `max_runners` from one to two; +2. raise its pool `capacity_budget` only as needed to admit that reviewed controller maximum; +3. keep `min_runners`, runner resources, identity, lifecycle, routing, trust, and engine pin unchanged; +4. run the complete strict validator, policy tests, and committed-secret scan; +5. obtain the repository's configured exact-head review and CI gates; and +6. merge normally. -Edit only that line to `CI_FLEET_MAX_RUNNERS=2`. Then verify without printing a diff: +Record the previous and new full private configuration commits. Do not edit a rendered host file or bypass the desired-state PR. -```bash -test "$(stat -c '%u:%g:%a' /etc/ci-fleet/ci-fleet.env)" = 0:0:600 -test "$(grep -c '^CI_FLEET_MAX_RUNNERS=' /etc/ci-fleet/ci-fleet.env)" -eq 1 -grep -qx 'CI_FLEET_MAX_RUNNERS=2' /etc/ci-fleet/ci-fleet.env -cmp -s \ - <(grep -v '^CI_FLEET_MAX_RUNNERS=' "$backup") \ - <(grep -v '^CI_FLEET_MAX_RUNNERS=' /etc/ci-fleet/ci-fleet.env) -env -i PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOME=/root \ - docker compose --env-file /etc/ci-fleet/ci-fleet.env -f deploy/compose.yaml config --quiet -``` - -Do not run the pilot preflight against MAX=2; it must continue to reject that value. Every Compose command below uses `env -i` so stale values exported when the pilot file was sourced cannot override the explicit `--env-file` during promotion or rollback. - -## 5. Recreate only the controller +## 4. Apply the exact merged configuration -Reconfirm zero runners and zero jobs immediately before stopping. Stop only the controller with enough grace for scale-set deletion: +Reconfirm the idle gates, then apply the exact merged private commit through the installed manager: ```bash -env -i PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOME=/root \ - docker compose --env-file /etc/ci-fleet/ci-fleet.env -f deploy/compose.yaml stop -t 60 controller +sudo /opt/ci-fleet/manager/current/scripts/install-worker-controller.sh \ + --upgrade \ + --config-repo ORGANIZATION/PRIVATE-CONFIGURATION \ + --ref NEW_PRIVATE_CONFIGURATION_COMMIT \ + --controller CONTROLLER_ID ``` -Verify the exact old scale set is absent and there is no runner before starting the replacement. Do not delete an apparent duplicate until ownership and zero active jobs are proven. - -```bash -env -i PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOME=/root \ - docker compose --env-file /etc/ci-fleet/ci-fleet.env -f deploy/compose.yaml \ - up -d --no-deps --force-recreate --timeout 60 controller -``` +The installer must create a protected checkpoint, drain the selected instance, validate the managed target, change only the selected controller, start it, and pass health before reporting convergence. A failed activation must restore the checkpoint and keep dispatch closed. -Do not recreate `runner-image`, remove volumes, rebuild images, or touch unrelated resources. +Do not recreate runner jobs, remove volumes, or touch unrelated Docker resources. -## 6. Verify effective state +## 5. Verify effective state -Wait boundedly for one sanitized `controller ready` record reporting `minRunners=0` and `maxRunners=2`. Require one running controller, restart count zero, one intended scale set, the unchanged experimental routing label and runner group, and no idle runner. +Require one running controller, restart count zero, one intended scale set, the unchanged routing label and runner group, no idle runner, and the exact reviewed MAX=2. -Reload the host configuration and run: +Run: ```bash scripts/capacity-preflight.sh --phase post-change --target-max 2 scripts/healthcheck.sh ``` -Require both to pass. The post-change preflight compares configured state with a filtered set of effective controller values and never prints arbitrary container environment entries. +Require both to pass, plus a clean desired-state `--check` and empty instance-scoped cleanup dry-run. -## 7. Run one separately authorized proof +## 6. Run one separately authorized proof Start bounded runner, task-job, project-resource, and host-metric observers before dispatch. Dispatch exactly the approved workload once. Do not retry a failed proof and do not raise MAX above two. -Observe runner creation/destruction, actual two-way overlap, no third runner, whole-host CPU/memory/disk thresholds, Docker/controller health, exact project identity, and automatic cleanup. Run the instance-scoped cleanup dry-run and healthcheck after all jobs terminate. - -## 8. Retain or restore (rollback) - -Retain MAX=2 only when the authorized workload succeeds, actual two-way job and runner overlap is proven, every predeclared resource threshold passes, no manual cleanup is required, all runner/project residue is zero, post-change capacity preflight passes, cleanup dry-run is empty, and healthcheck passes. +Observe runner creation/destruction, actual two-way overlap, no third runner, whole-host resource thresholds, Docker/controller health, exact project identity, and automatic cleanup. Repeat health, drift, and cleanup dry-run checks after all jobs terminate. -On any failure, keep dispatch gated, wait for exact job termination, and restore the backup: +## 7. Retain or restore -```bash -env -i PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOME=/root \ - docker compose --env-file /etc/ci-fleet/ci-fleet.env -f deploy/compose.yaml stop -t 60 controller -install -o root -g root -m 0600 "$backup" /etc/ci-fleet/ci-fleet.env -cmp -s "$backup" /etc/ci-fleet/ci-fleet.env -env -i PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOME=/root \ - docker compose --env-file /etc/ci-fleet/ci-fleet.env -f deploy/compose.yaml config --quiet -env -i PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOME=/root \ - docker compose --env-file /etc/ci-fleet/ci-fleet.env -f deploy/compose.yaml \ - up -d --no-deps --force-recreate --timeout 60 controller -``` +Retain MAX=2 only when every predeclared gate passes and no manual cleanup is required. -Verify a sanitized ready record with `MIN=0`, `MAX=1`, then run the pilot preflight and healthcheck from clean processes that source only the restored file: +On any failure, keep dispatch gated and apply the recorded previous private configuration commit through the same reviewed `--upgrade` path: ```bash -env -i PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOME=/root \ - bash -c 'set -a; . /etc/ci-fleet/ci-fleet.env; set +a; exec scripts/preflight.sh' -env -i PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOME=/root \ - bash -c 'set -a; . /etc/ci-fleet/ci-fleet.env; set +a; exec scripts/healthcheck.sh' +sudo /opt/ci-fleet/manager/current/scripts/install-worker-controller.sh \ + --upgrade \ + --config-repo ORGANIZATION/PRIVATE-CONFIGURATION \ + --ref PREVIOUS_PRIVATE_CONFIGURATION_COMMIT \ + --controller CONTROLLER_ID ``` -Run the instance-scoped cleanup dry-run. Require zero runners, zero jobs, one intended scale set, no duplicate controller, and no residue before reopening dispatch. +Require the restored MAX=1, clean desired-state check, health pass, zero runners/jobs/residue, one intended scale set, and empty instance-scoped cleanup dry-run before reopening dispatch. Preserve installer checkpoint and proof evidence without recording credential values. diff --git a/docs/DESIGN-DECISIONS.md b/docs/DESIGN-DECISIONS.md new file mode 100644 index 00000000..05ab5d51 --- /dev/null +++ b/docs/DESIGN-DECISIONS.md @@ -0,0 +1,16 @@ +# Accepted design decisions + +## Managed controller desired state after the isolated proof + +**Status:** accepted on 2026-07-22 for isolated CI fleet hosts. + +The isolated one-job proof required by [Issue #7](https://github.com/RandomDevelopment/ci-fleet/issues/7) passed on 2026-07-15. It proved the organization-owned App boundary, selected private runner group, `MIN=0` / `MAX=1` controller, read-only job permissions, one-job ephemeral runner lifecycle, scoped cleanup, zero final job residue, controller health, and preservation of existing project runners. + +[Issue #32](https://github.com/RandomDevelopment/ci-fleet/issues/32) and its reviewed implementation in PR #33 accept schema-v3 Git-authored controller desired state as the next migration phase. The managed installer may install, adopt, check, upgrade, roll back, or uninstall an isolated **ordinary-CI controller** only from: + +- a merged, publicly reachable `RandomDevelopment/ci-fleet` engine commit; +- a merged, secret-free private configuration commit; +- host-local root-owned credentials and identity; and +- a recoverable checkpoint with the documented health, drain, drift, and rollback gates. + +This decision does not authorize application production deployment, privileged delivery on ordinary-CI runners, unreviewed capacity increases, public-repository runner access, unrestricted Docker cleanup, legacy-runner retirement, or VM deletion. Those remain separately gated by repository policy and operator approval. diff --git a/docs/README.md b/docs/README.md index 9deab26b..4d6eb0e3 100644 --- a/docs/README.md +++ b/docs/README.md @@ -18,6 +18,7 @@ Use this index to find ci-fleet concepts, requirements, examples, and step-by-st | Split tests across parallel workers | [Project CI standard](PROJECT-STANDARD.md) and the [parallel workflow example](../examples/workflows/parallel-ci.yml.example) | | Configure automatic updates and cleanup | [Host maintenance](HOST-MAINTENANCE.md) | | Handle GitHub App, workflow, or deployment secrets | [Secrets model](SECRETS.md) and [security policy](../SECURITY.md) | +| Review accepted implementation scope | [Design decisions](DESIGN-DECISIONS.md) | | Run private CI or deployment for a public project | [Public projects, private delivery, and private configuration](PUBLIC-PRIVATE-CONFIGURATION.md) | | Review current priorities | [Roadmap](ROADMAP.md) | | See what informed the design | [Discovery summary](DISCOVERY-SUMMARY.md) | @@ -41,6 +42,7 @@ Use this index to find ci-fleet concepts, requirements, examples, and step-by-st - [System architecture](ARCHITECTURE.md) - [Runner controller design](CONTROLLER-DESIGN.md) +- [Accepted design decisions](DESIGN-DECISIONS.md) - [Public projects, private delivery, and private configuration](PUBLIC-PRIVATE-CONFIGURATION.md) - [Secrets model](SECRETS.md) - [Security policy](../SECURITY.md) @@ -67,7 +69,7 @@ These pages are normative for compatible projects and hosts: - [Maintain, drain, clean, update, and reboot hosts](HOST-MAINTENANCE.md) - [Use the public configuration-repository scaffold](../templates/config-repository/README.md) -The schema-v3 worker-controller installer is implemented as `scripts/install-worker-controller.sh`. It remains experimental; use the [desired-state guide](DESIRED-STATE.md), pin both configuration and engine commits, and retain the manual pilot runbook for isolated first-job proof. +The schema-v3 worker-controller installer is implemented as `scripts/install-worker-controller.sh`. Its accepted scope is isolated ordinary-CI fleet hosts under reviewed desired state; use the [desired-state guide](DESIRED-STATE.md), pin both configuration and engine commits, and retain the manual pilot runbook for each host's isolated first-job proof. ## Project integration examples diff --git a/scripts/install-worker-controller.sh b/scripts/install-worker-controller.sh index 8be10d58..47a7c94d 100755 --- a/scripts/install-worker-controller.sh +++ b/scripts/install-worker-controller.sh @@ -425,8 +425,12 @@ systemd_matches() { } drift_count() { - local count=0 - if [[ ! -f "$rendered_env" ]] || ! cmp -s "$candidate_env" "$rendered_env"; then + local count=0 expected_owner=0 + [[ "$testing" != 1 ]] || expected_owner=$(id -u) + if [[ ! -f "$rendered_env" ]] \ + || [[ $(stat -c %u "$rendered_env") != "$expected_owner" ]] \ + || [[ $(stat -c %a "$rendered_env") != 600 ]] \ + || ! cmp -s "$candidate_env" "$rendered_env"; then note 'DRIFT rendered_environment' count=$((count + 1)) fi @@ -579,6 +583,7 @@ make_checkpoint() { try_drain_current() { local deadline count old_release status paused=false force_nonterminal=${1:-false} + local drain_env=${2:-$rendered_env} fallback_release=${3:-} drain_error= status=$(controller_status) case "$status" in @@ -588,18 +593,20 @@ try_drain_current() { drain_error="cannot safely drain controller in non-terminal state: $status" return 1 fi - [[ -f "$rendered_env" ]] || { drain_error='cannot stop a non-terminal candidate without its rendered environment'; return 1; } + [[ -f "$drain_env" ]] || { drain_error='cannot stop a non-terminal candidate without its rendered environment'; return 1; } old_release=$(current_runtime_release) + [[ -n "$old_release" ]] || old_release=$fallback_release [[ -n "$old_release" ]] || { drain_error='cannot stop a non-terminal candidate without its runtime release'; return 1; } - compose "$old_release" "$rendered_env" stop controller >/dev/null 2>&1 || { drain_error="failed to stop non-terminal candidate state: $status"; return 1; } + compose "$old_release" "$drain_env" stop controller >/dev/null 2>&1 || { drain_error="failed to stop non-terminal candidate state: $status"; return 1; } status= ;; esac if [[ "$status" == running ]]; then - if [[ ! -f "$rendered_env" ]]; then drain_error='cannot safely drain a running controller without its rendered environment'; return 1; fi + if [[ ! -f "$drain_env" ]]; then drain_error='cannot safely drain a running controller without its rendered environment'; return 1; fi old_release=$(current_runtime_release) + [[ -n "$old_release" ]] || old_release=$fallback_release if [[ -z "$old_release" || ! -f "$old_release/deploy/compose.yaml" ]]; then drain_error='cannot locate the running controller Compose release for safe adoption'; return 1; fi - if ! compose "$old_release" "$rendered_env" pause controller >/dev/null; then drain_error='could not pause the controller for drain'; return 1; fi + if ! compose "$old_release" "$drain_env" pause controller >/dev/null; then drain_error='could not pause the controller for drain'; return 1; fi paused=true fi deadline=$((SECONDS + ${CI_FLEET_DRAIN_TIMEOUT_SECONDS:-300})) @@ -607,7 +614,7 @@ try_drain_current() { count=$(managed_runner_count) if [[ "$count" == 0 ]]; then break; fi if ((SECONDS >= deadline)); then - if [[ "$paused" == true ]]; then compose "$old_release" "$rendered_env" unpause controller >/dev/null || true; fi + if [[ "$paused" == true ]]; then compose "$old_release" "$drain_env" unpause controller >/dev/null || true; fi drain_error="drain timed out with $count managed runner(s) still present" return 1 fi @@ -618,18 +625,18 @@ try_drain_current() { note 'DRAIN_OK managed_runners=0' return 0 fi - compose "$old_release" "$rendered_env" kill --signal SIGTERM controller >/dev/null || { - compose "$old_release" "$rendered_env" unpause controller >/dev/null 2>&1 || true + compose "$old_release" "$drain_env" kill --signal SIGTERM controller >/dev/null || { + compose "$old_release" "$drain_env" unpause controller >/dev/null 2>&1 || true drain_error='failed to signal the paused controller for graceful scale-set cleanup' return 1 } if [[ $(docker inspect --format '{{.State.Paused}}' "$controller_container" 2>/dev/null || true) == true ]]; then - compose "$old_release" "$rendered_env" unpause controller >/dev/null || { + compose "$old_release" "$drain_env" unpause controller >/dev/null || { drain_error='failed to unpause the signaled controller for graceful shutdown' return 1 } fi - compose "$old_release" "$rendered_env" stop controller >/dev/null || { + compose "$old_release" "$drain_env" stop controller >/dev/null || { drain_error='could not stop the drained controller' return 1 } @@ -718,9 +725,13 @@ restore_systemd_snapshot() { } restore_checkpoint() { - local target restored_state failed=0 + local target restored_state failed=0 checkpoint_release= [[ -n "$checkpoint_dir" && -d "$checkpoint_dir" ]] || return 1 - if ! try_drain_current true; then + if [[ -f "$checkpoint_dir/install-state.json" || -f "$checkpoint_dir/ci-fleet.env" ]]; then + load_installed_controller_identity "$checkpoint_dir/install-state.json" "$checkpoint_dir/ci-fleet.env" + fi + [[ ! -f "$checkpoint_dir/release-target" ]] || checkpoint_release=$(<"$checkpoint_dir/release-target") + if ! try_drain_current true "$checkpoint_dir/ci-fleet.env" "$checkpoint_release"; then note "ROLLBACK_FAILED reason=$drain_error" return 1 fi @@ -809,7 +820,7 @@ perform_check() { } perform_converge() { - local count existing_status + local count existing_status desired_controller_id=$controller_id if [[ "$mode" == upgrade && ! -f "$state_file" ]]; then die '--upgrade requires an existing managed installation; use --install or --adopt' fi @@ -829,7 +840,12 @@ perform_converge() { install_release make_checkpoint transaction_active=true + if [[ "$mode" == adopt ]]; then + [[ -f "$state_file" || -f "$rendered_env" ]] || die '--adopt requires a trusted installed controller identity' + load_installed_controller_identity + fi drain_current + controller_id=$desired_controller_id [[ "$target_state" == active ]] || remove_inactive_managed_runners build_candidate activate_candidate @@ -845,7 +861,7 @@ latest_checkpoint() { perform_rollback() { checkpoint_dir=$(latest_checkpoint) [[ -n "$checkpoint_dir" ]] || die 'no controller checkpoint is available' - load_installed_controller_identity "$state_file" "$rendered_env" + load_installed_controller_identity "$checkpoint_dir/install-state.json" "$checkpoint_dir/ci-fleet.env" restore_checkpoint || die 'checkpoint restoration failed' note "ROLLBACK_OK checkpoint=$checkpoint_dir" } diff --git a/scripts/test-capacity-preflight.sh b/scripts/test-capacity-preflight.sh index 12de1e91..088e3de4 100755 --- a/scripts/test-capacity-preflight.sh +++ b/scripts/test-capacity-preflight.sh @@ -234,11 +234,11 @@ if grep -Fq 'CAPACITY_TEST_SECRET_SHOULD_NOT_PRINT' <<<"$secret_output"; then fa xtrace_output=$(bash -x "$repo_root/scripts/capacity-preflight.sh" --phase post-change --target-max 2 2>&1) || fail 'xtrace secret-output fixture unexpectedly failed' if grep -Fq 'CAPACITY_TEST_SECRET_SHOULD_NOT_PRINT' <<<"$xtrace_output"; then fail 'capacity preflight exposed controller environment through inherited xtrace'; fi -for term in backup force-recreate --no-deps healthcheck retain restore rollback; do +for term in 'private desired state' --upgrade --config-repo --ref healthcheck retain restore checkpoint; do grep -Fqi -- "$term" "$repo_root/docs/CAPACITY-PROMOTION.md" || fail "capacity procedure is missing $term" done -grep -Fq 'env -i' "$repo_root/docs/CAPACITY-PROMOTION.md" || fail 'capacity procedure does not isolate Compose interpolation from stale shell values' -grep -Fq 'exec scripts/preflight.sh' "$repo_root/docs/CAPACITY-PROMOTION.md" || fail 'rollback does not run pilot preflight from a clean restored environment' +grep -Fq 'PREVIOUS_PRIVATE_CONFIGURATION_COMMIT' "$repo_root/docs/CAPACITY-PROMOTION.md" || fail 'capacity rollback does not apply the previous reviewed desired state' +if grep -Eq 'Edit only|force-recreate|ci-fleet\.env\.before-max2' "$repo_root/docs/CAPACITY-PROMOTION.md"; then fail 'capacity procedure still edits rendered host state'; fi grep -Fq 'scripts/capacity-preflight.sh' "$repo_root/docs/ADDING-A-HOST.md" || fail 'host guide does not link the capacity procedure' if grep -Riq --exclude='test-capacity-preflight.sh' 'docker system prune' "$repo_root/scripts"; then fail 'unrestricted prune exists in scripts'; fi diff --git a/scripts/test-install-worker-controller.sh b/scripts/test-install-worker-controller.sh index cd7ac019..eb7b4ff8 100755 --- a/scripts/test-install-worker-controller.sh +++ b/scripts/test-install-worker-controller.sh @@ -287,6 +287,11 @@ expect_failure 'install state must be owned by root with mode 0600' env CI_FLEET expect_failure 'DRIFT install_state' "$installer" --check "${base_args[@]}" --ref "$ref_one" expect_success "$installer" --install "${base_args[@]}" --ref "$ref_one" >/dev/null [[ $(stat -c %a "$install_state") == 600 ]] || fail 'convergence did not repair install-state mode' +rendered_env=$root/etc/ci-fleet/ci-fleet.env +chmod 644 "$rendered_env" +expect_failure 'DRIFT rendered_environment' "$installer" --check "${base_args[@]}" --ref "$ref_one" +expect_success "$installer" --install "${base_args[@]}" --ref "$ref_one" >/dev/null +[[ $(stat -c %a "$rendered_env") == 600 ]] || fail 'convergence did not repair rendered-environment mode' export FAKE_WRONG_INSTALL_STATE_OWNER=$install_state expect_failure 'install state must be owned by root with mode 0600' env CI_FLEET_INSTALL_STATE_FILE="$install_state" CI_FLEET_INSTALLER="$installer" "$repo_root/scripts/check-installed-state.sh" unset FAKE_WRONG_INSTALL_STATE_OWNER @@ -432,6 +437,7 @@ grep -Fq 'CI_FLEET_MAX_RUNNERS=2' "$root/etc/ci-fleet/ci-fleet.env" || fail 'upg mkdir -p "$root/var/lib/ci-fleet/checkpoints/99999999-incomplete" printf 'restarting\n' >"$FAKE_CONTROLLER_STATUS_FILE" +rm -f "$root/var/lib/ci-fleet/install-state.json" "$root/etc/ci-fleet/ci-fleet.env" expect_success "$installer" --rollback >/dev/null [[ ! -f "$FAKE_CONTROLLER_STATUS_FILE" ]] || fail 'explicit rollback did not recover a restarting controller' grep -Fq 'CI_FLEET_MAX_RUNNERS=1' "$root/etc/ci-fleet/ci-fleet.env" || fail 'rollback did not restore capacity one' @@ -495,15 +501,25 @@ printf '%s\n' \ 'CI_FLEET_GITHUB_APP_CLIENT_ID=Iv1.EXAMPLE' \ 'CI_FLEET_GITHUB_APP_INSTALLATION_ID=123456' \ "CI_FLEET_GITHUB_APP_PRIVATE_KEY_FILE=$adopt_pem" \ - 'CI_FLEET_RUNNER_TTL=6h' >"$adopt_root/etc/ci-fleet/ci-fleet.env" + 'CI_FLEET_RUNNER_TTL=6h' \ + 'CI_FLEET_INSTANCE=legacy-ci-01' >"$adopt_root/etc/ci-fleet/ci-fleet.env" chmod 600 "$adopt_root/etc/ci-fleet/ci-fleet.env" : >"$FAKE_DOCKER_STATE" chmod 644 "$adopt_root/etc/ci-fleet/ci-fleet.env" expect_failure 'rendered environment must be owned by root with mode 0600' "$installer" --adopt "${base_args[@]}" --ref "$ref_one" chmod 600 "$adopt_root/etc/ci-fleet/ci-fleet.env" +export FAKE_RUNNER_STATE_ONCE=$tmp/adopt-managed-runner +: >"$FAKE_RUNNER_STATE_ONCE" +: >"$FAKE_DOCKER_PS_LOG" adopt=$(expect_success "$installer" --adopt "${base_args[@]}" --ref "$ref_one") grep -Fq 'CONVERGED mode=adopt' <<<"$adopt" || fail 'adoption did not converge' [[ -f "$adopt_root/etc/ci-fleet/host.env" ]] || fail 'adoption did not separate host-local values' +grep -Fq 'label=io.randomdevelopment.ci-fleet.instance=legacy-ci-01' "$FAKE_DOCKER_PS_LOG" || fail 'adoption did not drain the installed controller instance' +unset FAKE_RUNNER_STATE_ONCE + +grep -Fq 'Issue #7' "$repo_root/docs/DESIGN-DECISIONS.md" || fail 'isolated proof approval is not recorded' +if grep -Fq '/etc/ci-fleet/ci-fleet.env.before-max2' "$repo_root/docs/CAPACITY-PROMOTION.md"; then fail 'capacity runbook still edits rendered host state'; fi +grep -Fq -- '--upgrade' "$repo_root/docs/CAPACITY-PROMOTION.md" || fail 'capacity runbook does not apply reviewed desired state through the installer' printf 'INSTALLER_TESTS_OK\n' From 058f10d36f101395c08243e710024fe3dc4bc47d Mon Sep 17 00:00:00 2001 From: Nickfosts Hermes Date: Wed, 22 Jul 2026 01:20:22 -0500 Subject: [PATCH 50/57] chore: advance final engine pins --- templates/config-repository/examples/multi-host/fleet.json | 4 ++-- templates/config-repository/fleet.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/templates/config-repository/examples/multi-host/fleet.json b/templates/config-repository/examples/multi-host/fleet.json index 661bdf02..bc7f08fb 100644 --- a/templates/config-repository/examples/multi-host/fleet.json +++ b/templates/config-repository/examples/multi-host/fleet.json @@ -24,7 +24,7 @@ "state": "active", "scale_set_name": "sample-ci-primary", "lifecycle": "stable", - "engine_ref": "8c51334def45cb97e906f663e5a6e48d1a6763b2", + "engine_ref": "b1da1101c49f6cbe67da278cdf41150b30a224e8", "min_runners": 0, "max_runners": 4, "runner_resources": { @@ -38,7 +38,7 @@ "state": "active", "scale_set_name": "sample-ci-remote", "lifecycle": "stable", - "engine_ref": "8c51334def45cb97e906f663e5a6e48d1a6763b2", + "engine_ref": "b1da1101c49f6cbe67da278cdf41150b30a224e8", "min_runners": 0, "max_runners": 2, "runner_resources": { diff --git a/templates/config-repository/fleet.json b/templates/config-repository/fleet.json index 22861864..36ac1426 100644 --- a/templates/config-repository/fleet.json +++ b/templates/config-repository/fleet.json @@ -24,7 +24,7 @@ "state": "active", "scale_set_name": "example-ci-01", "lifecycle": "experimental", - "engine_ref": "8c51334def45cb97e906f663e5a6e48d1a6763b2", + "engine_ref": "b1da1101c49f6cbe67da278cdf41150b30a224e8", "min_runners": 0, "max_runners": 1, "runner_resources": { From 4e4b49169692e9030bc0b389127fbea14623214f Mon Sep 17 00:00:00 2001 From: Nickfosts Hermes Date: Wed, 22 Jul 2026 01:34:25 -0500 Subject: [PATCH 51/57] docs: align template with accepted controller scope --- templates/config-repository/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/config-repository/README.md b/templates/config-repository/README.md index 74d92bc7..6b3e90d6 100644 --- a/templates/config-repository/README.md +++ b/templates/config-repository/README.md @@ -88,7 +88,7 @@ Use `--adopt` instead of `--install` to bring an existing controller under Git-a The command runs on the target Linux Docker machine. It validates the pinned configuration, renders host-local runtime state, preserves root-owned secrets, drains before disruptive changes, verifies a recoverable checkpoint, installs maintenance services, checks health, and reports drift without exposing credentials. OpenClaw or another agent may invoke it, but no agent is required. -The engine-side implementation is `scripts/install-worker-controller.sh` in the parent ci-fleet repository and remains experimental. This configuration repository never installs a controller by itself. +The engine-side implementation is `scripts/install-worker-controller.sh` in the parent ci-fleet repository. Its accepted scope is isolated ordinary-CI fleet hosts under reviewed schema-v3 desired state; this configuration repository never installs a controller by itself. ## Drain, retire, and delete a controller host From 47277cfc9d7f2d8e6e121b5afaa6f6045c84b412 Mon Sep 17 00:00:00 2001 From: Nickfosts Hermes Date: Wed, 22 Jul 2026 02:28:56 -0500 Subject: [PATCH 52/57] fix: verify releases and rollback candidates --- scripts/install-worker-controller.sh | 78 +++++++++++++++++++++-- scripts/test-install-worker-controller.sh | 29 ++++++++- 2 files changed, 98 insertions(+), 9 deletions(-) diff --git a/scripts/install-worker-controller.sh b/scripts/install-worker-controller.sh index 47a7c94d..dbd1e111 100755 --- a/scripts/install-worker-controller.sh +++ b/scripts/install-worker-controller.sh @@ -367,15 +367,69 @@ raise SystemExit(0 if installed == candidate else 1) PY } +release_tree_digest() { + python3 - "$1" <<'PY' +import hashlib +import os +import stat +import sys + +root = os.path.abspath(sys.argv[1]) +excluded = {".ci-fleet-engine-ref", ".ci-fleet-tree-sha256"} +digest = hashlib.sha256() + + +def add(kind, relative, mode, payload=b""): + digest.update(kind) + digest.update(b"\0") + digest.update(relative.encode("utf-8", "surrogateescape")) + digest.update(b"\0") + digest.update(f"{mode:o}".encode("ascii")) + digest.update(b"\0") + digest.update(payload) + digest.update(b"\0") + + +def visit(directory): + for entry in sorted(os.scandir(directory), key=lambda item: item.name): + relative = os.path.relpath(entry.path, root) + if relative in excluded: + continue + metadata = entry.stat(follow_symlinks=False) + mode = stat.S_IMODE(metadata.st_mode) + if stat.S_ISDIR(metadata.st_mode): + add(b"directory", relative, mode) + visit(entry.path) + elif stat.S_ISREG(metadata.st_mode): + content = hashlib.sha256() + with open(entry.path, "rb") as handle: + for block in iter(lambda: handle.read(1024 * 1024), b""): + content.update(block) + add(b"file", relative, mode, content.digest()) + elif stat.S_ISLNK(metadata.st_mode): + add(b"symlink", relative, mode, os.readlink(entry.path).encode("utf-8", "surrogateescape")) + else: + raise SystemExit(f"unsupported release entry: {relative}") + + +visit(root) +print(digest.hexdigest()) +PY +} + runtime_release_complete() { - local path=$1 expected=$2 marker required - [[ -d "$path" && -f "$path/.ci-fleet-engine-ref" && -f "$path/deploy/compose.yaml" ]] || return 1 + local path=$1 expected=$2 marker required stored_digest actual_digest + [[ -d "$path" && -f "$path/.ci-fleet-engine-ref" && -f "$path/.ci-fleet-tree-sha256" && -f "$path/deploy/compose.yaml" ]] || return 1 [[ -x "$path/scripts/preflight.sh" && -x "$path/scripts/healthcheck.sh" && -x "$path/scripts/cleanup.sh" ]] || return 1 for required in controller/Dockerfile controller/go.mod controller/main.go controller/config.go controller/scaler.go controller/state.go runner/Dockerfile; do [[ -f "$path/$required" ]] || return 1 done marker=$(<"$path/.ci-fleet-engine-ref") - [[ "$marker" == "$expected" ]] + [[ "$marker" == "$expected" ]] || return 1 + stored_digest=$(<"$path/.ci-fleet-tree-sha256") + [[ "$stored_digest" =~ ^[0-9a-f]{64}$ ]] || return 1 + actual_digest=$(release_tree_digest "$path") || return 1 + [[ "$actual_digest" == "$stored_digest" ]] } manager_release_complete() { @@ -499,6 +553,8 @@ install_release() { tar -xf "$archive" -C "$staged_release" printf '%s\n' "$engine_ref" >"$staged_release/.ci-fleet-engine-ref" chmod 0644 "$staged_release/.ci-fleet-engine-ref" + release_tree_digest "$staged_release" >"$staged_release/.ci-fleet-tree-sha256" + chmod 0644 "$staged_release/.ci-fleet-tree-sha256" runtime_release_complete "$staged_release" "$engine_ref" || die 'staged engine release is incomplete' atomic_replace_directory "$staged_release" "$release_dir" } @@ -725,16 +781,24 @@ restore_systemd_snapshot() { } restore_checkpoint() { - local target restored_state failed=0 checkpoint_release= + local target restored_state failed=0 checkpoint_release= drain_env=$rendered_env drain_release= [[ -n "$checkpoint_dir" && -d "$checkpoint_dir" ]] || return 1 - if [[ -f "$checkpoint_dir/install-state.json" || -f "$checkpoint_dir/ci-fleet.env" ]]; then + [[ ! -f "$checkpoint_dir/release-target" ]] || checkpoint_release=$(<"$checkpoint_dir/release-target") + drain_release=$(current_runtime_release) + if [[ -f "$rendered_env" ]]; then + load_installed_controller_identity "$temporary/no-install-state" "$rendered_env" + elif [[ -f "$checkpoint_dir/install-state.json" || -f "$checkpoint_dir/ci-fleet.env" ]]; then load_installed_controller_identity "$checkpoint_dir/install-state.json" "$checkpoint_dir/ci-fleet.env" + drain_env=$checkpoint_dir/ci-fleet.env + [[ -n "$drain_release" ]] || drain_release=$checkpoint_release fi - [[ ! -f "$checkpoint_dir/release-target" ]] || checkpoint_release=$(<"$checkpoint_dir/release-target") - if ! try_drain_current true "$checkpoint_dir/ci-fleet.env" "$checkpoint_release"; then + if ! try_drain_current true "$drain_env" "$drain_release"; then note "ROLLBACK_FAILED reason=$drain_error" return 1 fi + if [[ -f "$checkpoint_dir/install-state.json" || -f "$checkpoint_dir/ci-fleet.env" ]]; then + load_installed_controller_identity "$checkpoint_dir/install-state.json" "$checkpoint_dir/ci-fleet.env" + fi trap - ERR set +e if [[ -f "$checkpoint_dir/ci-fleet.env" ]]; then diff --git a/scripts/test-install-worker-controller.sh b/scripts/test-install-worker-controller.sh index eb7b4ff8..322ed56c 100755 --- a/scripts/test-install-worker-controller.sh +++ b/scripts/test-install-worker-controller.sh @@ -81,6 +81,11 @@ case "${1:-}" in case "$argument" in config|build|up|stop|pause|unpause|kill|down|logs|rm) command=$argument ;; esac previous=$argument done + if [[ -n "${FAKE_COMPOSE_LOG:-}" ]]; then + instance= + [[ ! -f "$env_file" ]] || instance=$(awk -F= '$1 == "CI_FLEET_INSTANCE" {print $2}' "$env_file") + printf '%s|%s|%s\n' "$command" "$env_file" "$instance" >>"$FAKE_COMPOSE_LOG" + fi case "$command" in up) if [[ -n "${FAKE_FAIL_UP_ONCE:-}" && -f "$FAKE_FAIL_UP_ONCE" ]]; then @@ -371,6 +376,10 @@ printf '%s\n' "$bad_marker" >"$active_release/.ci-fleet-engine-ref" expect_failure 'DRIFT engine_release' "$installer" --check "${base_args[@]}" --ref "$ref_one" expect_success "$installer" --install "${base_args[@]}" --ref "$ref_one" >/dev/null [[ $(<"$active_release/.ci-fleet-engine-ref") == "$engine_ref" ]] || fail 'bad release marker was not repaired' +printf '\n# tampered runtime fixture\n' >>"$active_release/scripts/preflight.sh" +expect_failure 'DRIFT engine_release' "$installer" --check "${base_args[@]}" --ref "$ref_one" +expect_success "$installer" --install "${base_args[@]}" --ref "$ref_one" >/dev/null +if grep -Fq 'tampered runtime fixture' "$active_release/scripts/preflight.sh"; then fail 'modified runtime release was reused'; fi rm -f "$active_release/deploy/compose.yaml" export FAKE_FAIL_TAR_ONCE=$tmp/fail-tar-once : >"$FAKE_FAIL_TAR_ONCE" @@ -380,6 +389,10 @@ unset FAKE_FAIL_TAR_ONCE if compgen -G "$root/opt/ci-fleet/releases/.${engine_ref}.staging.*" >/dev/null; then fail 'interrupted release staging was not cleaned'; fi expect_success "$installer" --install "${base_args[@]}" --ref "$ref_one" >/dev/null manager_release=$(readlink -f "$root/opt/ci-fleet/manager/current") +printf '\n# tampered manager fixture\n' >>"$manager_release/scripts/check-installed-state.sh" +expect_failure 'DRIFT maintenance_timers' "$installer" --check "${base_args[@]}" --ref "$ref_one" +expect_success "$installer" --install "${base_args[@]}" --ref "$ref_one" >/dev/null +if grep -Fq 'tampered manager fixture' "$manager_release/scripts/check-installed-state.sh"; then fail 'modified manager release was reused'; fi rm -f "$manager_release/scripts/check-installed-state.sh" expect_failure 'DRIFT maintenance_timers' "$installer" --check "${base_args[@]}" --ref "$ref_one" expect_success "$installer" --install "${base_args[@]}" --ref "$ref_one" >/dev/null @@ -492,16 +505,19 @@ unset FAKE_RUNNER_STATE_ONCE FAKE_ALL_RUNNER_STATE adopt_root=$tmp/adopt-host export CI_FLEET_ROOT_PREFIX=$adopt_root export FAKE_DOCKER_STATE=$tmp/adopt-controller-running -mkdir -p "$adopt_root/etc/ci-fleet/secrets" "$adopt_root/opt/ci-fleet/deploy" +mkdir -p "$adopt_root/etc/ci-fleet/secrets" "$adopt_root/opt/ci-fleet/deploy" "$adopt_root/opt/ci-fleet/scripts" adopt_pem=$adopt_root/etc/ci-fleet/secrets/github-app.pem printf 'fixture only\n' >"$adopt_pem" chmod 600 "$adopt_pem" cp "$repo_root/deploy/compose.yaml" "$adopt_root/opt/ci-fleet/deploy/compose.yaml" +cp "$repo_root/scripts/healthcheck.sh" "$adopt_root/opt/ci-fleet/scripts/healthcheck.sh" +chmod 0755 "$adopt_root/opt/ci-fleet/scripts/healthcheck.sh" printf '%s\n' \ 'CI_FLEET_GITHUB_APP_CLIENT_ID=Iv1.EXAMPLE' \ 'CI_FLEET_GITHUB_APP_INSTALLATION_ID=123456' \ "CI_FLEET_GITHUB_APP_PRIVATE_KEY_FILE=$adopt_pem" \ 'CI_FLEET_RUNNER_TTL=6h' \ + 'CI_FLEET_CONTROLLER_STATE=active' \ 'CI_FLEET_INSTANCE=legacy-ci-01' >"$adopt_root/etc/ci-fleet/ci-fleet.env" chmod 600 "$adopt_root/etc/ci-fleet/ci-fleet.env" : >"$FAKE_DOCKER_STATE" @@ -509,6 +525,15 @@ chmod 644 "$adopt_root/etc/ci-fleet/ci-fleet.env" expect_failure 'rendered environment must be owned by root with mode 0600' "$installer" --adopt "${base_args[@]}" --ref "$ref_one" chmod 600 "$adopt_root/etc/ci-fleet/ci-fleet.env" +export FAKE_COMPOSE_LOG=$tmp/adopt-compose.log +: >"$FAKE_COMPOSE_LOG" +export FAKE_RESTART_AFTER_UP=$tmp/adopt-restart-after-up +: >"$FAKE_RESTART_AFTER_UP" +expect_failure 'ROLLBACK_RESTORED' "$installer" --adopt "${base_args[@]}" --ref "$ref_one" +unset FAKE_RESTART_AFTER_UP +grep -Fq "stop|$adopt_root/etc/ci-fleet/ci-fleet.env|example-ci-01" "$FAKE_COMPOSE_LOG" || fail 'rollback did not drain the candidate with its rendered environment and identity' +grep -Fq 'CI_FLEET_INSTANCE=legacy-ci-01' "$adopt_root/etc/ci-fleet/ci-fleet.env" || fail 'failed adoption did not restore the installed controller identity' +: >"$FAKE_COMPOSE_LOG" export FAKE_RUNNER_STATE_ONCE=$tmp/adopt-managed-runner : >"$FAKE_RUNNER_STATE_ONCE" : >"$FAKE_DOCKER_PS_LOG" @@ -516,7 +541,7 @@ adopt=$(expect_success "$installer" --adopt "${base_args[@]}" --ref "$ref_one") grep -Fq 'CONVERGED mode=adopt' <<<"$adopt" || fail 'adoption did not converge' [[ -f "$adopt_root/etc/ci-fleet/host.env" ]] || fail 'adoption did not separate host-local values' grep -Fq 'label=io.randomdevelopment.ci-fleet.instance=legacy-ci-01' "$FAKE_DOCKER_PS_LOG" || fail 'adoption did not drain the installed controller instance' -unset FAKE_RUNNER_STATE_ONCE +unset FAKE_RUNNER_STATE_ONCE FAKE_COMPOSE_LOG grep -Fq 'Issue #7' "$repo_root/docs/DESIGN-DECISIONS.md" || fail 'isolated proof approval is not recorded' if grep -Fq '/etc/ci-fleet/ci-fleet.env.before-max2' "$repo_root/docs/CAPACITY-PROMOTION.md"; then fail 'capacity runbook still edits rendered host state'; fi From 4291b9bf5737364dfaf8ef7911725cb62417e69b Mon Sep 17 00:00:00 2001 From: Nickfosts Hermes Date: Wed, 22 Jul 2026 02:29:15 -0500 Subject: [PATCH 53/57] chore: advance reviewed release integrity pins --- templates/config-repository/examples/multi-host/fleet.json | 4 ++-- templates/config-repository/fleet.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/templates/config-repository/examples/multi-host/fleet.json b/templates/config-repository/examples/multi-host/fleet.json index bc7f08fb..ee292af7 100644 --- a/templates/config-repository/examples/multi-host/fleet.json +++ b/templates/config-repository/examples/multi-host/fleet.json @@ -24,7 +24,7 @@ "state": "active", "scale_set_name": "sample-ci-primary", "lifecycle": "stable", - "engine_ref": "b1da1101c49f6cbe67da278cdf41150b30a224e8", + "engine_ref": "47277cfc9d7f2d8e6e121b5afaa6f6045c84b412", "min_runners": 0, "max_runners": 4, "runner_resources": { @@ -38,7 +38,7 @@ "state": "active", "scale_set_name": "sample-ci-remote", "lifecycle": "stable", - "engine_ref": "b1da1101c49f6cbe67da278cdf41150b30a224e8", + "engine_ref": "47277cfc9d7f2d8e6e121b5afaa6f6045c84b412", "min_runners": 0, "max_runners": 2, "runner_resources": { diff --git a/templates/config-repository/fleet.json b/templates/config-repository/fleet.json index 36ac1426..2a181e99 100644 --- a/templates/config-repository/fleet.json +++ b/templates/config-repository/fleet.json @@ -24,7 +24,7 @@ "state": "active", "scale_set_name": "example-ci-01", "lifecycle": "experimental", - "engine_ref": "b1da1101c49f6cbe67da278cdf41150b30a224e8", + "engine_ref": "47277cfc9d7f2d8e6e121b5afaa6f6045c84b412", "min_runners": 0, "max_runners": 1, "runner_resources": { From 0b71755f4b7197da8ac53395274d53fd3ce1d667 Mon Sep 17 00:00:00 2001 From: Nickfosts Hermes Date: Wed, 22 Jul 2026 02:39:59 -0500 Subject: [PATCH 54/57] fix: satisfy rollback shell lint --- scripts/install-worker-controller.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/install-worker-controller.sh b/scripts/install-worker-controller.sh index dbd1e111..c449dff2 100755 --- a/scripts/install-worker-controller.sh +++ b/scripts/install-worker-controller.sh @@ -781,7 +781,7 @@ restore_systemd_snapshot() { } restore_checkpoint() { - local target restored_state failed=0 checkpoint_release= drain_env=$rendered_env drain_release= + local target restored_state failed=0 checkpoint_release='' drain_env=$rendered_env drain_release='' [[ -n "$checkpoint_dir" && -d "$checkpoint_dir" ]] || return 1 [[ ! -f "$checkpoint_dir/release-target" ]] || checkpoint_release=$(<"$checkpoint_dir/release-target") drain_release=$(current_runtime_release) From 59a8a7a20417f928d86c19ba33fab8a6b36f6370 Mon Sep 17 00:00:00 2001 From: Nickfosts Hermes Date: Wed, 22 Jul 2026 02:40:39 -0500 Subject: [PATCH 55/57] chore: advance lint-clean engine pins --- templates/config-repository/examples/multi-host/fleet.json | 4 ++-- templates/config-repository/fleet.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/templates/config-repository/examples/multi-host/fleet.json b/templates/config-repository/examples/multi-host/fleet.json index ee292af7..9596f270 100644 --- a/templates/config-repository/examples/multi-host/fleet.json +++ b/templates/config-repository/examples/multi-host/fleet.json @@ -24,7 +24,7 @@ "state": "active", "scale_set_name": "sample-ci-primary", "lifecycle": "stable", - "engine_ref": "47277cfc9d7f2d8e6e121b5afaa6f6045c84b412", + "engine_ref": "0b71755f4b7197da8ac53395274d53fd3ce1d667", "min_runners": 0, "max_runners": 4, "runner_resources": { @@ -38,7 +38,7 @@ "state": "active", "scale_set_name": "sample-ci-remote", "lifecycle": "stable", - "engine_ref": "47277cfc9d7f2d8e6e121b5afaa6f6045c84b412", + "engine_ref": "0b71755f4b7197da8ac53395274d53fd3ce1d667", "min_runners": 0, "max_runners": 2, "runner_resources": { diff --git a/templates/config-repository/fleet.json b/templates/config-repository/fleet.json index 2a181e99..0b773f5b 100644 --- a/templates/config-repository/fleet.json +++ b/templates/config-repository/fleet.json @@ -24,7 +24,7 @@ "state": "active", "scale_set_name": "example-ci-01", "lifecycle": "experimental", - "engine_ref": "47277cfc9d7f2d8e6e121b5afaa6f6045c84b412", + "engine_ref": "0b71755f4b7197da8ac53395274d53fd3ce1d667", "min_runners": 0, "max_runners": 1, "runner_resources": { From 254b98a10ea200e39f19bbe75ab9a822de2d5966 Mon Sep 17 00:00:00 2001 From: Nickfosts Hermes Date: Wed, 22 Jul 2026 03:27:03 -0500 Subject: [PATCH 56/57] fix: harden convergence drain lifecycle --- docs/CAPACITY-PROMOTION.md | 16 +++++++++++--- scripts/install-worker-controller.sh | 16 ++++++++------ scripts/test-capacity-preflight.sh | 2 ++ scripts/test-install-worker-controller.sh | 26 ++++++++++++++++++++++- 4 files changed, 49 insertions(+), 11 deletions(-) diff --git a/docs/CAPACITY-PROMOTION.md b/docs/CAPACITY-PROMOTION.md index cf0a7753..a1615e87 100644 --- a/docs/CAPACITY-PROMOTION.md +++ b/docs/CAPACITY-PROMOTION.md @@ -33,7 +33,12 @@ Run the current installed preflight and health checks from clean processes. Requ From the installed reviewed release, run: ```bash -scripts/capacity-preflight.sh --phase pre-change --target-max 2 +sudo env -i PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOME=/root bash -c ' + set -a + . /etc/ci-fleet/ci-fleet.env + set +a + /opt/ci-fleet/current/scripts/capacity-preflight.sh --phase pre-change --target-max 2 +' ``` Require: @@ -80,8 +85,13 @@ Require one running controller, restart count zero, one intended scale set, the Run: ```bash -scripts/capacity-preflight.sh --phase post-change --target-max 2 -scripts/healthcheck.sh +sudo env -i PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOME=/root bash -c ' + set -a + . /etc/ci-fleet/ci-fleet.env + set +a + /opt/ci-fleet/current/scripts/capacity-preflight.sh --phase post-change --target-max 2 + /opt/ci-fleet/current/scripts/healthcheck.sh +' ``` Require both to pass, plus a clean desired-state `--check` and empty instance-scoped cleanup dry-run. diff --git a/scripts/install-worker-controller.sh b/scripts/install-worker-controller.sh index c449dff2..891929f7 100755 --- a/scripts/install-worker-controller.sh +++ b/scripts/install-worker-controller.sh @@ -211,8 +211,7 @@ verify_host_files() { load_installed_controller_identity() { local source_state=${1:-$state_file} source_env=${2:-$rendered_env} expected_owner=0 [[ "$testing" != 1 ]] || expected_owner=$(id -u) - if [[ -f "$source_state" ]]; then - [[ $(stat -c %u "$source_state") == "$expected_owner" && $(stat -c %a "$source_state") == 600 ]] || die "install state must be owned by root with mode 0600: $source_state" + if [[ -f "$source_state" && $(stat -c %u "$source_state") == "$expected_owner" && $(stat -c %a "$source_state") == 600 ]]; then controller_id=$(python3 - "$source_state" <<'PY' import json import sys @@ -226,6 +225,8 @@ PY elif [[ -f "$source_env" ]]; then [[ $(stat -c %u "$source_env") == "$expected_owner" && $(stat -c %a "$source_env") == 600 ]] || die "rendered environment must be owned by root with mode 0600: $source_env" controller_id=$(awk -F= '$1 == "CI_FLEET_INSTANCE" {count++; value=substr($0, index($0, "=") + 1)} END {if (count != 1) exit 1; print value}' "$source_env") || die "installed controller identity is invalid: $source_env" + elif [[ -f "$source_state" ]]; then + die "install state must be owned by root with mode 0600: $source_state" else die 'installed controller identity is unavailable' fi @@ -639,7 +640,7 @@ make_checkpoint() { try_drain_current() { local deadline count old_release status paused=false force_nonterminal=${1:-false} - local drain_env=${2:-$rendered_env} fallback_release=${3:-} + local drain_env=${2:-$rendered_env} fallback_release=${3:-} shutdown_timeout=${CI_FLEET_DRAIN_TIMEOUT_SECONDS:-300} drain_error= status=$(controller_status) case "$status" in @@ -653,7 +654,7 @@ try_drain_current() { old_release=$(current_runtime_release) [[ -n "$old_release" ]] || old_release=$fallback_release [[ -n "$old_release" ]] || { drain_error='cannot stop a non-terminal candidate without its runtime release'; return 1; } - compose "$old_release" "$drain_env" stop controller >/dev/null 2>&1 || { drain_error="failed to stop non-terminal candidate state: $status"; return 1; } + compose "$old_release" "$drain_env" stop --timeout "$shutdown_timeout" controller >/dev/null 2>&1 || { drain_error="failed to stop non-terminal candidate state: $status"; return 1; } status= ;; esac @@ -692,7 +693,7 @@ try_drain_current() { return 1 } fi - compose "$old_release" "$drain_env" stop controller >/dev/null || { + compose "$old_release" "$drain_env" stop --timeout "$shutdown_timeout" controller >/dev/null || { drain_error='could not stop the drained controller' return 1 } @@ -904,9 +905,10 @@ perform_converge() { install_release make_checkpoint transaction_active=true - if [[ "$mode" == adopt ]]; then - [[ -f "$state_file" || -f "$rendered_env" ]] || die '--adopt requires a trusted installed controller identity' + if [[ -f "$state_file" || -f "$rendered_env" ]]; then load_installed_controller_identity + elif [[ "$mode" == adopt ]]; then + die '--adopt requires a trusted installed controller identity' fi drain_current controller_id=$desired_controller_id diff --git a/scripts/test-capacity-preflight.sh b/scripts/test-capacity-preflight.sh index 088e3de4..83ef6554 100755 --- a/scripts/test-capacity-preflight.sh +++ b/scripts/test-capacity-preflight.sh @@ -238,6 +238,8 @@ for term in 'private desired state' --upgrade --config-repo --ref healthcheck re grep -Fqi -- "$term" "$repo_root/docs/CAPACITY-PROMOTION.md" || fail "capacity procedure is missing $term" done grep -Fq 'PREVIOUS_PRIVATE_CONFIGURATION_COMMIT' "$repo_root/docs/CAPACITY-PROMOTION.md" || fail 'capacity rollback does not apply the previous reviewed desired state' +[[ $(grep -Fc '. /etc/ci-fleet/ci-fleet.env' "$repo_root/docs/CAPACITY-PROMOTION.md") -ge 2 ]] || fail 'capacity procedure does not load rendered state for both preflight phases' +grep -Fq 'env -i' "$repo_root/docs/CAPACITY-PROMOTION.md" || fail 'capacity procedure does not isolate preflight from the caller environment' if grep -Eq 'Edit only|force-recreate|ci-fleet\.env\.before-max2' "$repo_root/docs/CAPACITY-PROMOTION.md"; then fail 'capacity procedure still edits rendered host state'; fi grep -Fq 'scripts/capacity-preflight.sh' "$repo_root/docs/ADDING-A-HOST.md" || fail 'host guide does not link the capacity procedure' if grep -Riq --exclude='test-capacity-preflight.sh' 'docker system prune' "$repo_root/scripts"; then fail 'unrestricted prune exists in scripts'; fi diff --git a/scripts/test-install-worker-controller.sh b/scripts/test-install-worker-controller.sh index 322ed56c..d7c168d7 100755 --- a/scripts/test-install-worker-controller.sh +++ b/scripts/test-install-worker-controller.sh @@ -84,7 +84,7 @@ case "${1:-}" in if [[ -n "${FAKE_COMPOSE_LOG:-}" ]]; then instance= [[ ! -f "$env_file" ]] || instance=$(awk -F= '$1 == "CI_FLEET_INSTANCE" {print $2}' "$env_file") - printf '%s|%s|%s\n' "$command" "$env_file" "$instance" >>"$FAKE_COMPOSE_LOG" + printf '%s|%s|%s|%s\n' "$command" "$env_file" "$instance" "$*" >>"$FAKE_COMPOSE_LOG" fi case "$command" in up) @@ -317,6 +317,26 @@ mv "$host_config.missing" "$host_config" second=$(expect_success "$installer" --install "${base_args[@]}" --ref "$ref_one") grep -Fq 'NO_CHANGE' <<<"$second" || fail 'idempotent rerun changed the host' expect_success "$installer" --check "${base_args[@]}" --ref "$ref_one" >/dev/null +python3 - "$install_state" <<'PY' +import json +import sys +path = sys.argv[1] +value = json.load(open(path, encoding="utf-8")) +value["controller"] = "legacy-ci-01" +with open(path, "w", encoding="utf-8") as handle: + json.dump(value, handle) + handle.write("\n") +PY +python3 -c 'from pathlib import Path; import sys; path = Path(sys.argv[1]); path.write_text(path.read_text().replace("CI_FLEET_INSTANCE=example-ci-01", "CI_FLEET_INSTANCE=legacy-ci-01"))' "$rendered_env" +chmod 600 "$install_state" "$rendered_env" +[[ $(stat -c '%u:%a' "$install_state") == "$(id -u):600" ]] || fail 'installed-identity fixture metadata is invalid' +export FAKE_RUNNER_STATE_ONCE=$tmp/repeat-install-managed-runner +: >"$FAKE_RUNNER_STATE_ONCE" +: >"$FAKE_DOCKER_PS_LOG" +expect_success "$installer" --install "${base_args[@]}" --ref "$ref_one" >/dev/null +grep -Fq 'label=io.randomdevelopment.ci-fleet.instance=legacy-ci-01' "$FAKE_DOCKER_PS_LOG" || fail 'repeat install did not drain the installed controller identity' +grep -Fq 'CI_FLEET_INSTANCE=example-ci-01' "$rendered_env" || fail 'repeat install did not restore the desired controller identity' +unset FAKE_RUNNER_STATE_ONCE printf '%040d\n' 0 >"$FAKE_CONTROLLER_PROVENANCE_FILE" expect_failure 'DRIFT controller_runtime' "$installer" --check "${base_args[@]}" --ref "$ref_one" expect_success "$installer" --install "${base_args[@]}" --ref "$ref_one" >/dev/null @@ -445,7 +465,11 @@ expect_failure 'ROLLBACK_RESTORED' "$installer" --upgrade "${base_args[@]}" --re unset FAKE_RESTART_AFTER_UP [[ ! -f "$FAKE_CONTROLLER_STATUS_FILE" && -f "$FAKE_DOCKER_STATE" ]] || fail 'restarting candidate blocked checkpoint restoration' grep -Fq 'CI_FLEET_MAX_RUNNERS=1' "$root/etc/ci-fleet/ci-fleet.env" || fail 'restarting candidate rollback did not restore capacity one' +export FAKE_COMPOSE_LOG=$tmp/upgrade-compose.log +: >"$FAKE_COMPOSE_LOG" expect_success "$installer" --upgrade "${base_args[@]}" --ref "$ref_two" >/dev/null +grep -Eq 'stop\|.*\|example-ci-01\|.* stop --timeout 2 controller$' "$FAKE_COMPOSE_LOG" || fail 'controller stop did not use the explicit graceful-shutdown timeout' +unset FAKE_COMPOSE_LOG grep -Fq 'CI_FLEET_MAX_RUNNERS=2' "$root/etc/ci-fleet/ci-fleet.env" || fail 'upgrade did not apply capacity two' mkdir -p "$root/var/lib/ci-fleet/checkpoints/99999999-incomplete" From 8365b3de38a3bb5f1733fb43b413a52737bccd18 Mon Sep 17 00:00:00 2001 From: Nickfosts Hermes Date: Wed, 22 Jul 2026 03:27:35 -0500 Subject: [PATCH 57/57] chore: advance drain-safe engine pins --- templates/config-repository/examples/multi-host/fleet.json | 4 ++-- templates/config-repository/fleet.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/templates/config-repository/examples/multi-host/fleet.json b/templates/config-repository/examples/multi-host/fleet.json index 9596f270..4d7acece 100644 --- a/templates/config-repository/examples/multi-host/fleet.json +++ b/templates/config-repository/examples/multi-host/fleet.json @@ -24,7 +24,7 @@ "state": "active", "scale_set_name": "sample-ci-primary", "lifecycle": "stable", - "engine_ref": "0b71755f4b7197da8ac53395274d53fd3ce1d667", + "engine_ref": "254b98a10ea200e39f19bbe75ab9a822de2d5966", "min_runners": 0, "max_runners": 4, "runner_resources": { @@ -38,7 +38,7 @@ "state": "active", "scale_set_name": "sample-ci-remote", "lifecycle": "stable", - "engine_ref": "0b71755f4b7197da8ac53395274d53fd3ce1d667", + "engine_ref": "254b98a10ea200e39f19bbe75ab9a822de2d5966", "min_runners": 0, "max_runners": 2, "runner_resources": { diff --git a/templates/config-repository/fleet.json b/templates/config-repository/fleet.json index 0b773f5b..bf3ccea4 100644 --- a/templates/config-repository/fleet.json +++ b/templates/config-repository/fleet.json @@ -24,7 +24,7 @@ "state": "active", "scale_set_name": "example-ci-01", "lifecycle": "experimental", - "engine_ref": "0b71755f4b7197da8ac53395274d53fd3ce1d667", + "engine_ref": "254b98a10ea200e39f19bbe75ab9a822de2d5966", "min_runners": 0, "max_runners": 1, "runner_resources": {