diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 10a16176..70ab604e 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -35,11 +35,33 @@ jobs: - name: Validate public configuration template run: | + cmp -s scripts/scan_committed_secrets.py templates/config-repository/scripts/scan_committed_secrets.py python3 templates/config-repository/scripts/test_policy.py + python3 templates/config-repository/scripts/scan_committed_secrets.py templates/config-repository/scripts/validate.sh templates/config-repository/scripts/validate.sh --config templates/config-repository/examples/multi-host/fleet.json temporary_directory="$(mktemp -d)" + if templates/config-repository/scripts/init.sh \ + --organization test-company \ + --project test-app \ + --output "${temporary_directory}/missing-engine.json"; then + echo 'initializer accepted a missing engine revision' >&2 + exit 1 + fi + test ! -e "${temporary_directory}/missing-engine.json" + if templates/config-repository/scripts/init.sh \ + --organization test-company \ + --project test-app \ + --registry BAD \ + --engine-ref "$(git rev-parse HEAD)" \ + --output "${temporary_directory}/invalid.json"; then + echo 'initializer accepted an invalid registry' >&2 + exit 1 + fi + test ! -e "${temporary_directory}/invalid.json" templates/config-repository/scripts/init.sh \ --organization test-company \ --project test-app \ + --engine-ref "$(git rev-parse HEAD)" \ --output "${temporary_directory}/fleet.json" + test "$(stat -c '%a' "${temporary_directory}/fleet.json")" = 644 diff --git a/.gitignore b/.gitignore index 29eb047a..5dff1bc2 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,8 @@ _work/ runner-data/ logs/ tmp/ +__pycache__/ +*.pyc # Editors and operating systems .vscode/ diff --git a/AGENTS.md b/AGENTS.md index 00593ae8..bbd965a1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -6,7 +6,7 @@ This repository defines a portable, public, self-hosted GitHub Actions CI fleet ## Current phase -The project is in architecture definition and isolated proof-of-concept planning. Do not create or alter production deployment paths until the design decisions are recorded and the isolated proof of concept is approved. +The isolated proof of concept is approved and the project is in controlled migration. The accepted scope and remaining gates are recorded in `docs/DESIGN-DECISIONS.md`. Managed ordinary-CI controller lifecycle is permitted only through reviewed schema-v3 desired state; production deployment paths remain separately gated. ## Sources of truth @@ -41,6 +41,8 @@ Agents modifying this repository or adapting a project MUST apply these document - Do not install project language runtimes in the generic runner image. - Do not put repository names, project allowlists, or project-specific logic in runner images or host configuration. - Keep project workflows independent of host, site, instance, and scale-set names; route through shared capability labels. +- Keep worker capacity in reviewed private infrastructure configuration. Application workflows submit all independent jobs and must not use `max-parallel` to model fleet size. +- Treat schema-v3 controller desired state as authoritative for routing, capacity, lifecycle, and engine revision; host-local files contain only rendered state and credentials. - Do not allow projects to select privileged runner groups through untrusted inputs. ## Verification expectations diff --git a/README.md b/README.md index 18516e30..25d336b2 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ Use one shared pool of disposable CI workers across multiple trusted private repositories, Docker hosts, virtual machines, bare-metal computers, home labs, remote sites, or VPS providers. Projects bring their own Dockerized build and test environment; fleet hosts stay generic. -> **Status:** Experimental. The first controller host is deployed, and the first manually dispatched runner pilot is pending. The project is not production-ready. +> **Status:** Experimental. Ephemeral runner pilots and multi-job workloads have been proven, and the schema-v3 Git-authored controller installer is available for reviewed adoption testing. The project is not production-ready. ## What problem does this solve? @@ -93,7 +93,7 @@ GitHub runner-group policy decides which repositories may schedule work. A share | --- | --- | --- | | Public fleet repository | Runner image, controller, lifecycle, setup, health checks, scoped cleanup, standards, examples | Real credentials, private host inventory, production configuration | | Project repository | Test Dockerfile, services, fixtures, migrations, test plan, `scripts/ci/run.sh` | Fleet controller credentials or host-specific setup | -| Private installation configuration | Organization settings, repository authorization, host capacity, network policy, monitoring, secrets | Project runtime dependencies or test logic | +| Private installation configuration | Organization settings, repository authorization, logical controller state, capacity budgets, network policy, and required secret names | Secret values, host addresses, project runtime dependencies, or test logic | A public application can use the same fleet indirectly. Its public repository keeps pull-request validation unprivileged, while a separate private delivery repository checks out an approved immutable commit and performs protected CI, release, or deployment work. The public repository itself never receives privileged runner-group access or fleet credentials. See [Public projects, private delivery, and private configuration](docs/PUBLIC-PRIVATE-CONFIGURATION.md). @@ -137,9 +137,9 @@ Read-only validation, repository-writing releases, staging, production deploymen | Public architecture, standards, examples, and migration rules | Available | | Docker runner and controller prototype | Available | | First isolated controller host | Deployed | -| First manually dispatched private-repository pilot job | Pending | -| One-command `sudo ./setup.sh` installer | Accepted design; not implemented | -| MailThisForMe migration | Planned after pilot | +| First manually dispatched private-repository pilot job | Proven | +| Schema-v3 worker-controller installer | Available for experimental install and adoption | +| MailThisForMe migration | Parallel task-matrix validation in progress | | TF2 Recommendation Engine migration | Planned after pilot | | Reusable tester and deployer components | Planned | | Production-ready release | Not yet | @@ -169,6 +169,7 @@ Supported deployment shapes include virtual machines, dedicated physical machine | Browse every guide, concept, standard, and example | [Documentation index](docs/README.md) | | Evaluate the current prototype safely | [Live pilot runbook](docs/LIVE-PILOT.md) | | Add another Docker host or location | [How to add a host](docs/ADDING-A-HOST.md) | +| Install, adopt, check, upgrade, or remove a controller | [Git-authored controller desired state](docs/DESIRED-STATE.md) | | Add a trusted private repository | [How to add a project](docs/ADDING-A-PROJECT.md) | | Convert existing GitHub Actions CI | [Migrating existing CI](docs/MIGRATING-EXISTING-CI.md) | | Build a compatible project contract | [Project CI standard](docs/PROJECT-STANDARD.md) | @@ -236,17 +237,21 @@ Use this fleet only for explicitly trusted repositories and workflows. Keep depl Read [SECURITY.md](SECURITY.md) and the [secrets model](docs/SECRETS.md) before registration or deployment. -## Installation direction +## Install or adopt a controller -The accepted operator experience is: +After creating the GitHub App/runner group and placing the host-local identity file, run one command on the target machine with a reviewed private configuration commit: ```bash -sudo ./setup.sh +sudo ./scripts/install-worker-controller.sh \ + --install \ + --config-repo example-org/example-fleet-config \ + --ref 1111111111111111111111111111111111111111 \ + --controller example-ci-01 ``` -That command will eventually bootstrap or validate GitHub configuration, install the controller and maintenance services, verify health, and support safe reruns, repair, upgrades, and removal directly from the target host. Phone interaction should be limited to unavoidable, non-secret GitHub approval links. +Use `--adopt` for an existing manually installed controller. The same script provides `--check`, `--upgrade`, `--rollback`, and `--uninstall`. It validates schema-v3 desired state, renders only non-secret host configuration, installs the pinned engine revision, drains before replacement, records a rollback checkpoint, and installs health, scoped-cleanup, and drift timers. -**That installer is not implemented yet.** For the experimental version, follow the [live pilot runbook](docs/LIVE-PILOT.md) and [deployment prototype](docs/DEPLOYMENT-PROTOTYPE.md). +The installer remains experimental and intentionally does not create GitHub credentials or broaden runner-group access. Phone-friendly GitHub bootstrap is tracked separately. Read [Git-authored controller desired state](docs/DESIRED-STATE.md) before using it on a live host; retain the [live pilot runbook](docs/LIVE-PILOT.md) for first-job proof. ## Examples diff --git a/controller/Dockerfile b/controller/Dockerfile index 4999dc6f..a2c17990 100644 --- a/controller/Dockerfile +++ b/controller/Dockerfile @@ -8,6 +8,8 @@ ARG CI_FLEET_COMMIT=unknown RUN CGO_ENABLED=0 go build -trimpath -ldflags="-s -w -X main.version=${CI_FLEET_VERSION} -X main.commitSHA=${CI_FLEET_COMMIT}" -o /out/ci-fleet-controller . FROM debian:13.6-slim +ARG CI_FLEET_COMMIT=unknown +LABEL org.opencontainers.image.revision="${CI_FLEET_COMMIT}" RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates && rm -rf /var/lib/apt/lists/* COPY --from=build /out/ci-fleet-controller /usr/local/bin/ci-fleet-controller USER 65532:65532 diff --git a/deploy/compose.yaml b/deploy/compose.yaml index 02c3d203..d84fd5da 100644 --- a/deploy/compose.yaml +++ b/deploy/compose.yaml @@ -6,6 +6,7 @@ services: build: context: ../runner args: + CI_FLEET_COMMIT: ${CI_FLEET_COMMIT:-unknown} RUNNER_VERSION: ${CI_FLEET_RUNNER_VERSION:-2.335.1} command: ["/bin/true"] deploy: diff --git a/docs/ADDING-A-HOST.md b/docs/ADDING-A-HOST.md index 3e42e1f5..e39361b2 100644 --- a/docs/ADDING-A-HOST.md +++ b/docs/ADDING-A-HOST.md @@ -1,6 +1,6 @@ # How to add a fleet host -Use this runbook to add a Proxmox VM, physical computer, remote-site machine, or VPS to an existing ci-fleet installation. +Use this runbook to add a virtual machine, physical computer, remote-site machine, or VPS to an existing ci-fleet installation. ## Outcome @@ -73,62 +73,41 @@ Before placing credentials on the host: A remote site needs outbound connectivity; it does not need project-specific inbound access. If one location is offline, compatible jobs remain queued for another available host. -## 4. Install a pinned ci-fleet checkout +## 4. Declare the host in private Git configuration -Use a reviewed immutable commit: +Create or update one schema-v3 controller entry in the organization's private configuration repository. Give it a unique logical ID and scale-set name, assign its shared runner pool, start with reviewed capacity, and pin a full ci-fleet engine commit. -```bash -sudo install -d -m 0755 /opt/ci-fleet -# Populate /opt/ci-fleet from a reviewed ci-fleet commit. -git -C /opt/ci-fleet rev-parse HEAD -``` +Run that repository's strict validator before merge. Do not put the machine's address, VM ID, storage, backup identifier, or credential in Git. -Do not follow a moving branch for unattended controller updates. Fleet image changes use reviewed rolling replacement and keep the previous version available for rollback. +See [Git-authored controller desired state](DESIRED-STATE.md) for the complete contract. -## 5. Create host-local configuration +## 5. Prepare host-local identity -Start from `deploy/ci-fleet.env.example`: +From a reviewed ci-fleet checkout: ```bash sudo install -d -m 0700 /etc/ci-fleet/secrets -sudo install -m 0600 deploy/ci-fleet.env.example /etc/ci-fleet/ci-fleet.env +sudo install -m 0600 host/host.env.example /etc/ci-fleet/host.env +sudo install -m 0600 /secure/source/github-app.pem /etc/ci-fleet/secrets/github-app.pem ``` -Set: - -- a unique `CI_FLEET_INSTANCE`; -- a unique `CI_FLEET_SCALE_SET_NAME`; -- the fleet's shared `CI_FLEET_LABELS`; -- the shared `CI_FLEET_RUNNER_GROUP`; -- host-specific maximum concurrency and runner CPU/memory limits; -- `CI_FLEET_MIN_RUNNERS=0` so idle runners do not persist. - -Repository names do not belong in this file. +Edit `/etc/ci-fleet/host.env` locally. Prefer a separately generated GitHub App private key per host so one location can be revoked without replacing every host key. Never expose the key to runner or project containers. -Provision the GitHub App private key directly into `/etc/ci-fleet/secrets/github-app.pem` with mode `0600`. Prefer a separately generated App private key per host so one location can be revoked without replacing every host key. Never expose the key to runner or project containers. - -## 6. Validate before starting +## 6. Install or adopt with one command ```bash -cd /opt/ci-fleet -set -a -. /etc/ci-fleet/ci-fleet.env -set +a -scripts/preflight.sh -docker compose -f deploy/compose.yaml build runner-image controller -scripts/preflight.sh +sudo ./scripts/install-worker-controller.sh \ + --install \ + --config-repo example-org/example-fleet-config \ + --ref 1111111111111111111111111111111111111111 \ + --controller example-ci-01 ``` -Continue only when preflight reports `PREFLIGHT_OK warnings=0`. - -## 7. Start at zero and prove one job +Use `--adopt` when converting an existing manual controller. The installer validates the complete configuration, renders host-local runtime state, installs the pinned engine, creates a controller checkpoint, builds images, installs maintenance timers, and verifies health. -Start with maximum concurrency one even if the host will eventually run more: +The configuration repository credential, when required, must be read-only and host-side. Credentials are never accepted in command arguments. -```bash -docker compose -f deploy/compose.yaml up -d --no-deps controller -docker compose -f deploy/compose.yaml logs --tail=100 controller -``` +## 7. Prove one job Confirm no runner exists while idle. Dispatch one manual, read-only job from an authorized private project and verify: @@ -141,15 +120,15 @@ Confirm no runner exists while idle. Dispatch one manual, read-only job from an Follow [Live pilot](LIVE-PILOT.md) for the complete proof and rollback. -## 8. Enable unattended operations +## 8. Verify unattended operations -Install the health and cleanup timers from [Host maintenance](HOST-MAINTENANCE.md). Monitor controller state, disk thresholds, Docker health, last successful job, cleanup failures, and pending reboot state. +The installer enables health, scoped cleanup, and pinned desired-state drift timers. Run each service once and inspect its journal as described in [Host maintenance](HOST-MAINTENANCE.md). Monitor controller state, disk thresholds, Docker health, last successful job, cleanup failures, drift, and pending reboot state. The steady-state host should need no project-specific edits. Adding a project changes its repository, private fleet configuration, and GitHub runner-group policy—not this host. -## 9. Increase capacity safely +## 9. Increase capacity safely through Git -Raise `CI_FLEET_MAX_RUNNERS` only after measuring CPU, memory, disk, network, cache growth, collisions, cancellation, and cleanup. Keep explicit per-runner limits. +Raise the controller's schema-v3 `max_runners` and its pool `capacity_budget` through a reviewed private configuration change only after measuring CPU, memory, disk, network, cache growth, collisions, cancellation, and cleanup. Keep explicit per-runner limits. Keep the one-runner pilot preflight unchanged. For any later increase, follow [Post-pilot capacity promotion](CAPACITY-PROMOTION.md), run `scripts/capacity-preflight.sh` before and after the controller-only recreation, and retain the new maximum only when the separately authorized workload and cleanup proof satisfy the predeclared resource policy. @@ -157,12 +136,13 @@ Use another host or location when it improves failure tolerance. Give every adde ## 10. Drain, replace, or remove a host -1. Stop new capacity by setting maximum capacity to zero or stopping the controller. -2. Confirm no managed runner is active. -3. Run cleanup in dry-run mode, review candidates, then apply scoped cleanup. -4. Verify the host's unique scale set is absent from GitHub. -5. Revoke that host's GitHub App private key. -6. Securely delete host-local credentials. -7. Remove or restore the machine according to the site's recovery policy. +1. Merge a private configuration change setting the controller to `drained`. +2. Apply that exact commit with `install-worker-controller.sh --upgrade`. +3. Confirm zero managed runners and run scoped cleanup. +4. Run `install-worker-controller.sh --uninstall` when the host will not return. +5. Verify the host's unique scale set is absent from GitHub. +6. Revoke that host's GitHub App private key and explicitly remove preserved credentials. +7. Set the controller to `disabled` or remove it from private configuration. +8. Delete or repurpose the machine according to the site's recovery policy. Other hosts and project workflows continue using the shared routing label. Replacing one host must not require editing any project workflow. diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index dae470f3..e0ce9a0e 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -38,7 +38,8 @@ A host may run multiple runners when resource measurements and project isolation flowchart LR A["ci-fleet"] --> B["Runner lifecycle"] C["Project repo"] --> D["Test environment"] - E["Host config"] --> F["Secrets and capacity"] + E["Private Git config"] --> F["Routing and capacity desired state"] + G["Host-local config"] --> H["Credentials and rendered runtime state"] ``` ### Fleet repository @@ -71,19 +72,19 @@ Each project is responsible for: - run-scoped cleanup; - release and deployment behavior. -### Deployment-local configuration +### Private installation configuration Each installation is responsible for: - real organization and runner-group names; -- host capacity; +- logical controller identities and capacity budgets; +- pinned controller engine revisions; - private network rules; -- secret provisioning; -- host inventory; +- required secret names; - monitoring destinations; - maintenance windows. -Deployment-local values must not be committed to this public repository. +These secret-free values belong in a reviewed private schema-v3 configuration repository. Machine addresses, VM IDs, storage and backup identifiers, credentials, secret values, and rendered runtime files remain host-local or in an external secret manager. See [Git-authored controller desired state](DESIRED-STATE.md). ## Target runner lifecycle @@ -127,7 +128,7 @@ The same fleet interface should support: - one Docker host with one or more runners; - several Docker hosts with one or more runners each; -- Proxmox virtual machines; +- virtual machines; - dedicated physical computers; - remote-site computers; - VPS infrastructure. @@ -152,6 +153,7 @@ Host-wide pruning must not run as an uncoordinated per-job operation. Hard cance - Host security updates are applied automatically. - Reboots drain runner capacity before interrupting the host. - Fleet updates use rolling replacement and an explicit rollback version. +- Controller routing, capacity, lifecycle, and engine updates use reviewed full commits from private desired-state configuration; hosts never follow an unreviewed moving branch. ## Project adoption diff --git a/docs/CAPACITY-PROMOTION.md b/docs/CAPACITY-PROMOTION.md index 8d1bc507..a1615e87 100644 --- a/docs/CAPACITY-PROMOTION.md +++ b/docs/CAPACITY-PROMOTION.md @@ -2,158 +2,118 @@ Use this procedure only after the strict one-runner pilot has passed. It validates and changes one already isolated controller; it does not authorize a project workflow by itself. -`scripts/preflight.sh` remains the initial pilot gate and accepts only `MIN=0`, `MAX=1`. `scripts/capacity-preflight.sh` is a separate, read-only post-pilot gate. It never edits host configuration. +Schema-v3 private desired state owns capacity. Never edit `/etc/ci-fleet/ci-fleet.env` directly: it is rendered state and the drift checker will replace local changes. ## Capacity policy Declare the policy before observing a larger workload: -- requested MAX is explicitly and exclusively two for this first post-pilot procedure; any other target is rejected; +- requested MAX is exactly two for this first post-pilot procedure; - `MIN` remains zero; -- the configured instance, scale set, routing label, runner group, Docker socket group, and per-runner limits must equal the running controller's effective values; +- both the controller `max_runners` and pool `capacity_budget` change in one reviewed private configuration PR; +- the configured instance, scale set, routing label, runner group, Docker socket group, and per-runner limits remain unchanged; - no managed runner, project job resource, unrelated running container, controller OOM, or current-boot kernel OOM evidence may exist; -- Docker filesystem use must be below `CI_FLEET_DISK_WARN_PERCENT` (80 by default); -- reserve the controller's Compose limit: 1 CPU and 512 MiB; -- reserve 1 CPU and 1 GiB for Docker overhead; -- reserve for the operating system the greater of 1 CPU or 15% of logical CPU capacity; -- reserve for the operating system the greater of 2 GiB or 20% of physical memory; -- admit `target MAX × CI_FLEET_RUNNER_CPUS` and `target MAX × CI_FLEET_RUNNER_MEMORY_MIB` only when those allocations plus all reserves fit; +- Docker filesystem use stays below `CI_FLEET_DISK_WARN_PERCENT` (80 by default); +- reserve 1 CPU and 512 MiB for the controller, 1 CPU and 1 GiB for Docker overhead, and the greater of 1 CPU/15% plus 2 GiB/20% for the operating system; +- admit `target MAX × CI_FLEET_RUNNER_CPUS` and `target MAX × CI_FLEET_RUNNER_MEMORY_MIB` only when the allocation and reserves fit; and - require currently available memory to cover all target runners plus controller and Docker reservations. -Runner limits are controller admission inputs. Project containers use the host Docker daemon as siblings of the runner container, so the separately authorized live proof must still observe whole-host CPU, memory, disk, collision, and cleanup behavior. +Project containers use the host Docker daemon as siblings of the runner container. The separately authorized live proof must therefore observe whole-host CPU, memory, disk, collisions, and cleanup. -During a live proof, retain the target only if every five-second sample keeps CPU busy below 85%, available memory at or above the greater of 2 GiB or 20% of total memory, and Docker filesystem use below 80%. Any OOM, unrelated workload, controller/Docker failure, third runner, observer gap, or cleanup residue requires restoration. +During that proof, retain MAX=2 only if every five-second sample keeps CPU busy below 85%, available memory at or above the greater of 2 GiB or 20% of total memory, and Docker filesystem use below 80%. Any OOM, unrelated workload, controller/Docker failure, third runner, observer gap, or cleanup residue requires restoration. -## 1. Verify idle pilot state +## 1. Gate dispatch and verify the one-runner state -Gate all dispatches that can target this controller. Confirm no queued, assigned, or running fleet job and no instance-owned runner in any state. Keep the dispatch gate closed until post-change verification completes. +Block dispatches that can target this controller. Confirm no queued, assigned, or running fleet job and no instance-owned runner in any state. Keep dispatch gated through post-change verification. -From the reviewed checkout, load the root-only host configuration without tracing or printing it: - -```bash -set -Eeuo pipefail -set +x -cd /opt/ci-fleet -set -a -. /etc/ci-fleet/ci-fleet.env -set +a -scripts/preflight.sh -``` - -Require exactly `PREFLIGHT_OK warnings=0`. This proves the original `MIN=0`, `MAX=1` pilot contract remains valid. +Run the current installed preflight and health checks from clean processes. Require the existing one-runner contract, controller health, and an empty instance-scoped cleanup dry-run. ## 2. Validate target capacity without changing it -```bash -scripts/capacity-preflight.sh --phase pre-change --target-max 2 -``` - -Require `CAPACITY_PREFLIGHT_OK phase=pre-change target_max=2 configured_max=1 effective_max=1`. Record only the safe budget summaries. - -## 3. Create one protected backup - -Use one UTC timestamp and refuse to overwrite an existing path: +From the installed reviewed release, run: ```bash -backup_dir=/etc/ci-fleet/backups -stamp=$(date -u +%Y%m%dT%H%M%SZ) -backup="$backup_dir/ci-fleet.env.before-max2.$stamp" -install -d -o root -g root -m 0700 "$backup_dir" -test ! -e "$backup" -install -o root -g root -m 0600 /etc/ci-fleet/ci-fleet.env "$backup" -cmp -s /etc/ci-fleet/ci-fleet.env "$backup" -sha256sum "$backup" | cut -d' ' -f1 +sudo env -i PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOME=/root bash -c ' + set -a + . /etc/ci-fleet/ci-fleet.env + set +a + /opt/ci-fleet/current/scripts/capacity-preflight.sh --phase pre-change --target-max 2 +' ``` -Record the path and checksum only. Never print or copy the backup contents, and do not delete the backup during the proof. - -## 4. Change one exact setting - -Before editing, require exactly one active assignment and the pilot value: +Require: -```bash -test "$(grep -c '^CI_FLEET_MAX_RUNNERS=' /etc/ci-fleet/ci-fleet.env)" -eq 1 -grep -qx 'CI_FLEET_MAX_RUNNERS=1' /etc/ci-fleet/ci-fleet.env +```text +CAPACITY_PREFLIGHT_OK phase=pre-change target_max=2 configured_max=1 effective_max=1 ``` -Edit only that line to `CI_FLEET_MAX_RUNNERS=2`. Then verify without printing a diff: +Record only the safe budget summary. -```bash -test "$(stat -c '%u:%g:%a' /etc/ci-fleet/ci-fleet.env)" = 0:0:600 -test "$(grep -c '^CI_FLEET_MAX_RUNNERS=' /etc/ci-fleet/ci-fleet.env)" -eq 1 -grep -qx 'CI_FLEET_MAX_RUNNERS=2' /etc/ci-fleet/ci-fleet.env -cmp -s \ - <(grep -v '^CI_FLEET_MAX_RUNNERS=' "$backup") \ - <(grep -v '^CI_FLEET_MAX_RUNNERS=' /etc/ci-fleet/ci-fleet.env) -env -i PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOME=/root \ - docker compose --env-file /etc/ci-fleet/ci-fleet.env -f deploy/compose.yaml config --quiet -``` +## 3. Review and merge private desired state -Do not run the pilot preflight against MAX=2; it must continue to reject that value. Every Compose command below uses `env -i` so stale values exported when the pilot file was sourced cannot override the explicit `--env-file` during promotion or rollback. +In the secret-free private configuration repository: -## 5. Recreate only the controller +1. raise only the selected controller's `max_runners` from one to two; +2. raise its pool `capacity_budget` only as needed to admit that reviewed controller maximum; +3. keep `min_runners`, runner resources, identity, lifecycle, routing, trust, and engine pin unchanged; +4. run the complete strict validator, policy tests, and committed-secret scan; +5. obtain the repository's configured exact-head review and CI gates; and +6. merge normally. -Reconfirm zero runners and zero jobs immediately before stopping. Stop only the controller with enough grace for scale-set deletion: +Record the previous and new full private configuration commits. Do not edit a rendered host file or bypass the desired-state PR. -```bash -env -i PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOME=/root \ - docker compose --env-file /etc/ci-fleet/ci-fleet.env -f deploy/compose.yaml stop -t 60 controller -``` +## 4. Apply the exact merged configuration -Verify the exact old scale set is absent and there is no runner before starting the replacement. Do not delete an apparent duplicate until ownership and zero active jobs are proven. +Reconfirm the idle gates, then apply the exact merged private commit through the installed manager: ```bash -env -i PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOME=/root \ - docker compose --env-file /etc/ci-fleet/ci-fleet.env -f deploy/compose.yaml \ - up -d --no-deps --force-recreate --timeout 60 controller +sudo /opt/ci-fleet/manager/current/scripts/install-worker-controller.sh \ + --upgrade \ + --config-repo ORGANIZATION/PRIVATE-CONFIGURATION \ + --ref NEW_PRIVATE_CONFIGURATION_COMMIT \ + --controller CONTROLLER_ID ``` -Do not recreate `runner-image`, remove volumes, rebuild images, or touch unrelated resources. +The installer must create a protected checkpoint, drain the selected instance, validate the managed target, change only the selected controller, start it, and pass health before reporting convergence. A failed activation must restore the checkpoint and keep dispatch closed. -## 6. Verify effective state +Do not recreate runner jobs, remove volumes, or touch unrelated Docker resources. -Wait boundedly for one sanitized `controller ready` record reporting `minRunners=0` and `maxRunners=2`. Require one running controller, restart count zero, one intended scale set, the unchanged experimental routing label and runner group, and no idle runner. +## 5. Verify effective state -Reload the host configuration and run: +Require one running controller, restart count zero, one intended scale set, the unchanged routing label and runner group, no idle runner, and the exact reviewed MAX=2. + +Run: ```bash -scripts/capacity-preflight.sh --phase post-change --target-max 2 -scripts/healthcheck.sh +sudo env -i PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOME=/root bash -c ' + set -a + . /etc/ci-fleet/ci-fleet.env + set +a + /opt/ci-fleet/current/scripts/capacity-preflight.sh --phase post-change --target-max 2 + /opt/ci-fleet/current/scripts/healthcheck.sh +' ``` -Require both to pass. The post-change preflight compares configured state with a filtered set of effective controller values and never prints arbitrary container environment entries. +Require both to pass, plus a clean desired-state `--check` and empty instance-scoped cleanup dry-run. -## 7. Run one separately authorized proof +## 6. Run one separately authorized proof Start bounded runner, task-job, project-resource, and host-metric observers before dispatch. Dispatch exactly the approved workload once. Do not retry a failed proof and do not raise MAX above two. -Observe runner creation/destruction, actual two-way overlap, no third runner, whole-host CPU/memory/disk thresholds, Docker/controller health, exact project identity, and automatic cleanup. Run the instance-scoped cleanup dry-run and healthcheck after all jobs terminate. - -## 8. Retain or restore (rollback) - -Retain MAX=2 only when the authorized workload succeeds, actual two-way job and runner overlap is proven, every predeclared resource threshold passes, no manual cleanup is required, all runner/project residue is zero, post-change capacity preflight passes, cleanup dry-run is empty, and healthcheck passes. +Observe runner creation/destruction, actual two-way overlap, no third runner, whole-host resource thresholds, Docker/controller health, exact project identity, and automatic cleanup. Repeat health, drift, and cleanup dry-run checks after all jobs terminate. -On any failure, keep dispatch gated, wait for exact job termination, and restore the backup: +## 7. Retain or restore -```bash -env -i PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOME=/root \ - docker compose --env-file /etc/ci-fleet/ci-fleet.env -f deploy/compose.yaml stop -t 60 controller -install -o root -g root -m 0600 "$backup" /etc/ci-fleet/ci-fleet.env -cmp -s "$backup" /etc/ci-fleet/ci-fleet.env -env -i PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOME=/root \ - docker compose --env-file /etc/ci-fleet/ci-fleet.env -f deploy/compose.yaml config --quiet -env -i PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOME=/root \ - docker compose --env-file /etc/ci-fleet/ci-fleet.env -f deploy/compose.yaml \ - up -d --no-deps --force-recreate --timeout 60 controller -``` +Retain MAX=2 only when every predeclared gate passes and no manual cleanup is required. -Verify a sanitized ready record with `MIN=0`, `MAX=1`, then run the pilot preflight and healthcheck from clean processes that source only the restored file: +On any failure, keep dispatch gated and apply the recorded previous private configuration commit through the same reviewed `--upgrade` path: ```bash -env -i PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOME=/root \ - bash -c 'set -a; . /etc/ci-fleet/ci-fleet.env; set +a; exec scripts/preflight.sh' -env -i PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOME=/root \ - bash -c 'set -a; . /etc/ci-fleet/ci-fleet.env; set +a; exec scripts/healthcheck.sh' +sudo /opt/ci-fleet/manager/current/scripts/install-worker-controller.sh \ + --upgrade \ + --config-repo ORGANIZATION/PRIVATE-CONFIGURATION \ + --ref PREVIOUS_PRIVATE_CONFIGURATION_COMMIT \ + --controller CONTROLLER_ID ``` -Run the instance-scoped cleanup dry-run. Require zero runners, zero jobs, one intended scale set, no duplicate controller, and no residue before reopening dispatch. +Require the restored MAX=1, clean desired-state check, health pass, zero runners/jobs/residue, one intended scale set, and empty instance-scoped cleanup dry-run before reopening dispatch. Preserve installer checkpoint and proof evidence without recording credential values. diff --git a/docs/DESIGN-DECISIONS.md b/docs/DESIGN-DECISIONS.md new file mode 100644 index 00000000..05ab5d51 --- /dev/null +++ b/docs/DESIGN-DECISIONS.md @@ -0,0 +1,16 @@ +# Accepted design decisions + +## Managed controller desired state after the isolated proof + +**Status:** accepted on 2026-07-22 for isolated CI fleet hosts. + +The isolated one-job proof required by [Issue #7](https://github.com/RandomDevelopment/ci-fleet/issues/7) passed on 2026-07-15. It proved the organization-owned App boundary, selected private runner group, `MIN=0` / `MAX=1` controller, read-only job permissions, one-job ephemeral runner lifecycle, scoped cleanup, zero final job residue, controller health, and preservation of existing project runners. + +[Issue #32](https://github.com/RandomDevelopment/ci-fleet/issues/32) and its reviewed implementation in PR #33 accept schema-v3 Git-authored controller desired state as the next migration phase. The managed installer may install, adopt, check, upgrade, roll back, or uninstall an isolated **ordinary-CI controller** only from: + +- a merged, publicly reachable `RandomDevelopment/ci-fleet` engine commit; +- a merged, secret-free private configuration commit; +- host-local root-owned credentials and identity; and +- a recoverable checkpoint with the documented health, drain, drift, and rollback gates. + +This decision does not authorize application production deployment, privileged delivery on ordinary-CI runners, unreviewed capacity increases, public-repository runner access, unrestricted Docker cleanup, legacy-runner retirement, or VM deletion. Those remain separately gated by repository policy and operator approval. diff --git a/docs/DESIRED-STATE.md b/docs/DESIRED-STATE.md new file mode 100644 index 00000000..3f5bd9d8 --- /dev/null +++ b/docs/DESIRED-STATE.md @@ -0,0 +1,156 @@ +# Git-authored controller desired state + +Schema v3 makes a reviewed private configuration repository the authority for controller routing, capacity, lifecycle, and engine revision within the fixed `RandomDevelopment/ci-fleet` public engine repository. A configuration-only change cannot redirect root execution to another repository. The target host keeps credentials and rendered runtime state outside Git. + +## Responsibility boundary + +| Location | Owns | +| --- | --- | +| Public ci-fleet repository | Schema, validator, renderer, installer, controller, maintenance, and fictional examples | +| Private configuration repository | Real project allowlists, runner pools, logical controller IDs, capacity budgets, controller state, resource limits, and pinned engine revisions | +| Root-owned host files | GitHub App identity, private key path, rendered environment, checkpoints, and installation state | +| Application repository | Independent CI tasks and shards; never fleet size or controller identity | + +Host addresses, VM IDs, storage names, backup identifiers, SSH details, tokens, private keys, and rendered `.env` files are rejected from the Git-authored configuration. + +## Schema v3 + +Each runner pool declares: + +- a logical GitHub runner group; +- stable shared routing labels; +- explicitly allowed repositories; +- `public_repositories: false`; +- an infrastructure `capacity_budget`; +- `job_submission_policy: all-independent-jobs`. + +Each controller has a unique object key and declares: + +- its pool and logical location; +- `active`, `drained`, or `disabled` state; +- a unique scale-set name; +- `experimental`, `stable`, or `retiring` lifecycle; +- a full pinned ci-fleet engine commit; +- a zero managed minimum and reviewed maximum runner capacity; +- CPU cores and memory per ephemeral runner. + +Active and drained controllers reserve their configured maximum against the pool budget. A drained controller has zero effective runtime capacity but keeps its reservation, so an undrain cannot silently overcommit the pool. Disabled controllers reserve no capacity. + +Managed prewarmed runners are not currently supported: `min_runners` is fixed at zero in schema, semantic validation, rendering, and preflight. This keeps idle privileged workers absent and prevents reviewed configuration from passing validation only to fail host adoption. + +The authoritative fictional contract is in [`templates/config-repository`](../templates/config-repository/README.md). + +## Application parallelism rule + +Application workflows submit every independent task and shard. They must not use GitHub Actions `strategy.max-parallel` to represent how many fleet workers exist. GitHub queues excess jobs; private infrastructure policy decides how many run simultaneously. + +An application may use a concurrency limit only for a documented external-system restriction such as a vendor rate limit or a single-writer test fixture. That exception must not be based on current worker count. + +## Prepare host-local identity + +Create the root-owned file once: + +```bash +sudo install -d -m 0700 /etc/ci-fleet/secrets +sudo install -m 0600 host/host.env.example /etc/ci-fleet/host.env +sudo install -m 0600 /secure/source/github-app.pem /etc/ci-fleet/secrets/github-app.pem +``` + +Edit `/etc/ci-fleet/host.env` locally. Managed installs intentionally reject alternate host-config paths so scheduled drift checks always verify the same identity file. It contains only the GitHub App client ID, installation ID, private-key path, and runner TTL. Both the host environment and PEM must be root-owned mode `0600`, and the TTL must be at least one hour. Neither file is committed, and the PEM is never printed by the installer. + +GitHub App and runner-group creation remain the bootstrap responsibility tracked by [issue #27](https://github.com/RandomDevelopment/ci-fleet/issues/27). The installer fails closed when those prerequisites are absent. + +## Install a fresh controller + +Run the command from a reviewed checkout of ci-fleet on the target Linux Docker machine: + +```bash +sudo ./scripts/install-worker-controller.sh \ + --install \ + --config-repo example-org/example-fleet-config \ + --ref 1111111111111111111111111111111111111111 \ + --controller example-ci-01 +``` + +`--ref` must be a full configuration commit SHA. The installer never follows a moving branch. For a private remote repository, configure a narrowly scoped read-only Git credential on the host before running the command. Do not embed credentials in the URL or command line. A local pinned Git checkout is also accepted. + +The installer: + +1. fetches only the requested configuration commit; +2. validates the complete schema and capacity relationships; +3. selects exactly one logical controller; +4. renders `/etc/ci-fleet/ci-fleet.env` without secret values; +5. fetches and verifies the pinned public engine commit; +6. creates a root-only controller checkpoint; +7. drains the current controller and waits for every managed runner to finish, including orphaned runners left after a stopped or crashed controller; +8. runs managed preflight and builds the pinned runner and controller images; +9. installs health, cleanup, and pinned-state drift unit definitions; +10. starts the controller only when its desired state is active and verifies runtime health; +11. atomically records redacted installation state, then enables the maintenance timers. + +A successful second `--install` run reports `NO_CHANGE` and performs no unnecessary replacement. A successful engine upgrade advances both the runtime release and the maintenance installer-manager to the same pinned commit; rollback restores both. + +## Adopt an existing controller + +Use adoption for a manually installed controller: + +```bash +sudo ./scripts/install-worker-controller.sh \ + --adopt \ + --config-repo example-org/example-fleet-config \ + --ref 1111111111111111111111111111111111111111 \ + --controller example-ci-01 +``` + +If `/etc/ci-fleet/host.env` does not exist, adoption extracts only the approved host identity fields from the existing root-owned `/etc/ci-fleet/ci-fleet.env`. It never copies project settings or secret values into Git. The running controller is paused, existing managed runners are allowed to finish, and activation proceeds only after the host is idle. + +## Check, upgrade, roll back, and remove + +```bash +# Read-only comparison with one pinned configuration commit +sudo ./scripts/install-worker-controller.sh --check \ + --config-repo example-org/example-fleet-config \ + --ref 1111111111111111111111111111111111111111 \ + --controller example-ci-01 + +# Apply a newer reviewed configuration and/or pinned engine +sudo ./scripts/install-worker-controller.sh --upgrade \ + --config-repo example-org/example-fleet-config \ + --ref 2222222222222222222222222222222222222222 \ + --controller example-ci-01 + +# Restore the latest root-only controller checkpoint +sudo ./scripts/install-worker-controller.sh --rollback + +# Drain and remove controller services and rendered state +sudo ./scripts/install-worker-controller.sh --uninstall +``` + +Uninstall removes the controller, timers, rendered environment, and active installation marker. It deliberately preserves `/etc/ci-fleet/host.env`, the secrets directory, and checkpoints so credential destruction and audit retention remain explicit operator decisions. + +No mode uses global Docker prune or removes unrelated workloads. + +## Drift and reviewed updates + +`ci-fleet-drift.timer` checks the host every fifteen minutes against the exact configuration SHA recorded at installation. It detects host edits, missing or stale runtime and installer-manager releases, runtime-state mismatch, altered metadata, and missing maintenance timers without applying changes. + +The host does not automatically follow or execute a moving branch. A new configuration becomes effective only when an operator or authorized external controller supplies its reviewed full commit SHA to `--upgrade`. Automatic dispatchers may watch a protected branch and invoke that exact command after merge, using read-only repository contents permission; their identity must remain host-side and unavailable to job runners. + +## Drain and retirement + +1. Merge a private configuration change setting the controller to `drained`. +2. Run `--upgrade` with that reviewed commit. +3. Verify zero managed runners and zero effective capacity. +4. Remove the controller's GitHub registration and revoke its host identity when it will not return. +5. Set it to `disabled` or remove its declaration in a later reviewed change. +6. Delete or repurpose the machine according to the installation's infrastructure policy. + +Legacy project-specific hosts remain until CI, promotion, and deployment no longer reference them. Deleting or replacing a generic controller does not require application workflow changes because projects route through shared labels. + +## Failure and recovery behavior + +Before mutation, the installer records the prior rendered environment, installation metadata, runtime release, installer-manager release, and maintenance unit/timer state under `/var/lib/ci-fleet/checkpoints`. Each checkpoint is staged and atomically renamed with a completion marker; rollback ignores partial staging directories. Build and validation happen before the active release changes. A failed activation or health check drains the candidate, restores those artifacts, restarts the prior controller only when no managed runner is active, and verifies prior-release health before reporting rollback success. A host-local installer lock serializes every check and mutation. Runtime and installer-manager releases are staged on their respective target filesystems and renamed atomically so a failed copy cannot masquerade as an installed immutable release. + +These controller checkpoints do not replace machine backups. Operators still create and verify VM snapshots, physical-host recovery media, or equivalent infrastructure backups according to their local policy. + +This contract implements the engine portions of [issue #32](https://github.com/RandomDevelopment/ci-fleet/issues/32) and integrates the installer, documentation, phone-first bootstrap, and capacity work tracked by [#21](https://github.com/RandomDevelopment/ci-fleet/issues/21), [#24](https://github.com/RandomDevelopment/ci-fleet/issues/24), [#27](https://github.com/RandomDevelopment/ci-fleet/issues/27), and [#30](https://github.com/RandomDevelopment/ci-fleet/issues/30). diff --git a/docs/HOST-MAINTENANCE.md b/docs/HOST-MAINTENANCE.md index 8e1faf28..72e0a145 100644 --- a/docs/HOST-MAINTENANCE.md +++ b/docs/HOST-MAINTENANCE.md @@ -9,6 +9,7 @@ Fleet hosts are generic Docker infrastructure. They must receive operating-syste - Schedule reboots only after draining the controller. - Run health checks every five minutes. - Run scoped cleanup daily. +- Compare installed state with its pinned Git configuration every fifteen minutes. - Never schedule `docker system prune`. - Alert before Docker storage reaches 80%; treat 90% as critical. @@ -25,25 +26,22 @@ sudo unattended-upgrade --dry-run --debug Review `/etc/apt/apt.conf.d/50unattended-upgrades` and confirm only the intended Debian security origins are enabled. Keep automatic reboot disabled; a generic CI host may still be executing a job when a package requests reboot. -## Install fleet timers +## Fleet timers -After the repository is installed at `/opt/ci-fleet` and `/etc/ci-fleet/ci-fleet.env` exists: +`scripts/install-worker-controller.sh` installs and enables all three timer pairs: -```bash -sudo install -m 0644 host/systemd/ci-fleet-health.service /etc/systemd/system/ -sudo install -m 0644 host/systemd/ci-fleet-health.timer /etc/systemd/system/ -sudo install -m 0644 host/systemd/ci-fleet-cleanup.service /etc/systemd/system/ -sudo install -m 0644 host/systemd/ci-fleet-cleanup.timer /etc/systemd/system/ -sudo systemctl daemon-reload -sudo systemctl enable --now ci-fleet-health.timer ci-fleet-cleanup.timer -``` +- `ci-fleet-health.timer` checks Docker, disk, and the controller's desired runtime state; +- `ci-fleet-cleanup.timer` removes only expired inactive fleet-owned resources; +- `ci-fleet-drift.timer` compares the installation with the exact pinned configuration commit without applying changes. Run each service manually once before relying on its timer: ```bash sudo systemctl start ci-fleet-health.service +sudo systemctl start ci-fleet-drift.service sudo journalctl -u ci-fleet-health.service --since today -sudo /opt/ci-fleet/scripts/cleanup.sh +sudo journalctl -u ci-fleet-drift.service --since today +sudo /opt/ci-fleet/current/scripts/cleanup.sh sudo systemctl start ci-fleet-cleanup.service ``` @@ -51,13 +49,15 @@ The manual cleanup command is intentionally a dry-run. Enable the applying servi ## Reboot procedure -1. Stop new capacity by stopping the controller. +1. Merge and apply a reviewed desired-state change setting the controller to `drained`, or otherwise pause the controller and wait for zero managed runners. 2. Confirm no managed runner container is active. 3. Apply updates and reboot. 4. Confirm Docker, disk, time synchronization, DNS, and outbound GitHub connectivity. -5. Run `scripts/preflight.sh`. -6. Start the controller and confirm `MIN=0` produces no idle container. +5. Run `scripts/install-worker-controller.sh --check` against the installed configuration commit. +6. Apply a reviewed `active` desired-state commit and confirm `MIN=0` produces no idle container. ## Dependency maintenance Dependabot proposes updates for GitHub Actions, Go modules, and both Dockerfiles. Those pull requests must pass inert validation and be reviewed before merge. This preserves unattended host security patching without silently changing the runner control plane. + +Controller and runner engine updates are also pinned. A merged private configuration change is applied with `install-worker-controller.sh --upgrade`; the host never follows a moving engine or configuration branch automatically. See [Git-authored controller desired state](DESIRED-STATE.md). diff --git a/docs/PROJECT-STANDARD.md b/docs/PROJECT-STANDARD.md index 24939b02..1e4e833b 100644 --- a/docs/PROJECT-STANDARD.md +++ b/docs/PROJECT-STANDARD.md @@ -50,6 +50,12 @@ Ordinary CI MUST target a wall-clock duration of five minutes or less when suffi Forty-five test-minutes divided among nine workers is a theoretical five-minute lower bound. Real plans generally need more than nine shards because job setup consumes part of the ceiling and test work is not perfectly balanced. Adding workers reduces wall-clock time only while runnable shards remain queued. +## Infrastructure-owned capacity + +Application workflows MUST submit every independent task and shard and MUST NOT use `strategy.max-parallel` to represent the number of available fleet workers. Excess jobs remain safely queued by GitHub. Pool budgets and controller maxima belong in the private schema-v3 infrastructure configuration described in [Git-authored controller desired state](DESIRED-STATE.md). + +A project MAY limit concurrency for a documented external-system restriction, such as an API rate limit or a single-writer fixture, only when that limit is independent of fleet size. Workflow-level concurrency groups MAY still cancel obsolete runs or serialize operations that are inherently unsafe to overlap. + ## Host independence A project CI job: @@ -109,7 +115,8 @@ Ordinary CI: - MUST NOT receive deployment, release, production, or internal-network credentials; - MUST NOT push branches, tags, releases, packages, or commits; - MUST set `timeout-minutes: 5` on every ordinary task-matrix job; -- SHOULD use concurrency controls appropriate to the project; +- SHOULD use concurrency groups to cancel obsolete runs or protect inherently single-writer operations; +- MUST NOT use concurrency settings to encode current fleet capacity; - MUST treat pull-request code as untrusted unless repository policy explicitly establishes otherwise. A job requiring write permission is not ordinary CI. It MUST be a separate job or workflow routed to an appropriate privileged runner group. diff --git a/docs/PUBLIC-PRIVATE-CONFIGURATION.md b/docs/PUBLIC-PRIVATE-CONFIGURATION.md index 12af79fd..32404da7 100644 --- a/docs/PUBLIC-PRIVATE-CONFIGURATION.md +++ b/docs/PUBLIC-PRIVATE-CONFIGURATION.md @@ -40,7 +40,7 @@ A private application repository may use the restricted runner group directly. A | Public application repository | Source code, public tests, public Dockerfiles, and unprivileged workflows | | Private application repository | Source code and workflows authorized to use an appropriate private runner group | | Private delivery repository for a public project | Approved source mappings, CI and deployment workflows, environment policy, promotion rules, and required secret names | -| Private organization configuration | Repository mappings, logical host groups, environment policy, capacity, image names, and internal operating notes | +| Private organization configuration | Repository mappings, logical controllers and host groups, environment policy, capacity budgets, pinned engine revisions, image names, and required secret names | | Secret manager, GitHub environment, or host-local protected file | Actual keys, tokens, passwords, and deployment credentials | Each organization generates its own private installation configuration from the public [configuration template](../templates/config-repository/README.md). The resulting private repository is an implementation detail of that organization and is not required to be accessible to users of the public project. @@ -78,6 +78,7 @@ The private delivery repository is the authorized GitHub Actions identity. Fleet - Private configuration may declare required secret names but never their values. - GitHub App keys, tokens, project secrets, and deployment credentials stay outside Git history. - Real private configuration is validated against the same public schema used by examples. +- Real machine addresses, VM IDs, storage/backup identifiers, credentials, and rendered runtime files remain outside both public and private Git history. - A host remains generic; adding a repository changes policy and project configuration, not every worker image. ## Secret locations diff --git a/docs/README.md b/docs/README.md index 31a77eaf..4d6eb0e3 100644 --- a/docs/README.md +++ b/docs/README.md @@ -11,12 +11,14 @@ Use this index to find ci-fleet concepts, requirements, examples, and step-by-st | Decide whether it fits my infrastructure | [Architecture](ARCHITECTURE.md) and the root [README](../README.md) | | Try the experimental implementation safely | [Live pilot runbook](LIVE-PILOT.md) | | Add another Docker host, VM, computer, or VPS | [Adding a host](ADDING-A-HOST.md) | +| Manage controller configuration from Git | [Git-authored controller desired state](DESIRED-STATE.md) | | Add a private project to the shared runner pool | [Adding a project](ADDING-A-PROJECT.md) | | Convert an existing GitHub Actions workflow | [Migrating existing CI](MIGRATING-EXISTING-CI.md) | | Make a project compliant | [Project CI standard](PROJECT-STANDARD.md) and [compliance checklist](COMPLIANCE-CHECKLIST.md) | | Split tests across parallel workers | [Project CI standard](PROJECT-STANDARD.md) and the [parallel workflow example](../examples/workflows/parallel-ci.yml.example) | | Configure automatic updates and cleanup | [Host maintenance](HOST-MAINTENANCE.md) | | Handle GitHub App, workflow, or deployment secrets | [Secrets model](SECRETS.md) and [security policy](../SECURITY.md) | +| Review accepted implementation scope | [Design decisions](DESIGN-DECISIONS.md) | | Run private CI or deployment for a public project | [Public projects, private delivery, and private configuration](PUBLIC-PRIVATE-CONFIGURATION.md) | | Review current priorities | [Roadmap](ROADMAP.md) | | See what informed the design | [Discovery summary](DISCOVERY-SUMMARY.md) | @@ -33,12 +35,14 @@ Use this index to find ci-fleet concepts, requirements, examples, and step-by-st | Scale set | One controller's uniquely named runner capacity advertised to GitHub. | | Shared routing label | The stable `runs-on` capability used by compatible projects, regardless of which physical host accepts the job. | | Test shard | One bounded slice of a larger test suite, designed to run independently and usually finish within five minutes. | -| Private delivery configuration | Organization names, repository allowlists, host inventory, capacity, network policy, and credentials kept outside this public repository. | +| Private delivery configuration | Repository allowlists, logical controllers, capacity budgets, environment policy, and required secret names kept in a private but secret-free repository. | +| Desired state | A reviewed schema-v3 declaration of runner pools and logical controllers stored in a private, secret-free Git repository. | ## Design and boundaries - [System architecture](ARCHITECTURE.md) - [Runner controller design](CONTROLLER-DESIGN.md) +- [Accepted design decisions](DESIGN-DECISIONS.md) - [Public projects, private delivery, and private configuration](PUBLIC-PRIVATE-CONFIGURATION.md) - [Secrets model](SECRETS.md) - [Security policy](../SECURITY.md) @@ -51,6 +55,7 @@ These pages are normative for compatible projects and hosts: - [Migration procedure](MIGRATING-EXISTING-CI.md) - [Compliance checklist](COMPLIANCE-CHECKLIST.md) - [Host maintenance standard](HOST-MAINTENANCE.md) +- [Git-authored controller desired state](DESIRED-STATE.md) - [Secrets model](SECRETS.md) - [Security policy](../SECURITY.md) @@ -58,12 +63,13 @@ These pages are normative for compatible projects and hosts: - [Run the live pilot](LIVE-PILOT.md) - [Add a host](ADDING-A-HOST.md) +- [Install, adopt, check, upgrade, roll back, or remove a controller](DESIRED-STATE.md) - [Add a project](ADDING-A-PROJECT.md) - [Deploy the current experimental prototype](DEPLOYMENT-PROTOTYPE.md) - [Maintain, drain, clean, update, and reboot hosts](HOST-MAINTENANCE.md) - [Use the public configuration-repository scaffold](../templates/config-repository/README.md) -The accepted target is a single target-host command, `sudo ./setup.sh`. Until that installer is implemented and released, follow the versioned pilot and deployment procedures above rather than assuming the future interface exists. +The schema-v3 worker-controller installer is implemented as `scripts/install-worker-controller.sh`. Its accepted scope is isolated ordinary-CI fleet hosts under reviewed desired state; use the [desired-state guide](DESIRED-STATE.md), pin both configuration and engine commits, and retain the manual pilot runbook for each host's isolated first-job proof. ## Project integration examples diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index e86a4993..63bc64a1 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -62,6 +62,8 @@ Exit condition: both projects pass on the new fleet without removing the old pat ## Phase 5: Distributed capacity +- Manage runner pools and controllers through the schema-v3 private desired-state contract. +- Enroll or adopt hosts with the idempotent worker-controller installer. - Add a second independently provisioned Docker host. - Verify identical deployment and recovery. - Add demand-based scaling. diff --git a/docs/SECRETS.md b/docs/SECRETS.md index 7a8f2948..85dcf084 100644 --- a/docs/SECRETS.md +++ b/docs/SECRETS.md @@ -18,22 +18,25 @@ Project test or integration secrets belong in GitHub repository or environment s Normal validation should use no secret when possible. Deployment credentials must not be available to the normal shared validation pool. -### Host-local secrets +### Host-local identity and secrets A single-host prototype may use root-owned files outside the repository checkout, for example: ```text /etc/ci-fleet/secrets/github-app.pem +/etc/ci-fleet/host.env /etc/ci-fleet/ci-fleet.env ``` -Credential files should be owned by root, readable only by their intended service, and mounted only into the controller. +`host.env` contains the App client ID, installation ID, private-key path, and runner TTL. `ci-fleet.env` is rendered from reviewed desired state plus those approved host fields. Neither contains the PEM or application secrets, but both remain root-owned mode `0600` because they describe the controller identity and private installation. + +Credential files should be owned by root, readable only by their intended service, and mounted only into the controller. The installer never accepts a private key, token, or Git credential in a command-line argument. For a distributed fleet, use a secret manager or encrypted configuration system with independently revocable host identities. ## Docker Compose pattern -A future controller service may receive a host-local secret as a mounted file: +The controller service receives its host-local private key as a mounted file: ```yaml services: diff --git a/host/ci-fleet.env.example b/host/ci-fleet.env.example deleted file mode 100644 index fae86d89..00000000 --- a/host/ci-fleet.env.example +++ /dev/null @@ -1,22 +0,0 @@ -# Proposed configuration names only. -# This is not yet a stable or runnable interface. -# Never place real secrets in this file. - -CI_FLEET_GITHUB_ORGANIZATION=example-organization -CI_FLEET_RUNNER_GROUP=trusted-private-ci -CI_FLEET_SCALE_SET=ci-fleet-experimental - -CI_FLEET_MIN_RUNNERS=0 -CI_FLEET_MAX_RUNNERS=1 - -CI_FLEET_RUNNER_CPUS=4 -CI_FLEET_RUNNER_MEMORY=8g - -CI_FLEET_BUILD_CACHE_LIMIT=40GB -CI_FLEET_DISK_ALERT_PERCENT=75 -CI_FLEET_DISK_DRAIN_PERCENT=85 -CI_FLEET_DISK_EMERGENCY_PERCENT=90 - -# Long-lived credentials must be mounted from a root-owned file or -# retrieved from an external secret manager. They must not be placed here. -CI_FLEET_GITHUB_APP_KEY_FILE=/run/secrets/github_app_private_key diff --git a/host/host.env.example b/host/host.env.example new file mode 100644 index 00000000..e862c18a --- /dev/null +++ b/host/host.env.example @@ -0,0 +1,8 @@ +# Copy to /etc/ci-fleet/host.env and set mode 0600. +# These values stay on the controller host and are never rendered from Git. +# The private key itself must be root-owned mode 0600 at the path below. + +CI_FLEET_GITHUB_APP_CLIENT_ID=Iv1.EXAMPLE +CI_FLEET_GITHUB_APP_INSTALLATION_ID=12345678 +CI_FLEET_GITHUB_APP_PRIVATE_KEY_FILE=/etc/ci-fleet/secrets/github-app.pem +CI_FLEET_RUNNER_TTL=6h diff --git a/host/systemd/ci-fleet-cleanup.service b/host/systemd/ci-fleet-cleanup.service index 1bd47e1f..5eb060f4 100644 --- a/host/systemd/ci-fleet-cleanup.service +++ b/host/systemd/ci-fleet-cleanup.service @@ -6,6 +6,6 @@ Wants=docker.service [Service] Type=oneshot User=root -WorkingDirectory=/opt/ci-fleet +WorkingDirectory=/opt/ci-fleet/current EnvironmentFile=/etc/ci-fleet/ci-fleet.env -ExecStart=/opt/ci-fleet/scripts/cleanup.sh --apply +ExecStart=/opt/ci-fleet/current/scripts/cleanup.sh --apply diff --git a/host/systemd/ci-fleet-drift.service b/host/systemd/ci-fleet-drift.service new file mode 100644 index 00000000..50e8704c --- /dev/null +++ b/host/systemd/ci-fleet-drift.service @@ -0,0 +1,10 @@ +[Unit] +Description=Check ci-fleet host against its pinned Git desired state +After=docker.service network-online.target +Wants=docker.service network-online.target + +[Service] +Type=oneshot +User=root +WorkingDirectory=/opt/ci-fleet/manager/current +ExecStart=/opt/ci-fleet/manager/current/scripts/check-installed-state.sh diff --git a/host/systemd/ci-fleet-drift.timer b/host/systemd/ci-fleet-drift.timer new file mode 100644 index 00000000..4c197a40 --- /dev/null +++ b/host/systemd/ci-fleet-drift.timer @@ -0,0 +1,11 @@ +[Unit] +Description=Check pinned ci-fleet desired state every fifteen minutes + +[Timer] +OnBootSec=5min +OnUnitActiveSec=15min +AccuracySec=1min +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/host/systemd/ci-fleet-health.service b/host/systemd/ci-fleet-health.service index 00480e25..b33bc8fc 100644 --- a/host/systemd/ci-fleet-health.service +++ b/host/systemd/ci-fleet-health.service @@ -6,6 +6,6 @@ Wants=docker.service [Service] Type=oneshot User=root -WorkingDirectory=/opt/ci-fleet +WorkingDirectory=/opt/ci-fleet/manager/current EnvironmentFile=/etc/ci-fleet/ci-fleet.env -ExecStart=/opt/ci-fleet/scripts/healthcheck.sh +ExecStart=/opt/ci-fleet/manager/current/scripts/healthcheck.sh diff --git a/runner/Dockerfile b/runner/Dockerfile index eefda989..6b0b6030 100644 --- a/runner/Dockerfile +++ b/runner/Dockerfile @@ -2,9 +2,11 @@ FROM debian:13.6-slim ARG TARGETARCH +ARG CI_FLEET_COMMIT=unknown ARG RUNNER_VERSION=2.335.1 ARG RUNNER_SHA256_AMD64=4ef2f25285f0ae4477f1fe1e346db76d2f3ebf03824e2ddd1973a2819bf6c8cf ARG RUNNER_SHA256_ARM64=6d1e85bfd1a506a8b17c1f1b9b57dba458ffed90898799aaa9f599520b0d9207 +LABEL org.opencontainers.image.revision="${CI_FLEET_COMMIT}" SHELL ["/bin/bash", "-o", "pipefail", "-c"] RUN apt-get update \ diff --git a/scripts/check-installed-state.sh b/scripts/check-installed-state.sh new file mode 100755 index 00000000..99407d99 --- /dev/null +++ b/scripts/check-installed-state.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +state_file=${CI_FLEET_INSTALL_STATE_FILE:-/var/lib/ci-fleet/install-state.json} +installer=${CI_FLEET_INSTALLER:-/opt/ci-fleet/manager/current/scripts/install-worker-controller.sh} + +[[ -f "$state_file" ]] || { echo "ERROR: installed desired-state record is missing: $state_file" >&2; exit 2; } +expected_owner=0 +[[ ${CI_FLEET_TESTING:-0} != 1 ]] || expected_owner=$(id -u) +[[ $(stat -c %u "$state_file") == "$expected_owner" && $(stat -c %a "$state_file") == 600 ]] || { echo "ERROR: install state must be owned by root with mode 0600: $state_file" >&2; exit 2; } +command -v python3 >/dev/null || { echo 'ERROR: python3 is required' >&2; exit 2; } +[[ -x "$installer" ]] || { echo "ERROR: installer is unavailable: $installer" >&2; exit 2; } + +if ! state_values=$(python3 - "$state_file" <<'PY' +import json +import sys + +try: + state = json.load(open(sys.argv[1], encoding="utf-8")) + values = [state[name] for name in ("controller", "config_repository", "config_ref")] + if not all(isinstance(value, str) and value for value in values): + raise ValueError +except (OSError, ValueError, KeyError, TypeError, json.JSONDecodeError): + raise SystemExit(2) +print("\n".join(values)) +PY +); then + echo "ERROR: installed desired-state record is invalid: $state_file" >&2 + exit 2 +fi +mapfile -t values <<<"$state_values" +[[ ${#values[@]} == 3 ]] || { echo "ERROR: installed desired-state record is incomplete: $state_file" >&2; exit 2; } +controller=${values[0]} +config_repository=${values[1]} +config_ref=${values[2]} + +exec "$installer" --check \ + --config-repo "$config_repository" \ + --ref "$config_ref" \ + --controller "$controller" diff --git a/scripts/desired_state.py b/scripts/desired_state.py new file mode 100755 index 00000000..42c66bbd --- /dev/null +++ b/scripts/desired_state.py @@ -0,0 +1,278 @@ +#!/usr/bin/env python3 +"""Validate, select, and render schema-v3 ci-fleet controller desired state.""" + +from __future__ import annotations + +import argparse +import importlib.util +import json +import os +import re +import stat +import sys +import tempfile +from pathlib import Path +from typing import Any + + +ROOT = Path(__file__).resolve().parents[1] +TEMPLATE_VALIDATOR = ROOT / "templates" / "config-repository" / "scripts" / "validate.py" +COMMIT_SHA = re.compile(r"^[0-9a-f]{40}$") +SAFE_IDENTIFIER = re.compile(r"^[A-Za-z0-9._-]+$") +SAFE_DURATION = re.compile(r"^[1-9][0-9]*(?:s|m|h)$") +SAFE_ABSOLUTE_PATH = re.compile(r"^/[A-Za-z0-9._/-]+$") +SAFE_ENV_VALUE = re.compile(r"^[A-Za-z0-9._/:,-]+$") +HOST_REQUIRED = { + "CI_FLEET_GITHUB_APP_CLIENT_ID", + "CI_FLEET_GITHUB_APP_INSTALLATION_ID", + "CI_FLEET_GITHUB_APP_PRIVATE_KEY_FILE", +} +HOST_OPTIONAL = {"CI_FLEET_RUNNER_TTL"} + + +class DesiredStateError(ValueError): + """A safe operator-facing desired-state error.""" + + +def load_template_validator(): + spec = importlib.util.spec_from_file_location("ci_fleet_template_validator", TEMPLATE_VALIDATOR) + if spec is None or spec.loader is None: + raise DesiredStateError("public configuration validator could not be loaded") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def load_and_validate_config(path: Path) -> dict[str, Any]: + module = load_template_validator() + validation = module.Validation() + config = module.load_json(path, validation) + if config is not None: + module.scan_secret_material(config, validation) + module.validate_config(config, validation, False) + if validation.errors: + raise DesiredStateError("configuration rejected:\n" + "\n".join(f"- {error}" for error in validation.errors)) + if not isinstance(config, dict) or config.get("schema_version") != 3: + raise DesiredStateError("configuration must use schema_version 3") + return config + + +def parse_env(path: Path, *, allow_unknown: bool) -> dict[str, str]: + try: + lines = path.read_text(encoding="utf-8").splitlines() + except FileNotFoundError as exc: + raise DesiredStateError(f"host configuration does not exist: {path}") from exc + values: dict[str, str] = {} + for number, raw in enumerate(lines, start=1): + line = raw.strip() + if not line or line.startswith("#"): + continue + if "=" not in line: + raise DesiredStateError(f"{path}:{number}: expected NAME=value") + name, value = line.split("=", 1) + if not re.fullmatch(r"[A-Z][A-Z0-9_]*", name): + raise DesiredStateError(f"{path}:{number}: invalid variable name") + if name in values: + raise DesiredStateError(f"{path}:{number}: duplicate variable {name}") + if any(character in value for character in "\r\n\0"): + raise DesiredStateError(f"{path}:{number}: multiline values are forbidden") + if not allow_unknown and name not in HOST_REQUIRED | HOST_OPTIONAL: + raise DesiredStateError(f"{path}:{number}: unsupported host-local variable {name}") + values[name] = value + return values + + +def validate_host_values(values: dict[str, str]) -> dict[str, str]: + missing = sorted(HOST_REQUIRED - values.keys()) + if missing: + raise DesiredStateError("host configuration is missing: " + ", ".join(missing)) + client_id = values["CI_FLEET_GITHUB_APP_CLIENT_ID"] + installation_id = values["CI_FLEET_GITHUB_APP_INSTALLATION_ID"] + key_file = values["CI_FLEET_GITHUB_APP_PRIVATE_KEY_FILE"] + if not SAFE_IDENTIFIER.fullmatch(client_id): + raise DesiredStateError("GitHub App client ID contains unsupported characters") + if not installation_id.isdigit() or int(installation_id) < 1: + raise DesiredStateError("GitHub App installation ID must be a positive integer") + if not SAFE_ABSOLUTE_PATH.fullmatch(key_file): + raise DesiredStateError("GitHub App private-key path must be an absolute shell-safe path") + ttl = values.get("CI_FLEET_RUNNER_TTL", "6h") + if not SAFE_DURATION.fullmatch(ttl): + raise DesiredStateError("runner TTL must be a positive duration ending in s, m, or h") + multiplier = {"s": 1, "m": 60, "h": 3600}[ttl[-1]] + if int(ttl[:-1]) * multiplier < 3600: + raise DesiredStateError("runner TTL must be at least one hour") + return { + "CI_FLEET_GITHUB_APP_CLIENT_ID": client_id, + "CI_FLEET_GITHUB_APP_INSTALLATION_ID": installation_id, + "CI_FLEET_GITHUB_APP_PRIVATE_KEY_FILE": key_file, + "CI_FLEET_RUNNER_TTL": ttl, + } + + +def select_controller(config: dict[str, Any], controller_id: str) -> tuple[dict[str, Any], dict[str, Any]]: + controllers = config["controllers"] + if controller_id not in controllers: + available = ", ".join(sorted(controllers)) + raise DesiredStateError(f"controller {controller_id!r} is not declared; available: {available}") + controller = controllers[controller_id] + pool = config["runner_pools"][controller["pool"]] + return controller, pool + + +def build_rendered_env( + config: dict[str, Any], + controller_id: str, + host_values: dict[str, str], + *, + config_repository: str, + config_ref: str, + docker_gid: int, +) -> tuple[dict[str, str], dict[str, Any]]: + controller, pool = select_controller(config, controller_id) + engine_commit = controller["engine_ref"] + if not COMMIT_SHA.fullmatch(config_ref): + raise DesiredStateError("configuration ref must be a full lowercase commit SHA") + if docker_gid < 0: + raise DesiredStateError("Docker socket GID must be a non-negative integer") + if not SAFE_ENV_VALUE.fullmatch(config_repository): + raise DesiredStateError("configuration repository identity must be shell-safe") + + state = controller["state"] + configured_max = controller["max_runners"] + effective_max = configured_max if state == "active" else 0 + short_commit = engine_commit[:12] + rendered = { + "CI_FLEET_CAPACITY_BUDGET": str(pool["capacity_budget"]), + "CI_FLEET_COMMIT": engine_commit, + "CI_FLEET_CONFIGURED_MAX_RUNNERS": str(configured_max), + "CI_FLEET_CONFIG_REF": config_ref, + "CI_FLEET_CONFIG_REPOSITORY": config_repository, + "CI_FLEET_CONTROLLER_IMAGE": f"ci-fleet-controller:{short_commit}", + "CI_FLEET_CONTROLLER_STATE": state, + "CI_FLEET_DESIRED_STATE_SCHEMA": "3", + "CI_FLEET_DOCKER_GID": str(docker_gid), + "CI_FLEET_ENGINE_REF": engine_commit, + "CI_FLEET_GITHUB_URL": f"https://github.com/{config['organization']['slug']}", + "CI_FLEET_INSTANCE": controller_id, + "CI_FLEET_LABELS": ",".join(pool["routing_labels"]), + "CI_FLEET_MAX_RUNNERS": str(effective_max), + "CI_FLEET_MIN_RUNNERS": str(controller["min_runners"] if state == "active" else 0), + "CI_FLEET_RUNNER_CPUS": str(controller["runner_resources"]["cpu_cores"]), + "CI_FLEET_RUNNER_GROUP": pool["runner_group"], + "CI_FLEET_RUNNER_IMAGE": f"ci-fleet-runner:{short_commit}", + "CI_FLEET_RUNNER_MEMORY_MIB": str(controller["runner_resources"]["memory_mib"]), + "CI_FLEET_SCALE_SET_NAME": controller["scale_set_name"], + "CI_FLEET_VERSION": short_commit, + **validate_host_values(host_values), + } + for name, value in rendered.items(): + if not SAFE_ENV_VALUE.fullmatch(value): + raise DesiredStateError(f"rendered value for {name} contains unsafe characters") + metadata = { + "schema_version": 1, + "controller": controller_id, + "controller_state": state, + "pool": controller["pool"], + "location": controller["location"], + "lifecycle": controller["lifecycle"], + "scale_set_name": controller["scale_set_name"], + "configured_max_runners": configured_max, + "effective_max_runners": effective_max, + "capacity_budget": pool["capacity_budget"], + "config_repository": config_repository, + "config_ref": config_ref, + "engine_ref": engine_commit, + "engine_repository": config["organization"]["delivery_engine"], + } + return rendered, metadata + + +def write_private(path: Path, content: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + descriptor, temporary_name = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent, text=True) + temporary = Path(temporary_name) + try: + os.fchmod(descriptor, stat.S_IRUSR | stat.S_IWUSR) + with os.fdopen(descriptor, "w", encoding="utf-8") as handle: + handle.write(content) + handle.flush() + os.fsync(handle.fileno()) + os.replace(temporary, path) + finally: + temporary.unlink(missing_ok=True) + + +def render_env(values: dict[str, str]) -> str: + return "".join(f"{name}={values[name]}\n" for name in sorted(values)) + + +def command_validate(args: argparse.Namespace) -> None: + config = load_and_validate_config(args.config) + print(f"DESIRED_STATE_OK schema={config['schema_version']} controllers={len(config['controllers'])}") + + +def command_extract_host(args: argparse.Namespace) -> None: + values = parse_env(args.source, allow_unknown=True) + selected = validate_host_values({name: value for name, value in values.items() if name in HOST_REQUIRED | HOST_OPTIONAL}) + write_private(args.output, render_env(selected)) + print(f"HOST_CONFIG_WRITTEN path={args.output}") + + +def command_render(args: argparse.Namespace) -> None: + config = load_and_validate_config(args.config) + host_values = parse_env(args.host_config, allow_unknown=False) + values, metadata = build_rendered_env( + config, + args.controller, + host_values, + config_repository=args.config_repository, + config_ref=args.config_ref, + docker_gid=args.docker_gid, + ) + write_private(args.output, render_env(values)) + write_private(args.metadata_output, json.dumps(metadata, indent=2, sort_keys=True) + "\n") + print( + "DESIRED_STATE_RENDERED " + f"controller={metadata['controller']} state={metadata['controller_state']} " + f"config_ref={metadata['config_ref']} engine_ref={metadata['engine_ref']}" + ) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + subparsers = parser.add_subparsers(dest="command", required=True) + + validate = subparsers.add_parser("validate", help="validate a complete schema-v3 configuration") + validate.add_argument("--config", type=Path, required=True) + validate.set_defaults(function=command_validate) + + extract = subparsers.add_parser("extract-host-env", help="extract approved host-local values during adoption") + extract.add_argument("--source", type=Path, required=True) + extract.add_argument("--output", type=Path, required=True) + extract.set_defaults(function=command_extract_host) + + render = subparsers.add_parser("render", help="render one controller into a host-local runtime environment") + render.add_argument("--config", type=Path, required=True) + render.add_argument("--controller", required=True) + render.add_argument("--host-config", type=Path, required=True) + render.add_argument("--config-repository", required=True) + render.add_argument("--config-ref", required=True) + render.add_argument("--docker-gid", type=int, required=True) + render.add_argument("--output", type=Path, required=True) + render.add_argument("--metadata-output", type=Path, required=True) + render.set_defaults(function=command_render) + return parser.parse_args() + + +def main() -> int: + args = parse_args() + try: + args.function(args) + except DesiredStateError as exc: + print(f"ERROR: {exc}", file=sys.stderr) + return 2 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/healthcheck.sh b/scripts/healthcheck.sh index f50621bc..4763d5da 100755 --- a/scripts/healthcheck.sh +++ b/scripts/healthcheck.sh @@ -4,6 +4,7 @@ set -Eeuo pipefail warn_percent=${CI_FLEET_DISK_WARN_PERCENT:-80} critical_percent=${CI_FLEET_DISK_CRITICAL_PERCENT:-90} controller_name=${CI_FLEET_CONTROLLER_CONTAINER:-ci-fleet-controller-1} +desired_state=${CI_FLEET_CONTROLLER_STATE:-active} status=0 emit() { printf '%s\n' "$*"; } @@ -13,15 +14,22 @@ if ! command -v docker >/dev/null || ! docker info >/dev/null 2>&1; then fi emit "OK docker_available" -used=$(df -P /var/lib/docker 2>/dev/null | awk 'NR==2 {gsub(/%/, "", $5); print $5}') +docker_root=${CI_FLEET_DOCKER_ROOT:-/var/lib/docker} +used=$(df -P "$docker_root" 2>/dev/null | awk 'NR==2 {gsub(/%/, "", $5); print $5}' || true) if [[ -z "$used" ]]; then used=$(df -P / | awk 'NR==2 {gsub(/%/, "", $5); print $5}'); fi if ((used >= critical_percent)); then emit "CRITICAL disk_used_percent=$used"; status=2 elif ((used >= warn_percent)); then emit "WARN disk_used_percent=$used"; ((status < 1)) && status=1 else emit "OK disk_used_percent=$used"; fi controller_state=$(docker inspect --format '{{.State.Status}}' "$controller_name" 2>/dev/null || true) -if [[ "$controller_state" == running ]]; then emit "OK controller_state=running" -else emit "CRITICAL controller_state=${controller_state:-missing}"; status=2; fi +if [[ "$desired_state" == active ]]; then + if [[ "$controller_state" == running ]]; then emit "OK controller_state=running" + else emit "CRITICAL controller_state=${controller_state:-missing} desired_state=active"; status=2; fi +elif [[ -z "$controller_state" || "$controller_state" == exited || "$controller_state" == created ]]; then + emit "OK controller_state=${controller_state:-missing} desired_state=$desired_state" +else + emit "CRITICAL controller_state=$controller_state desired_state=$desired_state"; status=2 +fi stale=$(docker ps -aq --filter label=io.randomdevelopment.ci-fleet.managed=true --filter status=exited | wc -l | tr -d ' ') if ((stale > 0)); then emit "WARN inactive_managed_containers=$stale"; ((status < 1)) && status=1 diff --git a/scripts/install-worker-controller.sh b/scripts/install-worker-controller.sh new file mode 100755 index 00000000..891929f7 --- /dev/null +++ b/scripts/install-worker-controller.sh @@ -0,0 +1,980 @@ +#!/usr/bin/env bash +set -Eeuo pipefail +set +x + +repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +mode= +config_repo= +config_ref= +controller_id= +host_config_arg= +config_source_checkout= +root_prefix=${CI_FLEET_ROOT_PREFIX:-} +testing=${CI_FLEET_TESTING:-0} +transaction_active=false +checkpoint_dir= +staging_paths=() + +usage() { + cat >&2 <<'EOF' +usage: + install-worker-controller.sh --check|--install|--adopt|--upgrade \ + --config-repo OWNER/REPOSITORY|PATH --ref FULL_COMMIT_SHA \ + --controller CONTROLLER_ID + + install-worker-controller.sh --rollback + install-worker-controller.sh --uninstall + +Modes are mutually exclusive. Remote private repositories use the target host's +preconfigured read-only Git credentials; credentials are never accepted in URLs +or command-line arguments. Managed installs always use /etc/ci-fleet/host.env. +EOF +} + +note() { printf '%s\n' "$*"; } +die() { + printf 'ERROR: %s\n' "$*" >&2 + if [[ ${transaction_active:-false} == true ]] && declare -F restore_checkpoint >/dev/null; then + restore_checkpoint || true + transaction_active=false + fi + exit 2 +} + +while (($#)); do + case "$1" in + --check|--install|--adopt|--upgrade|--rollback|--uninstall) + [[ -z "$mode" ]] || die 'select exactly one operating mode' + mode=${1#--} + shift + ;; + --config-repo) + (($# >= 2)) || die '--config-repo requires a value' + config_repo=$2 + shift 2 + ;; + --ref) + (($# >= 2)) || die '--ref requires a value' + config_ref=$2 + shift 2 + ;; + --controller) + (($# >= 2)) || die '--controller requires a value' + controller_id=$2 + shift 2 + ;; + --host-config) + (($# >= 2)) || die '--host-config requires a value' + host_config_arg=$2 + shift 2 + ;; + -h|--help) + usage + exit 0 + ;; + *) + usage + die "unknown argument: $1" + ;; + esac +done + +[[ -n "$mode" ]] || { usage; die 'an explicit operating mode is required'; } +if [[ -n "$root_prefix" && "$testing" != 1 ]]; then + die 'CI_FLEET_ROOT_PREFIX is test-only and requires CI_FLEET_TESTING=1' +fi +if [[ "$testing" != 1 && ${EUID:-$(id -u)} -ne 0 ]]; then + die 'run this installer as root' +fi + +root_path() { printf '%s%s' "$root_prefix" "$1"; } +is_git_checkout() { git -C "$1" rev-parse --is-inside-work-tree >/dev/null 2>&1; } + +install_root=$(root_path /opt/ci-fleet) +releases_dir=$install_root/releases +current_link=$install_root/current +manager_root=$install_root/manager +manager_releases=$manager_root/releases +manager_current=$manager_root/current +etc_dir=$(root_path /etc/ci-fleet) +rendered_env=$etc_dir/ci-fleet.env +default_host_config=$etc_dir/host.env +host_config=${host_config_arg:-$default_host_config} +state_root=$(root_path /var/lib/ci-fleet) +state_file=$state_root/install-state.json +checkpoints_dir=$state_root/checkpoints +systemd_dir=$(root_path /etc/systemd/system) +lock_file=$(root_path /run/ci-fleet-installer.lock) +controller_container=ci-fleet-controller-1 +unit_names=( + ci-fleet-health.service ci-fleet-health.timer + ci-fleet-cleanup.service ci-fleet-cleanup.timer + ci-fleet-drift.service ci-fleet-drift.timer +) +timer_names=(ci-fleet-health.timer ci-fleet-cleanup.timer ci-fleet-drift.timer) + +temporary=$(mktemp -d) +cleanup_temporary() { + local path + rm -rf "$temporary" + for path in "${staging_paths[@]}"; do + [[ -z "$path" ]] || rm -rf -- "$path" + done +} +trap cleanup_temporary EXIT + +require_commands() { + local command + for command in git python3 docker tar install cmp readlink systemctl stat awk grep date flock mktemp; do + command -v "$command" >/dev/null || die "$command is required" + done + docker info >/dev/null 2>&1 || die 'Docker daemon is unavailable' + docker compose version >/dev/null 2>&1 || die 'Docker Compose v2 is unavailable' +} + +validate_common_arguments() { + [[ -n "$config_repo" ]] || die '--config-repo is required for this mode' + [[ "$config_ref" =~ ^[0-9a-f]{40}$ ]] || die '--ref must be a full lowercase commit SHA' + [[ "$controller_id" =~ ^[a-z0-9][a-z0-9-]{0,62}$ ]] || die '--controller must be a lowercase logical ID' + [[ -z "$host_config_arg" || "$host_config" == "$default_host_config" ]] || die 'managed installs require the default /etc/ci-fleet/host.env path' + if [[ "$config_repo" == *://* || "$config_repo" == *@* ]]; then + die '--config-repo must not contain a URL or embedded credentials; use OWNER/REPOSITORY or a local path' + fi +} + +resolve_config() { + local resolved checkout + candidate_config=$temporary/fleet.json + if is_git_checkout "$config_repo"; then + config_identity=$(cd "$config_repo" && pwd -P) + config_source_checkout=$config_identity + resolved=$(git -C "$config_identity" rev-parse "$config_ref^{commit}" 2>/dev/null || true) + [[ "$resolved" == "$config_ref" ]] || die 'local configuration repository does not contain the requested commit' + git -C "$config_identity" show "$config_ref:fleet.json" >"$candidate_config" || die 'fleet.json is absent at the requested configuration commit' + return + fi + [[ "$config_repo" =~ ^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$ ]] || die '--config-repo must be OWNER/REPOSITORY or a local Git checkout' + checkout=$temporary/config-repository + config_source_checkout=$checkout + git init -q "$checkout" + git -C "$checkout" remote add origin "https://github.com/${config_repo}.git" + if ! GIT_TERMINAL_PROMPT=0 git -C "$checkout" fetch -q --filter=blob:none --depth=1 origin "$config_ref"; then + die 'configuration fetch failed; configure a read-only credential on this host or use a local pinned checkout' + fi + resolved=$(git -C "$checkout" rev-parse 'FETCH_HEAD^{commit}') + [[ "$resolved" == "$config_ref" ]] || die 'fetched configuration commit does not match --ref' + git -C "$checkout" show "$config_ref:fleet.json" >"$candidate_config" || die 'fleet.json is absent at the requested configuration commit' + config_identity=$config_repo +} + +validate_candidate_config_commit() { + local tree_paths=$temporary/config-tree-paths + git -C "$config_source_checkout" ls-tree -rz --name-only "$config_ref" >"$tree_paths" || die 'cannot inspect the configuration commit tree' + python3 "$repo_root/templates/config-repository/scripts/validate.py" \ + --config "$candidate_config" --strict --tree-paths "$tree_paths" || die 'configuration commit validation failed' + python3 "$repo_root/scripts/scan_committed_secrets.py" \ + --repository "$config_source_checkout" --commit "$config_ref" || die 'configuration commit secret scan failed' +} + +prepare_host_config() { + local expected_owner=0 + effective_host_config=$host_config + if [[ -f "$host_config" ]]; then + return + fi + if [[ -f "$rendered_env" && "$mode" == adopt ]]; then + [[ "$testing" != 1 ]] || expected_owner=$(id -u) + [[ $(stat -c %u "$rendered_env") == "$expected_owner" && $(stat -c %a "$rendered_env") == 600 ]] || die "rendered environment must be owned by root with mode 0600: $rendered_env" + install -d -m 0700 "$etc_dir" + python3 "$repo_root/scripts/desired_state.py" extract-host-env \ + --source "$rendered_env" --output "$effective_host_config" + return + fi + die "host-local GitHub App configuration is missing: $host_config" +} + +verify_host_files() { + local mode_bits owner expected_owner key_file + expected_owner=0 + [[ "$testing" != 1 ]] || expected_owner=$(id -u) + mode_bits=$(stat -c '%a' "$effective_host_config") + owner=$(stat -c '%u' "$effective_host_config") + [[ "$mode_bits" == 600 ]] || die "host configuration must have mode 0600: $effective_host_config" + [[ "$owner" == "$expected_owner" ]] || die 'host configuration must be owned by root' + key_file=$(awk -F= '$1 == "CI_FLEET_GITHUB_APP_PRIVATE_KEY_FILE" {print substr($0, index($0, "=") + 1)}' "$effective_host_config") + [[ -n "$key_file" && -f "$key_file" ]] || die 'GitHub App PEM file is missing' + mode_bits=$(stat -c '%a' "$key_file") + owner=$(stat -c '%u' "$key_file") + [[ "$mode_bits" == 600 && "$owner" == "$expected_owner" ]] || die 'GitHub App PEM must be owned by root and have mode 0600' +} + +load_installed_controller_identity() { + local source_state=${1:-$state_file} source_env=${2:-$rendered_env} expected_owner=0 + [[ "$testing" != 1 ]] || expected_owner=$(id -u) + if [[ -f "$source_state" && $(stat -c %u "$source_state") == "$expected_owner" && $(stat -c %a "$source_state") == 600 ]]; then + controller_id=$(python3 - "$source_state" <<'PY' +import json +import sys +try: + value = json.load(open(sys.argv[1], encoding="utf-8"))["controller"] +except (OSError, KeyError, TypeError, ValueError, json.JSONDecodeError): + raise SystemExit(2) +print(value) +PY + ) || die "installed controller identity is invalid: $source_state" + elif [[ -f "$source_env" ]]; then + [[ $(stat -c %u "$source_env") == "$expected_owner" && $(stat -c %a "$source_env") == 600 ]] || die "rendered environment must be owned by root with mode 0600: $source_env" + controller_id=$(awk -F= '$1 == "CI_FLEET_INSTANCE" {count++; value=substr($0, index($0, "=") + 1)} END {if (count != 1) exit 1; print value}' "$source_env") || die "installed controller identity is invalid: $source_env" + elif [[ -f "$source_state" ]]; then + die "install state must be owned by root with mode 0600: $source_state" + else + die 'installed controller identity is unavailable' + fi + [[ "$controller_id" =~ ^[a-z0-9][a-z0-9-]{0,62}$ ]] || die 'installed controller identity is invalid' +} + +docker_gid() { + if [[ "$testing" == 1 && -n ${CI_FLEET_DOCKER_GID_OVERRIDE:-} ]]; then + printf '%s' "$CI_FLEET_DOCKER_GID_OVERRIDE" + return + fi + stat -c '%g' /var/run/docker.sock +} + +render_candidate() { + local -a metadata_values + candidate_env=$temporary/ci-fleet.env + candidate_metadata=$temporary/metadata.json + python3 "$repo_root/scripts/desired_state.py" render \ + --config "$candidate_config" \ + --controller "$controller_id" \ + --host-config "$effective_host_config" \ + --config-repository "$config_identity" \ + --config-ref "$config_ref" \ + --docker-gid "$(docker_gid)" \ + --output "$candidate_env" \ + --metadata-output "$candidate_metadata" + mapfile -t metadata_values < <(python3 - "$candidate_metadata" <<'PY' +import json +import sys +value = json.load(open(sys.argv[1], encoding="utf-8")) +for key in ("controller_state", "engine_ref", "engine_repository"): + print(value[key]) +PY + ) + [[ ${#metadata_values[@]} == 3 ]] || die 'rendered controller metadata is incomplete' + target_state=${metadata_values[0]} + engine_ref=${metadata_values[1]} + engine_repository=${metadata_values[2]} + [[ "$engine_repository" == RandomDevelopment/ci-fleet ]] || die 'delivery engine repository is not the fixed reviewed public engine' + release_dir=$releases_dir/$engine_ref +} + +compose() { + local release=$1 env_file=$2 variable + local -a clean_environment=(env -i "PATH=$PATH" "HOME=${HOME:-/root}") + shift 2 + for variable in DOCKER_HOST DOCKER_CONTEXT DOCKER_TLS_VERIFY DOCKER_CERT_PATH DOCKER_CONFIG XDG_RUNTIME_DIR; do + [[ ! -v $variable ]] || clean_environment+=("$variable=${!variable}") + done + if [[ "$testing" == 1 ]]; then + for variable in ${!FAKE_@}; do clean_environment+=("$variable=${!variable}"); done + fi + "${clean_environment[@]}" docker compose --project-name ci-fleet --env-file "$env_file" -f "$release/deploy/compose.yaml" "$@" +} + +controller_status() { + docker inspect --format '{{.State.Status}}' "$controller_container" 2>/dev/null || true +} + +current_runtime_release() { + local target + if [[ -L "$current_link" ]]; then + target=$(readlink -f "$current_link" 2>/dev/null || true) + [[ -z "$target" ]] || printf '%s' "$target" + elif [[ -f "$install_root/deploy/compose.yaml" ]]; then + printf '%s' "$install_root" + fi +} + +managed_runner_count() { + docker ps -q \ + --filter label=io.randomdevelopment.ci-fleet.managed=true \ + --filter label=io.randomdevelopment.ci-fleet.kind=runner \ + --filter "label=io.randomdevelopment.ci-fleet.instance=$controller_id" | wc -l | tr -d ' ' +} + +managed_runner_total_count() { + docker ps --all -q \ + --filter label=io.randomdevelopment.ci-fleet.managed=true \ + --filter label=io.randomdevelopment.ci-fleet.kind=runner \ + --filter "label=io.randomdevelopment.ci-fleet.instance=$controller_id" | wc -l | tr -d ' ' +} + +remove_inactive_managed_runners() { + local -a containers=() + mapfile -t containers < <(docker ps --all -q \ + --filter label=io.randomdevelopment.ci-fleet.managed=true \ + --filter label=io.randomdevelopment.ci-fleet.kind=runner \ + --filter "label=io.randomdevelopment.ci-fleet.instance=$controller_id") + ((${#containers[@]} == 0)) || docker rm "${containers[@]}" >/dev/null +} + +controller_environment_matches() { + local actual expected key live + live=$(docker inspect --format '{{range .Config.Env}}{{println .}}{{end}}' "$controller_container" 2>/dev/null) || return 1 + for key in \ + CI_FLEET_GITHUB_URL CI_FLEET_SCALE_SET_NAME CI_FLEET_LABELS CI_FLEET_RUNNER_GROUP \ + CI_FLEET_RUNNER_IMAGE CI_FLEET_INSTANCE CI_FLEET_GITHUB_APP_CLIENT_ID \ + CI_FLEET_GITHUB_APP_INSTALLATION_ID CI_FLEET_MIN_RUNNERS CI_FLEET_MAX_RUNNERS \ + CI_FLEET_RUNNER_CPUS CI_FLEET_RUNNER_MEMORY_MIB CI_FLEET_RUNNER_TTL CI_FLEET_DOCKER_GID; do + expected=$(awk -F= -v key="$key" '$1 == key {print substr($0, index($0, "=") + 1)}' "$candidate_env") + [[ -n "$expected" ]] || return 1 + actual=$(awk -F= -v key="$key" '$1 == key {count++; value=substr($0, index($0, "=") + 1)} END {if (count != 1) exit 1; print value}' <<<"$live") || return 1 + [[ "$actual" == "$expected" ]] || return 1 + done +} + +runtime_matches() { + local expected=$1 expected_image expected_image_ref live_image provenance status + status=$(controller_status) + if [[ "$expected" == active ]]; then + [[ "$status" == running ]] || return 1 + expected_image_ref=$(awk -F= '$1 == "CI_FLEET_CONTROLLER_IMAGE" {print substr($0, index($0, "=") + 1)}' "$candidate_env") + [[ -n "$expected_image_ref" ]] || return 1 + live_image=$(docker inspect --format '{{.Image}}' "$controller_container" 2>/dev/null) || return 1 + expected_image=$(docker image inspect --format '{{.Id}}' "$expected_image_ref" 2>/dev/null) || return 1 + [[ "$live_image" == "$expected_image" ]] || return 1 + provenance=$(docker inspect --format '{{ index .Config.Labels "org.opencontainers.image.revision" }}' "$controller_container" 2>/dev/null) || return 1 + [[ "$provenance" == "$engine_ref" ]] || return 1 + controller_environment_matches + else + [[ -z "$status" || "$status" == exited || "$status" == created ]] + fi +} + +state_matches() { + local expected_owner=0 + [[ -f "$state_file" ]] || return 1 + [[ "$testing" != 1 ]] || expected_owner=$(id -u) + [[ $(stat -c %u "$state_file") == "$expected_owner" && $(stat -c %a "$state_file") == 600 ]] || return 1 + python3 - "$state_file" "$candidate_metadata" <<'PY' +import json +import sys +installed = json.load(open(sys.argv[1], encoding="utf-8")) +installed.pop("installed_at", None) +candidate = json.load(open(sys.argv[2], encoding="utf-8")) +raise SystemExit(0 if installed == candidate else 1) +PY +} + +release_tree_digest() { + python3 - "$1" <<'PY' +import hashlib +import os +import stat +import sys + +root = os.path.abspath(sys.argv[1]) +excluded = {".ci-fleet-engine-ref", ".ci-fleet-tree-sha256"} +digest = hashlib.sha256() + + +def add(kind, relative, mode, payload=b""): + digest.update(kind) + digest.update(b"\0") + digest.update(relative.encode("utf-8", "surrogateescape")) + digest.update(b"\0") + digest.update(f"{mode:o}".encode("ascii")) + digest.update(b"\0") + digest.update(payload) + digest.update(b"\0") + + +def visit(directory): + for entry in sorted(os.scandir(directory), key=lambda item: item.name): + relative = os.path.relpath(entry.path, root) + if relative in excluded: + continue + metadata = entry.stat(follow_symlinks=False) + mode = stat.S_IMODE(metadata.st_mode) + if stat.S_ISDIR(metadata.st_mode): + add(b"directory", relative, mode) + visit(entry.path) + elif stat.S_ISREG(metadata.st_mode): + content = hashlib.sha256() + with open(entry.path, "rb") as handle: + for block in iter(lambda: handle.read(1024 * 1024), b""): + content.update(block) + add(b"file", relative, mode, content.digest()) + elif stat.S_ISLNK(metadata.st_mode): + add(b"symlink", relative, mode, os.readlink(entry.path).encode("utf-8", "surrogateescape")) + else: + raise SystemExit(f"unsupported release entry: {relative}") + + +visit(root) +print(digest.hexdigest()) +PY +} + +runtime_release_complete() { + local path=$1 expected=$2 marker required stored_digest actual_digest + [[ -d "$path" && -f "$path/.ci-fleet-engine-ref" && -f "$path/.ci-fleet-tree-sha256" && -f "$path/deploy/compose.yaml" ]] || return 1 + [[ -x "$path/scripts/preflight.sh" && -x "$path/scripts/healthcheck.sh" && -x "$path/scripts/cleanup.sh" ]] || return 1 + for required in controller/Dockerfile controller/go.mod controller/main.go controller/config.go controller/scaler.go controller/state.go runner/Dockerfile; do + [[ -f "$path/$required" ]] || return 1 + done + marker=$(<"$path/.ci-fleet-engine-ref") + [[ "$marker" == "$expected" ]] || return 1 + stored_digest=$(<"$path/.ci-fleet-tree-sha256") + [[ "$stored_digest" =~ ^[0-9a-f]{64}$ ]] || return 1 + actual_digest=$(release_tree_digest "$path") || return 1 + [[ "$actual_digest" == "$stored_digest" ]] +} + +manager_release_complete() { + local path=$1 expected=$2 marker required unit + runtime_release_complete "$path" "$expected" || return 1 + [[ -x "$path/scripts/install-worker-controller.sh" && -x "$path/scripts/check-installed-state.sh" ]] || return 1 + for required in scripts/desired_state.py scripts/scan_committed_secrets.py templates/config-repository/fleet.schema.json templates/config-repository/scripts/validate.py; do + [[ -f "$path/$required" ]] || return 1 + done + [[ -x "$path/templates/config-repository/scripts/validate.sh" ]] || return 1 + for unit in "${unit_names[@]}"; do [[ -f "$path/host/systemd/$unit" ]] || return 1; done + marker=$(<"$path/.ci-fleet-engine-ref") + [[ "$marker" == "$expected" ]] +} + +release_matches() { + runtime_release_complete "$release_dir" "$engine_ref" || return 1 + [[ -L "$current_link" ]] || return 1 + [[ $(readlink -f "$current_link") == $(readlink -f "$release_dir") ]] +} + +managed_images_match() { + local image provenance + local -a expected_images=() + mapfile -t expected_images < <(awk -F= '$1 == "CI_FLEET_CONTROLLER_IMAGE" || $1 == "CI_FLEET_RUNNER_IMAGE" {print substr($0, index($0, "=") + 1)}' "$candidate_env") + [[ ${#expected_images[@]} == 2 ]] || return 1 + for image in "${expected_images[@]}"; do + provenance=$(docker image inspect --format '{{ index .Config.Labels "org.opencontainers.image.revision" }}' "$image" 2>/dev/null) || return 1 + [[ "$provenance" == "$engine_ref" ]] || return 1 + done +} + +systemd_matches() { + local expected_manager unit + expected_manager=$manager_releases/$engine_ref + manager_release_complete "$expected_manager" "$engine_ref" || return 1 + [[ -L "$manager_current" ]] || return 1 + [[ $(readlink -f "$manager_current") == $(readlink -f "$expected_manager") ]] || return 1 + for unit in "${unit_names[@]}"; do + [[ -f "$systemd_dir/$unit" ]] || return 1 + cmp -s "$expected_manager/host/systemd/$unit" "$systemd_dir/$unit" || return 1 + done + for unit in "${timer_names[@]}"; do + systemctl is-enabled --quiet "$unit" || return 1 + systemctl is-active --quiet "$unit" || return 1 + done +} + +drift_count() { + local count=0 expected_owner=0 + [[ "$testing" != 1 ]] || expected_owner=$(id -u) + if [[ ! -f "$rendered_env" ]] \ + || [[ $(stat -c %u "$rendered_env") != "$expected_owner" ]] \ + || [[ $(stat -c %a "$rendered_env") != 600 ]] \ + || ! cmp -s "$candidate_env" "$rendered_env"; then + note 'DRIFT rendered_environment' + count=$((count + 1)) + fi + release_matches || { note 'DRIFT engine_release'; count=$((count + 1)); } + state_matches || { note 'DRIFT install_state'; count=$((count + 1)); } + runtime_matches "$target_state" || { note 'DRIFT controller_runtime'; count=$((count + 1)); } + if [[ "$target_state" != active && $(managed_runner_total_count) != 0 ]]; then + note 'DRIFT managed_runners' + count=$((count + 1)) + fi + managed_images_match || { note 'DRIFT managed_images'; count=$((count + 1)); } + systemd_matches || { note 'DRIFT maintenance_timers'; count=$((count + 1)); } + DRIFT_COUNT=$count +} + +atomic_replace_directory() { + local replacement=$1 target=$2 + if [[ ! -e "$target" && ! -L "$target" ]]; then + mv "$replacement" "$target" + return + fi + python3 - "$replacement" "$target" <<'PY' +import ctypes +import os +import sys + +replacement, target = map(os.fsencode, sys.argv[1:]) +libc = ctypes.CDLL(None, use_errno=True) +renameat2 = getattr(libc, "renameat2", None) +if renameat2 is None: + raise OSError("atomic directory exchange is unavailable") +renameat2.argtypes = [ctypes.c_int, ctypes.c_char_p, ctypes.c_int, ctypes.c_char_p, ctypes.c_uint] +if renameat2(-100, replacement, -100, target, 2) != 0: # AT_FDCWD, RENAME_EXCHANGE + error = ctypes.get_errno() + raise OSError(error, os.strerror(error), os.fsdecode(target)) +parent = os.open(os.path.dirname(target), os.O_RDONLY | os.O_DIRECTORY) +try: + os.fsync(parent) +finally: + os.close(parent) +PY +} + +install_release() { + local archive checkout resolved staged_release + if runtime_release_complete "$release_dir" "$engine_ref"; then + return + fi + install -d -m 0755 "$releases_dir" + archive=$temporary/engine.tar + if is_git_checkout "$repo_root" && [[ $(git -C "$repo_root" rev-parse 'HEAD^{commit}') == "$engine_ref" ]]; then + git -C "$repo_root" archive --format=tar --output "$archive" HEAD + else + [[ "$engine_repository" =~ ^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$ ]] || die 'delivery engine repository is invalid' + checkout=$temporary/engine-repository + git init -q "$checkout" + git -C "$checkout" remote add origin "https://github.com/${engine_repository}.git" + GIT_TERMINAL_PROMPT=0 git -C "$checkout" fetch -q --depth=1 origin "$engine_ref" || die 'pinned ci-fleet engine commit could not be fetched' + resolved=$(git -C "$checkout" rev-parse 'FETCH_HEAD^{commit}') + [[ "$resolved" == "$engine_ref" ]] || die 'fetched ci-fleet engine commit does not match desired state' + git -C "$checkout" archive --format=tar --output "$archive" FETCH_HEAD + fi + staged_release=$(mktemp -d "$releases_dir/.${engine_ref}.staging.XXXXXX") + staging_paths+=("$staged_release") + chmod 0755 "$staged_release" + tar -xf "$archive" -C "$staged_release" + printf '%s\n' "$engine_ref" >"$staged_release/.ci-fleet-engine-ref" + chmod 0644 "$staged_release/.ci-fleet-engine-ref" + release_tree_digest "$staged_release" >"$staged_release/.ci-fleet-tree-sha256" + chmod 0644 "$staged_release/.ci-fleet-tree-sha256" + runtime_release_complete "$staged_release" "$engine_ref" || die 'staged engine release is incomplete' + atomic_replace_directory "$staged_release" "$release_dir" +} + +install_manager() { + local manager_commit manager_release archive staged_manager + manager_commit=$engine_ref + [[ "$manager_commit" =~ ^[0-9a-f]{40}$ ]] || die 'installer manager commit is invalid' + runtime_release_complete "$release_dir" "$manager_commit" || die 'desired engine release is unavailable for installer manager activation' + manager_release=$manager_releases/$manager_commit + if ! manager_release_complete "$manager_release" "$manager_commit"; then + install -d -m 0755 "$manager_releases" + archive=$temporary/manager.tar + tar -cf "$archive" -C "$release_dir" . + staged_manager=$(mktemp -d "$manager_releases/.${manager_commit}.staging.XXXXXX") + staging_paths+=("$staged_manager") + chmod 0755 "$staged_manager" + tar -xf "$archive" -C "$staged_manager" + printf '%s\n' "$manager_commit" >"$staged_manager/.ci-fleet-engine-ref" + chmod 0644 "$staged_manager/.ci-fleet-engine-ref" + manager_release_complete "$staged_manager" "$manager_commit" || die 'staged installer manager release is incomplete' + atomic_replace_directory "$staged_manager" "$manager_release" + fi + install -d -m 0755 "$manager_root" + ln -sfn "$manager_release" "$temporary/manager-current" + mv -Tf "$temporary/manager-current" "$manager_current" +} + +run_candidate_preflight() { + ( + set -a + # shellcheck disable=SC1090 + . "$candidate_env" + set +a + CI_FLEET_TESTING=$testing "$release_dir/scripts/preflight.sh" --managed + ) +} + +build_candidate() { + run_candidate_preflight + compose "$release_dir" "$candidate_env" config --quiet + compose "$release_dir" "$candidate_env" build runner-image controller +} + +make_checkpoint() { + local timestamp target unit timer final_checkpoint staged_checkpoint + timestamp=$(date -u +%Y%m%dT%H%M%SZ) + final_checkpoint=$checkpoints_dir/${timestamp}-$$ + install -d -m 0700 "$checkpoints_dir" + staged_checkpoint=$(mktemp -d "$checkpoints_dir/.checkpoint.staging.XXXXXX") + staging_paths+=("$staged_checkpoint") + checkpoint_dir=$staged_checkpoint + install -d -m 0700 "$checkpoint_dir/systemd" + [[ ! -f "$rendered_env" ]] || install -m 0600 "$rendered_env" "$checkpoint_dir/ci-fleet.env" + [[ ! -f "$state_file" ]] || install -m 0600 "$state_file" "$checkpoint_dir/install-state.json" + target=$(current_runtime_release) + if [[ -n "$target" ]]; then + printf '%s\n' "$target" >"$checkpoint_dir/release-target" + chmod 0600 "$checkpoint_dir/release-target" + fi + if [[ -L "$manager_current" ]]; then + target=$(readlink -f "$manager_current") + printf '%s\n' "$target" >"$checkpoint_dir/manager-target" + chmod 0600 "$checkpoint_dir/manager-target" + fi + for unit in "${unit_names[@]}"; do + [[ ! -f "$systemd_dir/$unit" ]] || install -m 0644 "$systemd_dir/$unit" "$checkpoint_dir/systemd/$unit" + done + : >"$checkpoint_dir/enabled-timers" + : >"$checkpoint_dir/active-timers" + for timer in "${timer_names[@]}"; do + if systemctl is-enabled --quiet "$timer" 2>/dev/null; then printf '%s\n' "$timer" >>"$checkpoint_dir/enabled-timers"; fi + if systemctl is-active --quiet "$timer" 2>/dev/null; then printf '%s\n' "$timer" >>"$checkpoint_dir/active-timers"; fi + done + chmod 0600 "$checkpoint_dir/enabled-timers" "$checkpoint_dir/active-timers" + : >"$checkpoint_dir/.complete" + chmod 0600 "$checkpoint_dir/.complete" + mv "$checkpoint_dir" "$final_checkpoint" + checkpoint_dir=$final_checkpoint + note "CHECKPOINT_CREATED path=$checkpoint_dir" +} + +try_drain_current() { + local deadline count old_release status paused=false force_nonterminal=${1:-false} + local drain_env=${2:-$rendered_env} fallback_release=${3:-} shutdown_timeout=${CI_FLEET_DRAIN_TIMEOUT_SECONDS:-300} + drain_error= + status=$(controller_status) + case "$status" in + running|''|exited|created|dead) ;; + *) + if [[ "$force_nonterminal" != true ]]; then + drain_error="cannot safely drain controller in non-terminal state: $status" + return 1 + fi + [[ -f "$drain_env" ]] || { drain_error='cannot stop a non-terminal candidate without its rendered environment'; return 1; } + old_release=$(current_runtime_release) + [[ -n "$old_release" ]] || old_release=$fallback_release + [[ -n "$old_release" ]] || { drain_error='cannot stop a non-terminal candidate without its runtime release'; return 1; } + compose "$old_release" "$drain_env" stop --timeout "$shutdown_timeout" controller >/dev/null 2>&1 || { drain_error="failed to stop non-terminal candidate state: $status"; return 1; } + status= + ;; + esac + if [[ "$status" == running ]]; then + if [[ ! -f "$drain_env" ]]; then drain_error='cannot safely drain a running controller without its rendered environment'; return 1; fi + old_release=$(current_runtime_release) + [[ -n "$old_release" ]] || old_release=$fallback_release + if [[ -z "$old_release" || ! -f "$old_release/deploy/compose.yaml" ]]; then drain_error='cannot locate the running controller Compose release for safe adoption'; return 1; fi + if ! compose "$old_release" "$drain_env" pause controller >/dev/null; then drain_error='could not pause the controller for drain'; return 1; fi + paused=true + fi + deadline=$((SECONDS + ${CI_FLEET_DRAIN_TIMEOUT_SECONDS:-300})) + while :; do + count=$(managed_runner_count) + if [[ "$count" == 0 ]]; then break; fi + if ((SECONDS >= deadline)); then + if [[ "$paused" == true ]]; then compose "$old_release" "$drain_env" unpause controller >/dev/null || true; fi + drain_error="drain timed out with $count managed runner(s) still present" + return 1 + fi + sleep 2 + done + note 'DRAIN_READY managed_runners=0' + if [[ "$status" != running ]]; then + note 'DRAIN_OK managed_runners=0' + return 0 + fi + compose "$old_release" "$drain_env" kill --signal SIGTERM controller >/dev/null || { + compose "$old_release" "$drain_env" unpause controller >/dev/null 2>&1 || true + drain_error='failed to signal the paused controller for graceful scale-set cleanup' + return 1 + } + if [[ $(docker inspect --format '{{.State.Paused}}' "$controller_container" 2>/dev/null || true) == true ]]; then + compose "$old_release" "$drain_env" unpause controller >/dev/null || { + drain_error='failed to unpause the signaled controller for graceful shutdown' + return 1 + } + fi + compose "$old_release" "$drain_env" stop --timeout "$shutdown_timeout" controller >/dev/null || { + drain_error='could not stop the drained controller' + return 1 + } + note 'DRAIN_OK managed_runners=0' +} + +drain_current() { + try_drain_current || die "$drain_error" +} + +install_systemd_units() { + local source=${1:-$repo_root} + install -d -m 0755 "$systemd_dir" + install -m 0644 "$source/host/systemd/ci-fleet-health.service" "$systemd_dir/" + install -m 0644 "$source/host/systemd/ci-fleet-health.timer" "$systemd_dir/" + install -m 0644 "$source/host/systemd/ci-fleet-cleanup.service" "$systemd_dir/" + install -m 0644 "$source/host/systemd/ci-fleet-cleanup.timer" "$systemd_dir/" + install -m 0644 "$source/host/systemd/ci-fleet-drift.service" "$systemd_dir/" + install -m 0644 "$source/host/systemd/ci-fleet-drift.timer" "$systemd_dir/" + systemctl daemon-reload +} + +remove_systemd_units() { + systemctl disable --now "${timer_names[@]}" >/dev/null 2>&1 || true + local unit + for unit in "${unit_names[@]}"; do rm -f "$systemd_dir/$unit"; done + systemctl daemon-reload +} + +activate_candidate() { + local staged_state + install -d -m 0700 "$etc_dir" "$state_root" "$checkpoints_dir" + install -m 0600 "$candidate_env" "$rendered_env" + ln -sfn "$release_dir" "$temporary/current" + mv -Tf "$temporary/current" "$current_link" + install_manager + install_systemd_units "$(readlink -f "$manager_current")" + if [[ "$target_state" == active ]]; then + compose "$release_dir" "$rendered_env" up -d --no-deps controller + sleep "${CI_FLEET_STARTUP_WAIT_SECONDS:-2}" + runtime_matches active || die 'controller did not remain running after activation' + if ! ( + set -a + # shellcheck disable=SC1090 + . "$rendered_env" + set +a + "$release_dir/scripts/healthcheck.sh" + ); then + die 'post-activation health check failed' + fi + else + compose "$release_dir" "$rendered_env" stop controller >/dev/null 2>&1 || true + if ! runtime_matches "$target_state"; then + compose "$release_dir" "$rendered_env" down --remove-orphans >/dev/null + runtime_matches "$target_state" || die 'controller did not reach the requested non-active state' + fi + fi + staged_state=$(mktemp "$state_root/.install-state.XXXXXX") + staging_paths+=("$staged_state") + python3 - "$candidate_metadata" "$staged_state" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" <<'PY' +import json +import sys +value = json.load(open(sys.argv[1], encoding="utf-8")) +value["installed_at"] = sys.argv[3] +with open(sys.argv[2], "w", encoding="utf-8") as handle: + json.dump(value, handle, indent=2, sort_keys=True) + handle.write("\n") +PY + chmod 0600 "$staged_state" + mv -f "$staged_state" "$state_file" + systemctl enable --now "${timer_names[@]}" >/dev/null +} + +restore_systemd_snapshot() { + local unit timer failed=0 + remove_systemd_units || failed=1 + for unit in "${unit_names[@]}"; do + [[ ! -f "$checkpoint_dir/systemd/$unit" ]] || install -m 0644 "$checkpoint_dir/systemd/$unit" "$systemd_dir/$unit" || failed=1 + done + systemctl daemon-reload || failed=1 + for timer in "${timer_names[@]}"; do + if grep -Fxq "$timer" "$checkpoint_dir/enabled-timers"; then systemctl enable "$timer" >/dev/null || failed=1; else systemctl disable "$timer" >/dev/null 2>&1 || true; fi + if grep -Fxq "$timer" "$checkpoint_dir/active-timers"; then systemctl start "$timer" || failed=1; else systemctl stop "$timer" >/dev/null 2>&1 || true; fi + done + return "$failed" +} + +restore_checkpoint() { + local target restored_state failed=0 checkpoint_release='' drain_env=$rendered_env drain_release='' + [[ -n "$checkpoint_dir" && -d "$checkpoint_dir" ]] || return 1 + [[ ! -f "$checkpoint_dir/release-target" ]] || checkpoint_release=$(<"$checkpoint_dir/release-target") + drain_release=$(current_runtime_release) + if [[ -f "$rendered_env" ]]; then + load_installed_controller_identity "$temporary/no-install-state" "$rendered_env" + elif [[ -f "$checkpoint_dir/install-state.json" || -f "$checkpoint_dir/ci-fleet.env" ]]; then + load_installed_controller_identity "$checkpoint_dir/install-state.json" "$checkpoint_dir/ci-fleet.env" + drain_env=$checkpoint_dir/ci-fleet.env + [[ -n "$drain_release" ]] || drain_release=$checkpoint_release + fi + if ! try_drain_current true "$drain_env" "$drain_release"; then + note "ROLLBACK_FAILED reason=$drain_error" + return 1 + fi + if [[ -f "$checkpoint_dir/install-state.json" || -f "$checkpoint_dir/ci-fleet.env" ]]; then + load_installed_controller_identity "$checkpoint_dir/install-state.json" "$checkpoint_dir/ci-fleet.env" + fi + trap - ERR + set +e + if [[ -f "$checkpoint_dir/ci-fleet.env" ]]; then + install -m 0600 "$checkpoint_dir/ci-fleet.env" "$rendered_env" || failed=1 + else + rm -f "$rendered_env" || failed=1 + fi + if [[ -f "$checkpoint_dir/install-state.json" ]]; then + install -m 0600 "$checkpoint_dir/install-state.json" "$state_file" || failed=1 + else + rm -f "$state_file" || failed=1 + fi + if [[ -f "$checkpoint_dir/release-target" ]]; then + target=$(<"$checkpoint_dir/release-target") + if [[ -d "$target" && -f "$target/deploy/compose.yaml" ]]; then + ln -sfn "$target" "$temporary/rollback-current" && mv -Tf "$temporary/rollback-current" "$current_link" || failed=1 + release_dir=$target + else + failed=1 + fi + else + rm -f "$current_link" || failed=1 + release_dir= + fi + if [[ -f "$checkpoint_dir/manager-target" ]]; then + target=$(<"$checkpoint_dir/manager-target") + if [[ "$target" == "$manager_releases/"* && -d "$target" ]]; then + ln -sfn "$target" "$temporary/rollback-manager" && mv -Tf "$temporary/rollback-manager" "$manager_current" || failed=1 + else + failed=1 + fi + else + rm -f "$manager_current" || failed=1 + fi + restore_systemd_snapshot || failed=1 + if [[ -n "$release_dir" && -f "$rendered_env" ]]; then + restored_state=$(awk -F= '$1 == "CI_FLEET_CONTROLLER_STATE" {print $2}' "$rendered_env") || failed=1 + [[ "$restored_state" == active || "$restored_state" == drained || "$restored_state" == disabled ]] || failed=1 + if [[ "$restored_state" == active && "$failed" == 0 ]]; then + if [[ $(managed_runner_count) != 0 ]]; then + failed=1 + else + compose "$release_dir" "$rendered_env" up -d --no-deps controller || failed=1 + if ((failed == 0)); then + ( + set -a + # shellcheck disable=SC1090 + . "$rendered_env" + set +a + "$release_dir/scripts/healthcheck.sh" + ) || failed=1 + fi + fi + fi + fi + set -e + trap on_error ERR + if ((failed != 0)); then + note "ROLLBACK_FAILED checkpoint=$checkpoint_dir" + return 1 + fi + note "ROLLBACK_RESTORED checkpoint=$checkpoint_dir" +} + +on_error() { + local status=$? + if $transaction_active; then + restore_checkpoint || true + fi + exit "$status" +} +trap on_error ERR + +perform_check() { + local count + drift_count + count=$DRIFT_COUNT + if ((count > 0)); then + note "CHECK_FAILED drift=$count" + exit 3 + fi + note "CHECK_OK controller=$controller_id config_ref=$config_ref engine_ref=$engine_ref state=$target_state" +} + +perform_converge() { + local count existing_status desired_controller_id=$controller_id + if [[ "$mode" == upgrade && ! -f "$state_file" ]]; then + die '--upgrade requires an existing managed installation; use --install or --adopt' + fi + existing_status=$(controller_status) + if [[ "$mode" == adopt && ! -f "$rendered_env" && ! -f "$state_file" && -z "$existing_status" ]]; then + die '--adopt requires an existing controller or configuration; use --install for a fresh host' + fi + if [[ "$mode" == install && -f "$rendered_env" && ! -f "$state_file" ]]; then + die 'an unmanaged controller configuration exists; use --adopt' + fi + drift_count + count=$DRIFT_COUNT + if ((count == 0)); then + note "NO_CHANGE controller=$controller_id config_ref=$config_ref engine_ref=$engine_ref state=$target_state" + return + fi + install_release + make_checkpoint + transaction_active=true + if [[ -f "$state_file" || -f "$rendered_env" ]]; then + load_installed_controller_identity + elif [[ "$mode" == adopt ]]; then + die '--adopt requires a trusted installed controller identity' + fi + drain_current + controller_id=$desired_controller_id + [[ "$target_state" == active ]] || remove_inactive_managed_runners + build_candidate + activate_candidate + transaction_active=false + note "CONVERGED mode=$mode controller=$controller_id config_ref=$config_ref engine_ref=$engine_ref state=$target_state" +} + +latest_checkpoint() { + [[ -d "$checkpoints_dir" ]] || return 0 + { find "$checkpoints_dir" -mindepth 2 -maxdepth 2 -type f -name .complete ! -path "$checkpoints_dir/.checkpoint.staging.*/*" -printf '%T@ %h\n' 2>/dev/null || true; } | sort -nr | awk 'NR == 1 {print $2}' +} + +perform_rollback() { + checkpoint_dir=$(latest_checkpoint) + [[ -n "$checkpoint_dir" ]] || die 'no controller checkpoint is available' + load_installed_controller_identity "$checkpoint_dir/install-state.json" "$checkpoint_dir/ci-fleet.env" + restore_checkpoint || die 'checkpoint restoration failed' + note "ROLLBACK_OK checkpoint=$checkpoint_dir" +} + +perform_uninstall() { + local old_release= + load_installed_controller_identity + old_release=$(current_runtime_release) + make_checkpoint + transaction_active=true + drain_current + remove_inactive_managed_runners + if [[ -n "$old_release" && -f "$rendered_env" ]]; then + compose "$old_release" "$rendered_env" down --remove-orphans || true + ( + set -a + # shellcheck disable=SC1090 + . "$rendered_env" + set +a + "$old_release/scripts/cleanup.sh" --apply --instance "${CI_FLEET_INSTANCE:-}" || true + ) + fi + remove_systemd_units + rm -f "$current_link" "$rendered_env" "$state_file" + rm -f "$manager_current" + transaction_active=false + note "UNINSTALL_OK host_config_preserved=$host_config secrets_preserved=$etc_dir/secrets" +} + +require_commands +install -d -m 0755 "$(dirname "$lock_file")" +exec 9>"$lock_file" +flock -n 9 || die 'another ci-fleet installer or drift check is already running' +case "$mode" in + check|install|adopt|upgrade) + validate_common_arguments + resolve_config + validate_candidate_config_commit + prepare_host_config + verify_host_files + render_candidate + if [[ "$mode" == check ]]; then perform_check; else perform_converge; fi + ;; + rollback) + perform_rollback + ;; + uninstall) + perform_uninstall + ;; +esac diff --git a/scripts/preflight.sh b/scripts/preflight.sh index 7a418bd2..7751d516 100755 --- a/scripts/preflight.sh +++ b/scripts/preflight.sh @@ -3,6 +3,17 @@ set -Eeuo pipefail failures=0 warnings=0 +managed=false + +usage() { printf 'usage: %s [--managed]\n' "$0" >&2; } +while (($#)); do + case "$1" in + --managed) managed=true ;; + -h|--help) usage; exit 0 ;; + *) usage; exit 2 ;; + esac + shift +done ok() { printf 'OK %s\n' "$*"; } warn() { printf 'WARN %s\n' "$*"; warnings=$((warnings + 1)); } @@ -24,8 +35,14 @@ for name in "${required[@]}"; do if [[ -n "${!name:-}" ]]; then ok "$name is set"; else fail "$name is missing"; fi done -if [[ "${CI_FLEET_MIN_RUNNERS:-0}" != 0 ]]; then fail "CI_FLEET_MIN_RUNNERS must be 0 for the pilot"; fi -if [[ "${CI_FLEET_MAX_RUNNERS:-1}" != 1 ]]; then fail "CI_FLEET_MAX_RUNNERS must be 1 for the pilot"; fi +if [[ "${CI_FLEET_MIN_RUNNERS:-0}" != 0 ]]; then fail "CI_FLEET_MIN_RUNNERS must be 0"; fi +if $managed; then + if [[ ! "${CI_FLEET_MAX_RUNNERS:-}" =~ ^[0-9]+$ ]]; then + fail "CI_FLEET_MAX_RUNNERS must be a non-negative integer for a managed host" + fi +else + if [[ "${CI_FLEET_MAX_RUNNERS:-1}" != 1 ]]; then fail "CI_FLEET_MAX_RUNNERS must be 1 for the pilot"; fi +fi if [[ "${CI_FLEET_SCALE_SET_NAME:-}" != *"${CI_FLEET_INSTANCE:-__missing__}"* ]]; then fail "scale-set name must include the unique fleet instance ID" fi @@ -39,7 +56,12 @@ if command -v docker >/dev/null; then if docker compose version >/dev/null 2>&1; then ok "Docker Compose plugin is available"; else fail "Docker Compose plugin is unavailable"; fi fi -socket_gid=$(stat -c '%g' /var/run/docker.sock 2>/dev/null || true) +docker_socket=${CI_FLEET_DOCKER_SOCKET:-/var/run/docker.sock} +if [[ "${CI_FLEET_TESTING:-0}" == 1 && -n "${CI_FLEET_DOCKER_GID_OVERRIDE:-}" ]]; then + socket_gid=$CI_FLEET_DOCKER_GID_OVERRIDE +else + socket_gid=$(stat -c '%g' "$docker_socket" 2>/dev/null || true) +fi if [[ -n "$socket_gid" && "$socket_gid" == "${CI_FLEET_DOCKER_GID:-}" ]]; then ok "Docker socket group matches CI_FLEET_DOCKER_GID" else @@ -50,13 +72,16 @@ secret=${CI_FLEET_GITHUB_APP_PRIVATE_KEY_FILE:-} if [[ -f "$secret" ]]; then mode=$(stat -c '%a' "$secret") owner=$(stat -c '%u' "$secret") - if [[ "$mode" == 600 && "$owner" == 0 ]]; then ok "GitHub App PEM is root-owned mode 0600" - else fail "GitHub App PEM must be root-owned mode 0600 (found uid=$owner mode=$mode)"; fi + expected_owner=0 + [[ "${CI_FLEET_TESTING:-0}" != 1 ]] || expected_owner=$(id -u) + if [[ "$mode" == 600 && "$owner" == "$expected_owner" ]]; then ok "GitHub App PEM owner and mode are valid" + else fail "GitHub App PEM must be owned by the installer account with mode 0600 (found uid=$owner mode=$mode)"; fi else fail "GitHub App PEM file is missing" fi -disk_used=$(df -P /var/lib/docker 2>/dev/null | awk 'NR==2 {gsub(/%/, "", $5); print $5}') +docker_root=${CI_FLEET_DOCKER_ROOT:-/var/lib/docker} +disk_used=$(df -P "$docker_root" 2>/dev/null | awk 'NR==2 {gsub(/%/, "", $5); print $5}' || true) if [[ -z "$disk_used" ]]; then disk_used=$(df -P / | awk 'NR==2 {gsub(/%/, "", $5); print $5}'); fi if ((disk_used >= 80)); then fail "Docker filesystem is ${disk_used}% full"; else ok "Docker filesystem is ${disk_used}% full"; fi diff --git a/scripts/scan_committed_secrets.py b/scripts/scan_committed_secrets.py new file mode 100644 index 00000000..cd191118 --- /dev/null +++ b/scripts/scan_committed_secrets.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +"""Fail when tracked files contain high-confidence credential material.""" + +from __future__ import annotations + +import argparse +import re +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +PATTERN = re.compile( + rb"BEGIN (?:[A-Z0-9]+ )*PRIVATE KEY|" + rb"github_pat_[A-Za-z0-9_]{20,}|" + rb"gh[opusr]_[A-Za-z0-9]{20,}|" + rb"AKIA[0-9A-Z]{16}|" + rb"(?:postgres|mysql|mongodb(?:\+srv)?|redis)://[^\s/:]+:[^\s/@]+@" +) +assert all(PATTERN.search(b"BEGIN " + kind + b"PRIVATE KEY") for kind in (b"", b"RSA ", b"DSA ", b"EC ", b"OPENSSH ", b"ENCRYPTED ")) +assert all(PATTERN.search(prefix + b"x" * 20) for prefix in (b"gho_", b"ghp_", b"ghr_", b"ghs_", b"ghu_")) +assert PATTERN.search(b"AKIA" + b"A" * 16) +assert PATTERN.search(b"mysql" + b"://fixture-user:***@example.invalid/database") + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--repository", type=Path, default=ROOT) + parser.add_argument("--commit", help="immutable commit whose blobs should be scanned") + return parser.parse_args() + + +def main() -> int: + args = parse_args() + repository = args.repository.resolve() + if args.commit is not None and not re.fullmatch(r"[0-9a-f]{40}", args.commit): + print("--commit must be a full lowercase commit SHA", file=sys.stderr) + return 2 + listing = ["git", "-C", str(repository)] + listing += ["ls-tree", "-rz", "--name-only", args.commit] if args.commit else ["ls-files", "-z"] + tracked = subprocess.run( + listing, + check=True, + stdout=subprocess.PIPE, + ).stdout.split(b"\0") + findings: list[str] = [] + for raw in tracked: + if not raw: + continue + relative = raw.decode("utf-8") + if args.commit is None: + data = (repository / relative).read_bytes() + else: + data = subprocess.run( + ["git", "-C", str(repository), "cat-file", "blob", f"{args.commit}:{relative}"], + check=True, + stdout=subprocess.PIPE, + ).stdout + for match in PATTERN.finditer(data): + line = data.count(b"\n", 0, match.start()) + 1 + findings.append(f"{relative}:{line}") + if findings: + print("possible committed secret detected:", file=sys.stderr) + print("\n".join(findings), file=sys.stderr) + return 1 + print("OK: no high-confidence secret material in tracked files") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/test-capacity-preflight.sh b/scripts/test-capacity-preflight.sh index 12de1e91..83ef6554 100755 --- a/scripts/test-capacity-preflight.sh +++ b/scripts/test-capacity-preflight.sh @@ -234,11 +234,13 @@ if grep -Fq 'CAPACITY_TEST_SECRET_SHOULD_NOT_PRINT' <<<"$secret_output"; then fa xtrace_output=$(bash -x "$repo_root/scripts/capacity-preflight.sh" --phase post-change --target-max 2 2>&1) || fail 'xtrace secret-output fixture unexpectedly failed' if grep -Fq 'CAPACITY_TEST_SECRET_SHOULD_NOT_PRINT' <<<"$xtrace_output"; then fail 'capacity preflight exposed controller environment through inherited xtrace'; fi -for term in backup force-recreate --no-deps healthcheck retain restore rollback; do +for term in 'private desired state' --upgrade --config-repo --ref healthcheck retain restore checkpoint; do grep -Fqi -- "$term" "$repo_root/docs/CAPACITY-PROMOTION.md" || fail "capacity procedure is missing $term" done -grep -Fq 'env -i' "$repo_root/docs/CAPACITY-PROMOTION.md" || fail 'capacity procedure does not isolate Compose interpolation from stale shell values' -grep -Fq 'exec scripts/preflight.sh' "$repo_root/docs/CAPACITY-PROMOTION.md" || fail 'rollback does not run pilot preflight from a clean restored environment' +grep -Fq 'PREVIOUS_PRIVATE_CONFIGURATION_COMMIT' "$repo_root/docs/CAPACITY-PROMOTION.md" || fail 'capacity rollback does not apply the previous reviewed desired state' +[[ $(grep -Fc '. /etc/ci-fleet/ci-fleet.env' "$repo_root/docs/CAPACITY-PROMOTION.md") -ge 2 ]] || fail 'capacity procedure does not load rendered state for both preflight phases' +grep -Fq 'env -i' "$repo_root/docs/CAPACITY-PROMOTION.md" || fail 'capacity procedure does not isolate preflight from the caller environment' +if grep -Eq 'Edit only|force-recreate|ci-fleet\.env\.before-max2' "$repo_root/docs/CAPACITY-PROMOTION.md"; then fail 'capacity procedure still edits rendered host state'; fi grep -Fq 'scripts/capacity-preflight.sh' "$repo_root/docs/ADDING-A-HOST.md" || fail 'host guide does not link the capacity procedure' if grep -Riq --exclude='test-capacity-preflight.sh' 'docker system prune' "$repo_root/scripts"; then fail 'unrestricted prune exists in scripts'; fi diff --git a/scripts/test-install-worker-controller.sh b/scripts/test-install-worker-controller.sh new file mode 100755 index 00000000..d7c168d7 --- /dev/null +++ b/scripts/test-install-worker-controller.sh @@ -0,0 +1,574 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +tmp=$(mktemp -d) +trap 'rm -rf "$tmp"' EXIT +fake_bin=$tmp/bin +mkdir -p "$fake_bin" +export REAL_STAT +REAL_STAT=$(command -v stat) +export REAL_TAR +REAL_TAR=$(command -v tar) + +cat >"$fake_bin/docker" <<'EOF' +#!/usr/bin/env bash +set -u +state=${FAKE_DOCKER_STATE:?} +status_file=${FAKE_CONTROLLER_STATUS_FILE:-} +paused_state=${FAKE_PAUSED_STATE:-} +case "${1:-}" in + info) exit 0 ;; + inspect) + [[ -f "$state" ]] || exit 1 + if [[ "$*" == *'.Config.Env'* ]]; then + [[ -n "${FAKE_CONTROLLER_ENV_FILE:-}" && -f "$FAKE_CONTROLLER_ENV_FILE" ]] || exit 1 + cat "$FAKE_CONTROLLER_ENV_FILE" + elif [[ "$*" == *'org.opencontainers.image.revision'* ]]; then + if [[ -n "${FAKE_CONTROLLER_PROVENANCE_FILE:-}" && -f "$FAKE_CONTROLLER_PROVENANCE_FILE" ]]; then cat "$FAKE_CONTROLLER_PROVENANCE_FILE"; else printf '%s\n' "${FAKE_ENGINE_REF:?}"; fi + elif [[ "$*" == *'{{.Image}}'* ]]; then + [[ -n "${FAKE_CONTROLLER_IMAGE_ID_FILE:-}" && -f "$FAKE_CONTROLLER_IMAGE_ID_FILE" ]] || exit 1 + cat "$FAKE_CONTROLLER_IMAGE_ID_FILE" + elif [[ "$*" == *'.State.Status'* ]]; then + if [[ -n "$status_file" && -f "$status_file" ]]; then cat "$status_file"; else printf '%s\n' "${FAKE_CONTROLLER_STATUS:-running}"; fi + elif [[ "$*" == *'.State.Paused'* ]]; then + if [[ -n "$paused_state" && -f "$paused_state" ]]; then printf 'true\n'; else printf 'false\n'; fi + else + printf 'running\n' + fi + ;; + ps) + [[ -z "${FAKE_DOCKER_PS_LOG:-}" ]] || printf '%s\n' "$*" >>"$FAKE_DOCKER_PS_LOG" + if [[ "$*" == *'--all'* && -n "${FAKE_ALL_RUNNER_STATE:-}" && -f "$FAKE_ALL_RUNNER_STATE" ]]; then + printf 'managed-runner-all-state\n' + elif [[ -n "${FAKE_RUNNER_STATE_ONCE:-}" && -f "$FAKE_RUNNER_STATE_ONCE" ]]; then + rm -f "$FAKE_RUNNER_STATE_ONCE" + printf 'managed-runner\n' + elif [[ -n "${FAKE_RUNNER_STATE:-}" && -f "$FAKE_RUNNER_STATE" ]]; then + printf 'managed-runner\n' + fi + exit 0 + ;; + image) + [[ "${2:-}" == inspect ]] || exit 1 + image=${!#} + [[ -z "${FAKE_IMAGE_INSPECT_LOG:-}" ]] || printf '%s\n' "$image" >>"$FAKE_IMAGE_INSPECT_LOG" + if [[ "$image" == "${FAKE_RUNNER_IMAGE:-}" ]]; then + image_state=${FAKE_RUNNER_IMAGE_STATE:-} + elif [[ "$image" == "${FAKE_CONTROLLER_IMAGE:-}" ]]; then + image_state=${FAKE_CONTROLLER_IMAGE_STATE:-} + else + exit 1 + fi + [[ -f "$image_state" ]] || exit 1 + if [[ "$*" == *'{{.Id}}'* ]]; then printf 'sha256:%s\n' "$(<"$image_state")"; else cat "$image_state"; fi + ;; + rm) + (($# >= 2)) || exit 1 + [[ -z "${FAKE_ALL_RUNNER_STATE:-}" ]] || rm -f "$FAKE_ALL_RUNNER_STATE" + ;; + volume|network) + [[ "${2:-}" == ls ]] && exit 0 + [[ "${2:-}" == inspect ]] && exit 1 + exit 0 + ;; + compose) + if [[ "${2:-}" == version ]]; then exit 0; fi + [[ -z "${COMPOSE_PROJECT_NAME:-}" && -z "${CI_FLEET_MAX_RUNNERS:-}" ]] || exit 44 + command= env_file= previous= + for argument in "$@"; do + [[ "$previous" != --env-file ]] || env_file=$argument + case "$argument" in config|build|up|stop|pause|unpause|kill|down|logs|rm) command=$argument ;; esac + previous=$argument + done + if [[ -n "${FAKE_COMPOSE_LOG:-}" ]]; then + instance= + [[ ! -f "$env_file" ]] || instance=$(awk -F= '$1 == "CI_FLEET_INSTANCE" {print $2}' "$env_file") + printf '%s|%s|%s|%s\n' "$command" "$env_file" "$instance" "$*" >>"$FAKE_COMPOSE_LOG" + fi + case "$command" in + up) + if [[ -n "${FAKE_FAIL_UP_ONCE:-}" && -f "$FAKE_FAIL_UP_ONCE" ]]; then + rm -f "$FAKE_FAIL_UP_ONCE" + exit 42 + fi + : >"$state" + [[ -z "${FAKE_CONTROLLER_PROVENANCE_FILE:-}" ]] || printf '%s\n' "${FAKE_ENGINE_REF:?}" >"$FAKE_CONTROLLER_PROVENANCE_FILE" + [[ -z "${FAKE_CONTROLLER_IMAGE_ID_FILE:-}" ]] || printf 'sha256:%s\n' "${FAKE_ENGINE_REF:?}" >"$FAKE_CONTROLLER_IMAGE_ID_FILE" + [[ -z "${FAKE_CONTROLLER_ENV_FILE:-}" ]] || cp "$env_file" "$FAKE_CONTROLLER_ENV_FILE" + if [[ -n "${FAKE_RESTART_AFTER_UP:-}" && -f "$FAKE_RESTART_AFTER_UP" ]]; then + rm -f "$FAKE_RESTART_AFTER_UP" + printf 'restarting\n' >"$status_file" + elif [[ -n "$status_file" ]]; then + rm -f "$status_file" + fi + ;; + stop) + if [[ -n "${FAKE_STOP_FAIL:-}" && -f "$FAKE_STOP_FAIL" ]]; then exit 42; fi + rm -f "$state"; [[ -z "$status_file" ]] || rm -f "$status_file"; [[ -z "$paused_state" ]] || rm -f "$paused_state"; [[ -z "${FAKE_CONTROLLER_PROVENANCE_FILE:-}" ]] || rm -f "$FAKE_CONTROLLER_PROVENANCE_FILE"; [[ -z "${FAKE_CONTROLLER_IMAGE_ID_FILE:-}" ]] || rm -f "$FAKE_CONTROLLER_IMAGE_ID_FILE"; [[ -z "${FAKE_CONTROLLER_ENV_FILE:-}" ]] || rm -f "$FAKE_CONTROLLER_ENV_FILE" + ;; + down|rm) rm -f "$state"; [[ -z "$status_file" ]] || rm -f "$status_file"; [[ -z "$paused_state" ]] || rm -f "$paused_state"; [[ -z "${FAKE_CONTROLLER_PROVENANCE_FILE:-}" ]] || rm -f "$FAKE_CONTROLLER_PROVENANCE_FILE"; [[ -z "${FAKE_CONTROLLER_IMAGE_ID_FILE:-}" ]] || rm -f "$FAKE_CONTROLLER_IMAGE_ID_FILE"; [[ -z "${FAKE_CONTROLLER_ENV_FILE:-}" ]] || rm -f "$FAKE_CONTROLLER_ENV_FILE" ;; + pause) + [[ -z "$paused_state" ]] || : >"$paused_state" + [[ -z "${FAKE_RUNNER_STATE:-}" ]] || rm -f "$FAKE_RUNNER_STATE" + ;; + unpause) [[ -z "$paused_state" ]] || rm -f "$paused_state" ;; + kill) + if [[ -n "${FAKE_FAIL_KILL_ONCE:-}" && -f "$FAKE_FAIL_KILL_ONCE" ]]; then + rm -f "$FAKE_FAIL_KILL_ONCE" + exit 43 + fi + [[ -z "$paused_state" ]] || rm -f "$paused_state" + ;; + build) + [[ -z "${FAKE_RUNNER_IMAGE_STATE:-}" ]] || printf '%s\n' "${FAKE_ENGINE_REF:?}" >"$FAKE_RUNNER_IMAGE_STATE" + [[ -z "${FAKE_CONTROLLER_IMAGE_STATE:-}" ]] || printf '%s\n' "${FAKE_ENGINE_REF:?}" >"$FAKE_CONTROLLER_IMAGE_STATE" + ;; + config|logs) ;; + *) exit 1 ;; + esac + ;; + *) exit 1 ;; +esac +EOF + +cat >"$fake_bin/systemctl" <<'EOF' +#!/usr/bin/env bash +if [[ "${1:-}" == enable && "${2:-}" == --now && ! -f "${CI_FLEET_ROOT_PREFIX:-}/var/lib/ci-fleet/install-state.json" ]]; then + exit 98 +fi +if [[ -n "${FAKE_DISABLED_TIMER:-}" && ( "${1:-}" == is-enabled || "${1:-}" == is-active ) && $# == 3 && "${3:-}" == "$FAKE_DISABLED_TIMER" ]]; then + exit 1 +fi +exit 0 +EOF + +cat >"$fake_bin/stat" <<'EOF' +#!/usr/bin/env bash +if [[ -n "${FAKE_WRONG_HOST_CONFIG_OWNER:-}" && "${1:-}" == -c && "${2:-}" == %u && "${3:-}" == "$FAKE_WRONG_HOST_CONFIG_OWNER" ]]; then + printf '99999\n' + exit 0 +fi +if [[ -n "${FAKE_WRONG_INSTALL_STATE_OWNER:-}" && "${1:-}" == -c && "${2:-}" == %u && "${3:-}" == "$FAKE_WRONG_INSTALL_STATE_OWNER" ]]; then + printf '99999\n' + exit 0 +fi +exec "$REAL_STAT" "$@" +EOF +chmod 700 "$fake_bin/docker" "$fake_bin/systemctl" "$fake_bin/stat" + +cat >"$fake_bin/tar" <<'EOF' +#!/usr/bin/env bash +if [[ -n "${FAKE_FAIL_TAR_ONCE:-}" && -f "$FAKE_FAIL_TAR_ONCE" ]]; then + rm -f "$FAKE_FAIL_TAR_ONCE" + exit 45 +fi +exec "$REAL_TAR" "$@" +EOF +chmod 700 "$fake_bin/tar" + +export PATH="$fake_bin:$PATH" +export FAKE_DOCKER_STATE=$tmp/docker-controller-running +export FAKE_CONTROLLER_STATUS_FILE=$tmp/docker-controller-status +export FAKE_PAUSED_STATE=$tmp/docker-controller-paused +export FAKE_CONTROLLER_PROVENANCE_FILE=$tmp/docker-controller-provenance +export FAKE_CONTROLLER_IMAGE_ID_FILE=$tmp/docker-controller-image-id +export FAKE_CONTROLLER_ENV_FILE=$tmp/docker-controller-env +export FAKE_DOCKER_PS_LOG=$tmp/docker-ps.log +export CI_FLEET_TESTING=1 +export CI_FLEET_DOCKER_GID_OVERRIDE=998 +export CI_FLEET_STARTUP_WAIT_SECONDS=0 +export CI_FLEET_DRAIN_TIMEOUT_SECONDS=2 +export COMPOSE_PROJECT_NAME=caller-controlled-project +export CI_FLEET_MAX_RUNNERS=999 + +fail() { printf 'FAIL %s\n' "$*" >&2; exit 1; } +expect_success() { + local output + output=$("$@" 2>&1) || fail "expected success: $*; output=$output" + printf '%s\n' "$output" +} +expect_failure() { + local expected=$1 output + shift + if output=$("$@" 2>&1); then fail "expected failure: $*"; fi + grep -Fq -- "$expected" <<<"$output" || fail "missing failure [$expected]: $output" +} +expect_command_failure() { + local output + if output=$("$@" 2>&1); then fail "expected failure: $*"; fi +} + +engine_ref=$(git -C "$repo_root" rev-parse 'HEAD^{commit}') +export FAKE_ENGINE_REF=$engine_ref +runner_image="ci-fleet-runner:${engine_ref:0:12}" +export FAKE_RUNNER_IMAGE=$runner_image +export FAKE_CONTROLLER_IMAGE=ci-fleet-controller:${engine_ref:0:12} +export FAKE_RUNNER_IMAGE_STATE=$tmp/runner-image-present +export FAKE_CONTROLLER_IMAGE_STATE=$tmp/controller-image-present +export FAKE_IMAGE_INSPECT_LOG=$tmp/image-inspects +for dockerfile in "$repo_root/controller/Dockerfile" "$repo_root/runner/Dockerfile"; do + grep -Fq "LABEL org.opencontainers.image.revision=\"\${CI_FLEET_COMMIT}\"" "$dockerfile" || fail "managed image lacks engine provenance label: $dockerfile" +done +grep -Fq "CI_FLEET_COMMIT: \${CI_FLEET_COMMIT:-unknown}" "$repo_root/deploy/compose.yaml" || fail 'runner build lacks engine provenance argument' +config_repo=$tmp/config-repo +git init -q "$config_repo" +git -C "$config_repo" config user.name fixture +git -C "$config_repo" config user.email fixture@example.invalid + +write_config() { + local state=$1 maximum=$2 budget=$3 + python3 - "$repo_root/templates/config-repository/fleet.json" "$config_repo/fleet.json" "$engine_ref" "$state" "$maximum" "$budget" <<'PY' +import json +import sys +source, target, engine_ref, state, maximum, budget = sys.argv[1:] +value = json.load(open(source, encoding="utf-8")) +value["organization"]["slug"] = "fixture-org" +value["runner_pools"]["trusted-ci"]["allowed_repositories"] = ["fixture-org/example-app"] +value["projects"]["example-app"]["repository"] = "fixture-org/example-app" +controller = value["controllers"]["example-ci-01"] +controller["engine_ref"] = engine_ref +controller["state"] = state +controller["max_runners"] = int(maximum) +value["runner_pools"]["trusted-ci"]["capacity_budget"] = int(budget) +with open(target, "w", encoding="utf-8") as handle: + json.dump(value, handle, indent=2) + handle.write("\n") +PY + git -C "$config_repo" add fleet.json + git -C "$config_repo" commit -q -m "fixture $state $maximum" + git -C "$config_repo" rev-parse HEAD +} + +root=$tmp/host +export CI_FLEET_ROOT_PREFIX=$root +export CI_FLEET_DOCKER_ROOT=$root/var/lib/docker +mkdir -p "$root/etc/ci-fleet/secrets" "$CI_FLEET_DOCKER_ROOT" +pem=$root/etc/ci-fleet/secrets/github-app.pem +printf 'fixture only\n' >"$pem" +chmod 600 "$pem" +host_config=$root/etc/ci-fleet/host.env +printf '%s\n' \ + 'CI_FLEET_GITHUB_APP_CLIENT_ID=Iv1.EXAMPLE' \ + 'CI_FLEET_GITHUB_APP_INSTALLATION_ID=123456' \ + "CI_FLEET_GITHUB_APP_PRIVATE_KEY_FILE=$pem" \ + 'CI_FLEET_RUNNER_TTL=6h' >"$host_config" +chmod 600 "$host_config" + +ref_one=$(write_config active 1 1) +printf 'fixture only\n' >"$config_repo/.env" +git -C "$config_repo" add -f .env +git -C "$config_repo" commit -q -m 'forbidden config path fixture' +forbidden_ref=$(git -C "$config_repo" rev-parse HEAD) +git -C "$config_repo" reset -q --hard "$ref_one" +printf 'ghp_%020d\n' 0 >"$config_repo/README.md" +git -C "$config_repo" add README.md +git -C "$config_repo" commit -q -m 'forbidden config content fixture' +secret_ref=$(git -C "$config_repo" rev-parse HEAD) +git -C "$config_repo" reset -q --hard "$ref_one" +installer=$repo_root/scripts/install-worker-controller.sh +base_args=(--config-repo "$config_repo" --controller example-ci-01) + +staged_checkpoint="$root/var/lib/ci-fleet/checkpoints/.checkpoint.staging.interrupted" +mkdir -p "$staged_checkpoint" +: >"$staged_checkpoint/.complete" +expect_failure 'no controller checkpoint is available' "$installer" --rollback +rm -rf "$staged_checkpoint" +expect_failure 'secret-bearing files are forbidden' "$installer" --check "${base_args[@]}" --ref "$forbidden_ref" +expect_failure 'possible committed secret detected' "$installer" --check "${base_args[@]}" --ref "$secret_ref" +export FAKE_WRONG_HOST_CONFIG_OWNER=$host_config +expect_failure 'host configuration must be owned by root' "$installer" --install "${base_args[@]}" --ref "$ref_one" +unset FAKE_WRONG_HOST_CONFIG_OWNER +expect_failure 'managed installs require the default' "$installer" --check "${base_args[@]}" --ref "$ref_one" --host-config "$tmp/custom-host.env" + +first=$(expect_success "$installer" --install "${base_args[@]}" --ref "$ref_one") +grep -Fq 'CONVERGED mode=install' <<<"$first" || fail 'fresh install did not converge' +[[ -L "$root/opt/ci-fleet/current" && -f "$root/var/lib/ci-fleet/install-state.json" ]] || fail 'fresh install state is incomplete' +[[ $(readlink -f "$root/opt/ci-fleet/manager/current") == "$root/opt/ci-fleet/manager/releases/$engine_ref" ]] || fail 'installer manager did not activate the desired engine release' +[[ -f "$FAKE_DOCKER_STATE" ]] || fail 'active controller was not started' +install_state=$root/var/lib/ci-fleet/install-state.json +chmod 644 "$install_state" +expect_failure 'install state must be owned by root with mode 0600' env CI_FLEET_INSTALL_STATE_FILE="$install_state" CI_FLEET_INSTALLER="$installer" "$repo_root/scripts/check-installed-state.sh" +expect_failure 'DRIFT install_state' "$installer" --check "${base_args[@]}" --ref "$ref_one" +expect_success "$installer" --install "${base_args[@]}" --ref "$ref_one" >/dev/null +[[ $(stat -c %a "$install_state") == 600 ]] || fail 'convergence did not repair install-state mode' +rendered_env=$root/etc/ci-fleet/ci-fleet.env +chmod 644 "$rendered_env" +expect_failure 'DRIFT rendered_environment' "$installer" --check "${base_args[@]}" --ref "$ref_one" +expect_success "$installer" --install "${base_args[@]}" --ref "$ref_one" >/dev/null +[[ $(stat -c %a "$rendered_env") == 600 ]] || fail 'convergence did not repair rendered-environment mode' +export FAKE_WRONG_INSTALL_STATE_OWNER=$install_state +expect_failure 'install state must be owned by root with mode 0600' env CI_FLEET_INSTALL_STATE_FILE="$install_state" CI_FLEET_INSTALLER="$installer" "$repo_root/scripts/check-installed-state.sh" +unset FAKE_WRONG_INSTALL_STATE_OWNER +expect_success env CI_FLEET_INSTALL_STATE_FILE="$install_state" CI_FLEET_INSTALLER="$installer" "$repo_root/scripts/check-installed-state.sh" >/dev/null + +export FAKE_DISABLED_TIMER=ci-fleet-cleanup.timer +expect_failure 'DRIFT maintenance_timers' "$installer" --check "${base_args[@]}" --ref "$ref_one" +unset FAKE_DISABLED_TIMER + +export FAKE_CONTROLLER_STATUS=restarting +expect_failure 'cannot safely drain controller in non-terminal state: restarting' "$installer" --uninstall +unset FAKE_CONTROLLER_STATUS + +mv "$host_config" "$host_config.missing" +expect_failure 'host-local GitHub App configuration is missing' "$installer" --check "${base_args[@]}" --ref "$ref_one" +mv "$host_config.missing" "$host_config" + +second=$(expect_success "$installer" --install "${base_args[@]}" --ref "$ref_one") +grep -Fq 'NO_CHANGE' <<<"$second" || fail 'idempotent rerun changed the host' +expect_success "$installer" --check "${base_args[@]}" --ref "$ref_one" >/dev/null +python3 - "$install_state" <<'PY' +import json +import sys +path = sys.argv[1] +value = json.load(open(path, encoding="utf-8")) +value["controller"] = "legacy-ci-01" +with open(path, "w", encoding="utf-8") as handle: + json.dump(value, handle) + handle.write("\n") +PY +python3 -c 'from pathlib import Path; import sys; path = Path(sys.argv[1]); path.write_text(path.read_text().replace("CI_FLEET_INSTANCE=example-ci-01", "CI_FLEET_INSTANCE=legacy-ci-01"))' "$rendered_env" +chmod 600 "$install_state" "$rendered_env" +[[ $(stat -c '%u:%a' "$install_state") == "$(id -u):600" ]] || fail 'installed-identity fixture metadata is invalid' +export FAKE_RUNNER_STATE_ONCE=$tmp/repeat-install-managed-runner +: >"$FAKE_RUNNER_STATE_ONCE" +: >"$FAKE_DOCKER_PS_LOG" +expect_success "$installer" --install "${base_args[@]}" --ref "$ref_one" >/dev/null +grep -Fq 'label=io.randomdevelopment.ci-fleet.instance=legacy-ci-01' "$FAKE_DOCKER_PS_LOG" || fail 'repeat install did not drain the installed controller identity' +grep -Fq 'CI_FLEET_INSTANCE=example-ci-01' "$rendered_env" || fail 'repeat install did not restore the desired controller identity' +unset FAKE_RUNNER_STATE_ONCE +printf '%040d\n' 0 >"$FAKE_CONTROLLER_PROVENANCE_FILE" +expect_failure 'DRIFT controller_runtime' "$installer" --check "${base_args[@]}" --ref "$ref_one" +expect_success "$installer" --install "${base_args[@]}" --ref "$ref_one" >/dev/null +[[ $(<"$FAKE_CONTROLLER_PROVENANCE_FILE") == "$engine_ref" ]] || fail 'controller convergence did not restore running image provenance' +printf 'sha256:%040d\n' 0 >"$FAKE_CONTROLLER_IMAGE_ID_FILE" +expect_failure 'DRIFT controller_runtime' "$installer" --check "${base_args[@]}" --ref "$ref_one" +expect_success "$installer" --install "${base_args[@]}" --ref "$ref_one" >/dev/null +[[ $(<"$FAKE_CONTROLLER_IMAGE_ID_FILE") == "sha256:$engine_ref" ]] || fail 'controller convergence did not restore live image identity' +python3 -c 'from pathlib import Path; import sys; path = Path(sys.argv[1]); path.write_text(path.read_text().replace("CI_FLEET_MAX_RUNNERS=1", "CI_FLEET_MAX_RUNNERS=9"))' "$FAKE_CONTROLLER_ENV_FILE" +grep -Fxq 'CI_FLEET_MAX_RUNNERS=9' "$FAKE_CONTROLLER_ENV_FILE" || fail 'live-environment fixture did not mutate' +expect_failure 'DRIFT controller_runtime' "$installer" --check "${base_args[@]}" --ref "$ref_one" +expect_success "$installer" --install "${base_args[@]}" --ref "$ref_one" >/dev/null +grep -Fxq 'CI_FLEET_MAX_RUNNERS=1' "$FAKE_CONTROLLER_ENV_FILE" || fail 'controller convergence did not restore live environment' + +rm -f "$FAKE_RUNNER_IMAGE_STATE" +expect_failure 'DRIFT managed_images' "$installer" --check "${base_args[@]}" --ref "$ref_one" +image_repair=$(expect_success "$installer" --install "${base_args[@]}" --ref "$ref_one") +grep -Fq 'CONVERGED mode=install' <<<"$image_repair" || fail 'missing runner image did not trigger convergence' +[[ -f "$FAKE_RUNNER_IMAGE_STATE" && -f "$FAKE_CONTROLLER_IMAGE_STATE" ]] || fail 'candidate build did not restore both managed images' +rm -f "$FAKE_CONTROLLER_IMAGE_STATE" +expect_failure 'DRIFT managed_images' "$installer" --check "${base_args[@]}" --ref "$ref_one" +expect_success "$installer" --install "${base_args[@]}" --ref "$ref_one" >/dev/null +[[ -f "$FAKE_RUNNER_IMAGE_STATE" && -f "$FAKE_CONTROLLER_IMAGE_STATE" ]] || fail 'candidate build did not restore the controller image' +printf '%040d\n' 0 >"$FAKE_RUNNER_IMAGE_STATE" +expect_failure 'DRIFT managed_images' "$installer" --check "${base_args[@]}" --ref "$ref_one" +expect_success "$installer" --install "${base_args[@]}" --ref "$ref_one" >/dev/null +[[ $(<"$FAKE_RUNNER_IMAGE_STATE") == "$engine_ref" && $(<"$FAKE_CONTROLLER_IMAGE_STATE") == "$engine_ref" ]] || fail 'candidate build did not restore managed image provenance' +if docker image inspect unrelated:image >/dev/null 2>&1; then fail 'unrelated image fixture unexpectedly exists'; fi +: >"$FAKE_IMAGE_INSPECT_LOG" +expect_success "$installer" --check "${base_args[@]}" --ref "$ref_one" >/dev/null +grep -Fxq "$FAKE_RUNNER_IMAGE" "$FAKE_IMAGE_INSPECT_LOG" || fail 'runner image was not inspected' +grep -Fxq "$FAKE_CONTROLLER_IMAGE" "$FAKE_IMAGE_INSPECT_LOG" || fail 'controller image was not inspected' +if grep -Fvx -e "$FAKE_RUNNER_IMAGE" -e "$FAKE_CONTROLLER_IMAGE" "$FAKE_IMAGE_INSPECT_LOG" >/dev/null; then fail 'an unrelated image was inspected'; fi + +active_release=$(readlink -f "$root/opt/ci-fleet/current") +unrelated_release=$root/opt/ci-fleet/releases/unrelated-release +mkdir -p "$unrelated_release" +: >"$unrelated_release/preserve" +mv "$active_release/deploy/compose.yaml" "$active_release/deploy/compose.yaml.missing" +expect_failure 'DRIFT engine_release' "$installer" --check "${base_args[@]}" --ref "$ref_one" +expect_success "$installer" --install "${base_args[@]}" --ref "$ref_one" >/dev/null +[[ -f "$active_release/deploy/compose.yaml" && -f "$unrelated_release/preserve" ]] || fail 'Compose repair removed unrelated release state' +rm -f "$active_release/scripts/preflight.sh" +expect_failure 'DRIFT engine_release' "$installer" --check "${base_args[@]}" --ref "$ref_one" +expect_success "$installer" --install "${base_args[@]}" --ref "$ref_one" >/dev/null +[[ -x "$active_release/scripts/preflight.sh" ]] || fail 'required runtime script was not repaired' +rm -f "$active_release/controller/main.go" "$active_release/runner/Dockerfile" +expect_failure 'DRIFT engine_release' "$installer" --check "${base_args[@]}" --ref "$ref_one" +expect_success "$installer" --install "${base_args[@]}" --ref "$ref_one" >/dev/null +[[ -f "$active_release/controller/main.go" && -f "$active_release/runner/Dockerfile" ]] || fail 'runtime build inputs were not repaired' +rm -f "$active_release/.ci-fleet-engine-ref" +expect_failure 'DRIFT engine_release' "$installer" --check "${base_args[@]}" --ref "$ref_one" +expect_success "$installer" --install "${base_args[@]}" --ref "$ref_one" >/dev/null +[[ $(<"$active_release/.ci-fleet-engine-ref") == "$engine_ref" ]] || fail 'missing release marker was not repaired' +if [[ ${engine_ref: -1} == 0 ]]; then bad_marker="${engine_ref%?}1"; else bad_marker="${engine_ref%?}0"; fi +printf '%s\n' "$bad_marker" >"$active_release/.ci-fleet-engine-ref" +expect_failure 'DRIFT engine_release' "$installer" --check "${base_args[@]}" --ref "$ref_one" +expect_success "$installer" --install "${base_args[@]}" --ref "$ref_one" >/dev/null +[[ $(<"$active_release/.ci-fleet-engine-ref") == "$engine_ref" ]] || fail 'bad release marker was not repaired' +printf '\n# tampered runtime fixture\n' >>"$active_release/scripts/preflight.sh" +expect_failure 'DRIFT engine_release' "$installer" --check "${base_args[@]}" --ref "$ref_one" +expect_success "$installer" --install "${base_args[@]}" --ref "$ref_one" >/dev/null +if grep -Fq 'tampered runtime fixture' "$active_release/scripts/preflight.sh"; then fail 'modified runtime release was reused'; fi +rm -f "$active_release/deploy/compose.yaml" +export FAKE_FAIL_TAR_ONCE=$tmp/fail-tar-once +: >"$FAKE_FAIL_TAR_ONCE" +expect_command_failure "$installer" --install "${base_args[@]}" --ref "$ref_one" +unset FAKE_FAIL_TAR_ONCE +[[ ! -f "$active_release/deploy/compose.yaml" ]] || fail 'interrupted repair replaced the detectable prior state' +if compgen -G "$root/opt/ci-fleet/releases/.${engine_ref}.staging.*" >/dev/null; then fail 'interrupted release staging was not cleaned'; fi +expect_success "$installer" --install "${base_args[@]}" --ref "$ref_one" >/dev/null +manager_release=$(readlink -f "$root/opt/ci-fleet/manager/current") +printf '\n# tampered manager fixture\n' >>"$manager_release/scripts/check-installed-state.sh" +expect_failure 'DRIFT maintenance_timers' "$installer" --check "${base_args[@]}" --ref "$ref_one" +expect_success "$installer" --install "${base_args[@]}" --ref "$ref_one" >/dev/null +if grep -Fq 'tampered manager fixture' "$manager_release/scripts/check-installed-state.sh"; then fail 'modified manager release was reused'; fi +rm -f "$manager_release/scripts/check-installed-state.sh" +expect_failure 'DRIFT maintenance_timers' "$installer" --check "${base_args[@]}" --ref "$ref_one" +expect_success "$installer" --install "${base_args[@]}" --ref "$ref_one" >/dev/null +[[ -x "$manager_release/scripts/check-installed-state.sh" ]] || fail 'incomplete manager release was not repaired consistently' +rm -f "$manager_release/scripts/desired_state.py" "$manager_release/templates/config-repository/fleet.schema.json" +expect_failure 'DRIFT maintenance_timers' "$installer" --check "${base_args[@]}" --ref "$ref_one" +expect_success "$installer" --install "${base_args[@]}" --ref "$ref_one" >/dev/null +[[ -f "$manager_release/scripts/desired_state.py" && -f "$manager_release/templates/config-repository/fleet.schema.json" ]] || fail 'manager helper inputs were not repaired' +mv "$active_release" "$active_release.saved" +expect_failure 'DRIFT engine_release' "$installer" --check "${base_args[@]}" --ref "$ref_one" +mv "$active_release.saved" "$active_release" + +relative=$(cd "$tmp" && expect_success "$installer" --install --config-repo config-repo --controller example-ci-01 --ref "$ref_one") +grep -Fq 'NO_CHANGE' <<<"$relative" || fail 'relative configuration path was not normalized before drift comparison' +grep -Fq "CI_FLEET_CONFIG_REPOSITORY=$config_repo" "$root/etc/ci-fleet/ci-fleet.env" || fail 'rendered configuration path is not absolute' + +printf '\n' >>"$root/etc/ci-fleet/ci-fleet.env" +printf '\n# drift\n' >>"$root/etc/systemd/system/ci-fleet-health.timer" +expect_failure 'DRIFT rendered_environment' "$installer" --check "${base_args[@]}" --ref "$ref_one" +expect_failure 'DRIFT maintenance_timers' "$installer" --check "${base_args[@]}" --ref "$ref_one" +complete_release_inode=$(stat -c '%i' "$active_release") +expect_success "$installer" --install "${base_args[@]}" --ref "$ref_one" >/dev/null +[[ $(stat -c '%i' "$active_release") == "$complete_release_inode" ]] || fail 'complete immutable release was replaced instead of reused' + +prior_manager=$root/opt/ci-fleet/manager/releases/prior-manager +cp -a "$(readlink -f "$root/opt/ci-fleet/manager/current")" "$prior_manager" +ln -sfn "$prior_manager" "$root/opt/ci-fleet/manager/current" +expect_failure 'DRIFT maintenance_timers' "$installer" --check "${base_args[@]}" --ref "$ref_one" + +ref_two=$(write_config active 2 2) +export FAKE_FAIL_KILL_ONCE=$tmp/fail-kill-once +: >"$FAKE_FAIL_KILL_ONCE" +expect_failure 'failed to signal the paused controller' "$installer" --upgrade "${base_args[@]}" --ref "$ref_two" +unset FAKE_FAIL_KILL_ONCE +[[ ! -f "$FAKE_PAUSED_STATE" && -f "$FAKE_DOCKER_STATE" ]] || fail 'failed drain left the prior controller paused' +export FAKE_RUNNER_STATE=$tmp/managed-runner-active +: >"$FAKE_RUNNER_STATE" +export FAKE_FAIL_UP_ONCE=$tmp/fail-up-once +: >"$FAKE_FAIL_UP_ONCE" +expect_failure 'ROLLBACK_RESTORED' "$installer" --upgrade "${base_args[@]}" --ref "$ref_two" +[[ ! -f "$FAKE_RUNNER_STATE" ]] || fail 'upgrade preflight ran before the active runner was drained' +unset FAKE_RUNNER_STATE +unset FAKE_FAIL_UP_ONCE +grep -Fq 'CI_FLEET_MAX_RUNNERS=1' "$root/etc/ci-fleet/ci-fleet.env" || fail 'failed activation did not restore capacity one' +[[ $(readlink -f "$root/opt/ci-fleet/manager/current") == "$prior_manager" ]] || fail 'failed activation did not restore the prior manager release' +[[ -f "$FAKE_DOCKER_STATE" ]] || fail 'failed activation did not restore the prior controller runtime' +export FAKE_RESTART_AFTER_UP=$tmp/restart-after-up +: >"$FAKE_RESTART_AFTER_UP" +expect_failure 'ROLLBACK_RESTORED' "$installer" --upgrade "${base_args[@]}" --ref "$ref_two" +unset FAKE_RESTART_AFTER_UP +[[ ! -f "$FAKE_CONTROLLER_STATUS_FILE" && -f "$FAKE_DOCKER_STATE" ]] || fail 'restarting candidate blocked checkpoint restoration' +grep -Fq 'CI_FLEET_MAX_RUNNERS=1' "$root/etc/ci-fleet/ci-fleet.env" || fail 'restarting candidate rollback did not restore capacity one' +export FAKE_COMPOSE_LOG=$tmp/upgrade-compose.log +: >"$FAKE_COMPOSE_LOG" +expect_success "$installer" --upgrade "${base_args[@]}" --ref "$ref_two" >/dev/null +grep -Eq 'stop\|.*\|example-ci-01\|.* stop --timeout 2 controller$' "$FAKE_COMPOSE_LOG" || fail 'controller stop did not use the explicit graceful-shutdown timeout' +unset FAKE_COMPOSE_LOG +grep -Fq 'CI_FLEET_MAX_RUNNERS=2' "$root/etc/ci-fleet/ci-fleet.env" || fail 'upgrade did not apply capacity two' + +mkdir -p "$root/var/lib/ci-fleet/checkpoints/99999999-incomplete" +printf 'restarting\n' >"$FAKE_CONTROLLER_STATUS_FILE" +rm -f "$root/var/lib/ci-fleet/install-state.json" "$root/etc/ci-fleet/ci-fleet.env" +expect_success "$installer" --rollback >/dev/null +[[ ! -f "$FAKE_CONTROLLER_STATUS_FILE" ]] || fail 'explicit rollback did not recover a restarting controller' +grep -Fq 'CI_FLEET_MAX_RUNNERS=1' "$root/etc/ci-fleet/ci-fleet.env" || fail 'rollback did not restore capacity one' + +ref_three=$(write_config drained 2 2) +printf 'dead\n' >"$FAKE_CONTROLLER_STATUS_FILE" +export FAKE_STOP_FAIL=$tmp/stop-dead-fails +: >"$FAKE_STOP_FAIL" +expect_success "$installer" --upgrade "${base_args[@]}" --ref "$ref_three" >/dev/null +unset FAKE_STOP_FAIL +[[ ! -f "$FAKE_DOCKER_STATE" && ! -f "$FAKE_CONTROLLER_STATUS_FILE" ]] || fail 'non-active convergence retained a dead controller' +grep -Fq 'CI_FLEET_CONTROLLER_STATE=drained' "$root/etc/ci-fleet/ci-fleet.env" || fail 'drained state was not rendered' +grep -Fq 'CI_FLEET_MAX_RUNNERS=0' "$root/etc/ci-fleet/ci-fleet.env" || fail 'drained controller retained effective capacity' +[[ ! -f "$FAKE_DOCKER_STATE" ]] || fail 'drained controller remained running' +export FAKE_RUNNER_STATE=$tmp/drained-managed-runner +: >"$FAKE_RUNNER_STATE" +expect_failure 'DRIFT managed_runners' "$installer" --check "${base_args[@]}" --ref "$ref_three" +rm -f "$FAKE_RUNNER_STATE" +unset FAKE_RUNNER_STATE +export FAKE_ALL_RUNNER_STATE=$tmp/drained-exited-managed-runner +: >"$FAKE_ALL_RUNNER_STATE" +: >"$FAKE_DOCKER_PS_LOG" +expect_failure 'DRIFT managed_runners' "$installer" --check "${base_args[@]}" --ref "$ref_three" +expect_success "$installer" --install "${base_args[@]}" --ref "$ref_three" >/dev/null +[[ ! -f "$FAKE_ALL_RUNNER_STATE" ]] || fail 'non-active convergence did not remove stopped managed runners' +grep -Fq 'label=io.randomdevelopment.ci-fleet.instance=example-ci-01' "$FAKE_DOCKER_PS_LOG" || fail 'managed runner cleanup was not scoped to the selected instance' +unset FAKE_ALL_RUNNER_STATE + +ref_four=$(write_config disabled 2 2) +expect_success "$installer" --upgrade "${base_args[@]}" --ref "$ref_four" >/dev/null +grep -Fq 'CI_FLEET_CONTROLLER_STATE=disabled' "$root/etc/ci-fleet/ci-fleet.env" || fail 'disabled state was not rendered' +export FAKE_RUNNER_STATE=$tmp/disabled-managed-runner +: >"$FAKE_RUNNER_STATE" +expect_failure 'DRIFT managed_runners' "$installer" --check "${base_args[@]}" --ref "$ref_four" +rm -f "$FAKE_RUNNER_STATE" +unset FAKE_RUNNER_STATE + +export FAKE_RUNNER_STATE_ONCE=$tmp/orphaned-managed-runner +: >"$FAKE_RUNNER_STATE_ONCE" +export FAKE_ALL_RUNNER_STATE=$tmp/uninstall-stopped-managed-runner +: >"$FAKE_ALL_RUNNER_STATE" +: >"$FAKE_DOCKER_PS_LOG" +expect_success "$installer" --uninstall >/dev/null +[[ ! -f "$FAKE_RUNNER_STATE_ONCE" ]] || fail 'uninstall did not wait for an orphaned managed runner' +[[ ! -f "$FAKE_ALL_RUNNER_STATE" ]] || fail 'uninstall retained stopped managed runners' +grep -Fq 'label=io.randomdevelopment.ci-fleet.instance=example-ci-01' "$FAKE_DOCKER_PS_LOG" || fail 'uninstall runner cleanup was not scoped to the installed instance' +if grep -Eq 'label=io.randomdevelopment.ci-fleet.instance=$' "$FAKE_DOCKER_PS_LOG"; then fail 'uninstall runner cleanup used an empty instance filter'; fi +unset FAKE_RUNNER_STATE_ONCE FAKE_ALL_RUNNER_STATE +[[ ! -e "$root/opt/ci-fleet/current" && ! -e "$root/var/lib/ci-fleet/install-state.json" ]] || fail 'uninstall left active installation state' +[[ -f "$host_config" && -f "$pem" ]] || fail 'uninstall removed preserved host credentials' + +adopt_root=$tmp/adopt-host +export CI_FLEET_ROOT_PREFIX=$adopt_root +export FAKE_DOCKER_STATE=$tmp/adopt-controller-running +mkdir -p "$adopt_root/etc/ci-fleet/secrets" "$adopt_root/opt/ci-fleet/deploy" "$adopt_root/opt/ci-fleet/scripts" +adopt_pem=$adopt_root/etc/ci-fleet/secrets/github-app.pem +printf 'fixture only\n' >"$adopt_pem" +chmod 600 "$adopt_pem" +cp "$repo_root/deploy/compose.yaml" "$adopt_root/opt/ci-fleet/deploy/compose.yaml" +cp "$repo_root/scripts/healthcheck.sh" "$adopt_root/opt/ci-fleet/scripts/healthcheck.sh" +chmod 0755 "$adopt_root/opt/ci-fleet/scripts/healthcheck.sh" +printf '%s\n' \ + 'CI_FLEET_GITHUB_APP_CLIENT_ID=Iv1.EXAMPLE' \ + 'CI_FLEET_GITHUB_APP_INSTALLATION_ID=123456' \ + "CI_FLEET_GITHUB_APP_PRIVATE_KEY_FILE=$adopt_pem" \ + 'CI_FLEET_RUNNER_TTL=6h' \ + 'CI_FLEET_CONTROLLER_STATE=active' \ + 'CI_FLEET_INSTANCE=legacy-ci-01' >"$adopt_root/etc/ci-fleet/ci-fleet.env" +chmod 600 "$adopt_root/etc/ci-fleet/ci-fleet.env" +: >"$FAKE_DOCKER_STATE" +chmod 644 "$adopt_root/etc/ci-fleet/ci-fleet.env" +expect_failure 'rendered environment must be owned by root with mode 0600' "$installer" --adopt "${base_args[@]}" --ref "$ref_one" +chmod 600 "$adopt_root/etc/ci-fleet/ci-fleet.env" + +export FAKE_COMPOSE_LOG=$tmp/adopt-compose.log +: >"$FAKE_COMPOSE_LOG" +export FAKE_RESTART_AFTER_UP=$tmp/adopt-restart-after-up +: >"$FAKE_RESTART_AFTER_UP" +expect_failure 'ROLLBACK_RESTORED' "$installer" --adopt "${base_args[@]}" --ref "$ref_one" +unset FAKE_RESTART_AFTER_UP +grep -Fq "stop|$adopt_root/etc/ci-fleet/ci-fleet.env|example-ci-01" "$FAKE_COMPOSE_LOG" || fail 'rollback did not drain the candidate with its rendered environment and identity' +grep -Fq 'CI_FLEET_INSTANCE=legacy-ci-01' "$adopt_root/etc/ci-fleet/ci-fleet.env" || fail 'failed adoption did not restore the installed controller identity' +: >"$FAKE_COMPOSE_LOG" +export FAKE_RUNNER_STATE_ONCE=$tmp/adopt-managed-runner +: >"$FAKE_RUNNER_STATE_ONCE" +: >"$FAKE_DOCKER_PS_LOG" +adopt=$(expect_success "$installer" --adopt "${base_args[@]}" --ref "$ref_one") +grep -Fq 'CONVERGED mode=adopt' <<<"$adopt" || fail 'adoption did not converge' +[[ -f "$adopt_root/etc/ci-fleet/host.env" ]] || fail 'adoption did not separate host-local values' +grep -Fq 'label=io.randomdevelopment.ci-fleet.instance=legacy-ci-01' "$FAKE_DOCKER_PS_LOG" || fail 'adoption did not drain the installed controller instance' +unset FAKE_RUNNER_STATE_ONCE FAKE_COMPOSE_LOG + +grep -Fq 'Issue #7' "$repo_root/docs/DESIGN-DECISIONS.md" || fail 'isolated proof approval is not recorded' +if grep -Fq '/etc/ci-fleet/ci-fleet.env.before-max2' "$repo_root/docs/CAPACITY-PROMOTION.md"; then fail 'capacity runbook still edits rendered host state'; fi +grep -Fq -- '--upgrade' "$repo_root/docs/CAPACITY-PROMOTION.md" || fail 'capacity runbook does not apply reviewed desired state through the installer' + +printf 'INSTALLER_TESTS_OK\n' diff --git a/scripts/test_desired_state.py b/scripts/test_desired_state.py new file mode 100755 index 00000000..5e7eab51 --- /dev/null +++ b/scripts/test_desired_state.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +"""Deterministic tests for schema-v3 controller selection and rendering.""" + +from __future__ import annotations + +import copy +import json +import tempfile +import unittest +from pathlib import Path + +from desired_state import ( + DesiredStateError, + build_rendered_env, + load_and_validate_config, + parse_env, + validate_host_values, +) + + +ROOT = Path(__file__).resolve().parents[1] +CONFIG_COMMIT = "1" * 40 + + +def config() -> dict: + return json.loads((ROOT / "templates" / "config-repository" / "fleet.json").read_text(encoding="utf-8")) + + +def host_values() -> dict[str, str]: + return { + "CI_FLEET_GITHUB_APP_CLIENT_ID": "Iv1.EXAMPLE", + "CI_FLEET_GITHUB_APP_INSTALLATION_ID": "123456", + "CI_FLEET_GITHUB_APP_PRIVATE_KEY_FILE": "/etc/ci-fleet/secrets/github-app.pem", + "CI_FLEET_RUNNER_TTL": "6h", + } + + +class DesiredStateTests(unittest.TestCase): + def render(self, value: dict | None = None): + return build_rendered_env( + value or config(), + "example-ci-01", + host_values(), + config_repository="example-org/example-fleet-config", + config_ref=CONFIG_COMMIT, + docker_gid=998, + ) + + def test_active_controller_renders_configured_capacity(self) -> None: + environment, metadata = self.render() + self.assertEqual(environment["CI_FLEET_MAX_RUNNERS"], "1") + self.assertEqual(environment["CI_FLEET_CONFIGURED_MAX_RUNNERS"], "1") + self.assertEqual(environment["CI_FLEET_LABELS"], "docker-ci") + self.assertEqual(environment["CI_FLEET_COMMIT"], environment["CI_FLEET_ENGINE_REF"]) + self.assertEqual(metadata["controller_state"], "active") + + def test_drained_controller_renders_zero_effective_capacity(self) -> None: + value = config() + value["controllers"]["example-ci-01"]["state"] = "drained" + environment, metadata = self.render(value) + self.assertEqual(environment["CI_FLEET_MAX_RUNNERS"], "0") + self.assertEqual(environment["CI_FLEET_CONFIGURED_MAX_RUNNERS"], "1") + self.assertEqual(metadata["effective_max_runners"], 0) + + def test_disabled_controller_renders_zero_effective_capacity(self) -> None: + value = config() + value["controllers"]["example-ci-01"]["state"] = "disabled" + environment, _ = self.render(value) + self.assertEqual(environment["CI_FLEET_MAX_RUNNERS"], "0") + + def test_missing_controller_fails_closed(self) -> None: + with self.assertRaisesRegex(DesiredStateError, "is not declared"): + build_rendered_env( + config(), + "missing-ci-01", + host_values(), + config_repository="example-org/example-fleet-config", + config_ref=CONFIG_COMMIT, + docker_gid=998, + ) + + def test_capacity_overcommit_is_rejected_by_public_contract(self) -> None: + value = config() + value["controllers"]["example-ci-01"]["max_runners"] = 2 + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "fleet.json" + path.write_text(json.dumps(value), encoding="utf-8") + with self.assertRaisesRegex(DesiredStateError, "capacity_budget"): + load_and_validate_config(path) + + def test_project_max_parallel_is_rejected(self) -> None: + value = config() + value["projects"]["example-app"]["ci_contract"]["max_parallel"] = 1 + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "fleet.json" + path.write_text(json.dumps(value), encoding="utf-8") + with self.assertRaisesRegex(DesiredStateError, "max_parallel"): + load_and_validate_config(path) + + def test_unknown_host_local_variable_is_rejected(self) -> None: + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "host.env" + path.write_text( + "CI_FLEET_GITHUB_APP_CLIENT_ID=Iv1.EXAMPLE\n" + "CI_FLEET_GITHUB_APP_INSTALLATION_ID=123\n" + "CI_FLEET_GITHUB_APP_PRIVATE_KEY_FILE=/safe/key.pem\n" + "PROJECT_SECRET=forbidden\n", + encoding="utf-8", + ) + with self.assertRaisesRegex(DesiredStateError, "unsupported host-local variable"): + parse_env(path, allow_unknown=False) + + def test_duplicate_host_local_variable_is_rejected(self) -> None: + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "host.env" + path.write_text( + "CI_FLEET_GITHUB_APP_CLIENT_ID=first\n" + "CI_FLEET_GITHUB_APP_CLIENT_ID=second\n", + encoding="utf-8", + ) + with self.assertRaisesRegex(DesiredStateError, "duplicate variable"): + parse_env(path, allow_unknown=False) + + def test_host_values_require_absolute_key_path(self) -> None: + values = copy.deepcopy(host_values()) + values["CI_FLEET_GITHUB_APP_PRIVATE_KEY_FILE"] = "relative.pem" + with self.assertRaisesRegex(DesiredStateError, "absolute shell-safe"): + validate_host_values(values) + + def test_host_values_reject_sub_hour_ttl(self) -> None: + values = copy.deepcopy(host_values()) + values["CI_FLEET_RUNNER_TTL"] = "30m" + with self.assertRaisesRegex(DesiredStateError, "at least one hour"): + validate_host_values(values) + + def test_shell_metacharacter_in_config_identity_is_rejected(self) -> None: + with self.assertRaisesRegex(DesiredStateError, "must be shell-safe"): + build_rendered_env( + config(), + "example-ci-01", + host_values(), + config_repository="/tmp/config$(id)", + config_ref=CONFIG_COMMIT, + docker_gid=998, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/validate.sh b/scripts/validate.sh index 9628d645..df4d3901 100755 --- a/scripts/validate.sh +++ b/scripts/validate.sh @@ -5,11 +5,18 @@ repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) cd "$repo_root" for script in scripts/*.sh examples/project/scripts/ci/*.sh; do bash -n "$script"; done -python3 -m py_compile .github/actions/plan/plan.py .github/actions/plan/test_plan.py +python3 -m py_compile \ + .github/actions/plan/plan.py \ + .github/actions/plan/test_plan.py \ + scripts/desired_state.py \ + scripts/scan_committed_secrets.py \ + scripts/test_desired_state.py python3 .github/actions/plan/test_plan.py +python3 scripts/test_desired_state.py python3 .github/actions/plan/plan.py --plan examples/project/scripts/ci/plan.json --group fast >/dev/null python3 .github/actions/plan/plan.py --plan examples/project/scripts/ci/plan.json --group full >/dev/null scripts/test-capacity-preflight.sh +scripts/test-install-worker-controller.sh tmp=$(mktemp) trap 'rm -f "$tmp"' EXIT @@ -20,13 +27,12 @@ export CI_FLEET_GITHUB_APP_CLIENT_ID=validation export CI_FLEET_GITHUB_APP_INSTALLATION_ID=1 export CI_FLEET_GITHUB_APP_PRIVATE_KEY_FILE="$tmp" export CI_FLEET_DOCKER_GID=999 +export CI_FLEET_RUNNER_IMAGE=${CI_FLEET_RUNNER_IMAGE:-ci-fleet-runner:dev} +export CI_FLEET_CONTROLLER_IMAGE=${CI_FLEET_CONTROLLER_IMAGE:-ci-fleet-controller:dev} docker compose -f deploy/compose.yaml config --quiet docker compose -f deploy/compose.yaml build runner-image controller -docker run --rm --entrypoint /bin/bash ci-fleet-runner:dev -c './bin/Runner.Listener --version && docker --version && docker compose version && git --version' +docker run --rm --entrypoint /bin/bash "$CI_FLEET_RUNNER_IMAGE" -c './bin/Runner.Listener --version && docker --version && docker compose version && git --version' CI_FLEET_INSTANCE=validation scripts/cleanup.sh -if rg -n --hidden --glob '!*.example' --glob '!scripts/validate.sh' --glob '!docs/SECRETS.md' --glob '!SECURITY.md' '(BEGIN (RSA |EC |OPENSSH )?PRIVATE KEY|github_pat_|ghp_[A-Za-z0-9]{20,})' .; then - echo "possible committed secret detected" >&2 - exit 1 -fi +python3 scripts/scan_committed_secrets.py diff --git a/templates/config-repository/.github/workflows/validate.yml b/templates/config-repository/.github/workflows/validate.yml index ad0549a7..27e1fd1d 100644 --- a/templates/config-repository/.github/workflows/validate.yml +++ b/templates/config-repository/.github/workflows/validate.yml @@ -26,11 +26,14 @@ jobs: - name: Validate shell and Python syntax run: | bash -n scripts/init.sh scripts/validate.sh - python3 -m py_compile scripts/init.py scripts/validate.py scripts/test_policy.py + python3 -m py_compile scripts/init.py scripts/validate.py scripts/test_policy.py scripts/scan_committed_secrets.py - name: Prove forbidden configurations fail closed run: python3 scripts/test_policy.py + - name: Scan committed file contents for secrets + run: python3 scripts/scan_committed_secrets.py + - name: Validate reference configurations run: | ./scripts/validate.sh @@ -39,7 +42,28 @@ jobs: - name: Prove initialized configurations pass strict policy run: | temporary_directory="$(mktemp -d)" + engine_ref=$(python3 -c 'import json; value = json.load(open("fleet.json", encoding="utf-8")); print(next(iter(value["controllers"].values()))["engine_ref"])') + if ./scripts/init.sh \ + --organization test-company \ + --project test-app \ + --output "${temporary_directory}/missing-engine.json"; then + echo 'initializer accepted a missing engine revision' >&2 + exit 1 + fi + test ! -e "${temporary_directory}/missing-engine.json" + if ./scripts/init.sh \ + --organization test-company \ + --project test-app \ + --registry BAD \ + --engine-ref "${engine_ref}" \ + --output "${temporary_directory}/invalid.json"; then + echo 'initializer accepted an invalid registry' >&2 + exit 1 + fi + test ! -e "${temporary_directory}/invalid.json" ./scripts/init.sh \ --organization test-company \ --project test-app \ + --engine-ref "${engine_ref}" \ --output "${temporary_directory}/fleet.json" + test "$(stat -c '%a' "${temporary_directory}/fleet.json")" = 644 diff --git a/templates/config-repository/AGENTS.md b/templates/config-repository/AGENTS.md index d340d357..c6d46a8a 100644 --- a/templates/config-repository/AGENTS.md +++ b/templates/config-repository/AGENTS.md @@ -2,7 +2,7 @@ ## Purpose -This repository maps projects to CI pools, deployment environments, logical host groups, and container images. It never stores secret values. +This repository maps projects to CI pools, Git-authored controller desired state, deployment environments, logical host groups, and container images. It never stores secret values or host-local infrastructure details. ## Required verification @@ -15,11 +15,14 @@ Before committing configuration changes, run: ## Hard rules - Never add real `.env` files, credentials, tokens, private keys, cookies, or passwords. +- Never add addresses, VM IDs, storage identifiers, backup identifiers, SSH details, or rendered runtime configuration. - Do not weaken `public_repositories: false` for Docker-socket runner pools. +- Infrastructure configuration owns capacity. Application workflows submit all independent jobs and do not use `max-parallel` to model fleet size. +- The sum of active and drained controller maxima must not exceed the pool capacity budget. +- Controller engine revisions and reusable workflows must be pinned to full reviewed commit SHAs. - Production environments must require approval and must not deploy automatically. - CI runner hosts and application deployment hosts are separate roles. - Image promotion uses immutable digests; do not rebuild separately for production. -- Reusable workflows must be pinned to a full reviewed commit SHA. - Keep ordinary task jobs at a five-minute hard ceiling and expected shard payload at four minutes or less. - Preserve deterministic task/shard isolation; Compose identity must include task and shard as well as run identity. -- Update schema and validator together. +- Update schema, initializer, validator, tests, examples, and documentation together. diff --git a/templates/config-repository/README.md b/templates/config-repository/README.md index 9c536c13..6b3e90d6 100644 --- a/templates/config-repository/README.md +++ b/templates/config-repository/README.md @@ -1,31 +1,42 @@ # ci-fleet configuration template -This is the public, secret-free starting point for an organization's private `ci-fleet` configuration repository. It records which trusted projects may use each CI pool, which logical deployment environments exist, and the standardized commands every project must expose. +This is the public, secret-free starting point for an organization's private `ci-fleet` configuration repository. It records which trusted projects may use each CI pool, the reviewed desired state for controller machines, infrastructure capacity budgets, logical deployment environments, and the standardized commands every project must expose. -It does **not** contain runner registration tokens, deploy credentials, private keys, host addresses, or `.env` files. +It does **not** contain runner registration tokens, deploy credentials, private keys, host addresses, VM IDs, storage names, backup identifiers, or `.env` files. ```mermaid flowchart LR - E[Public ci-fleet engine] -->|pinned reusable workflow| P[Private project repos] - C[Private config created from this template] -->|logical policy| P - P -->|task matrix| R[Trusted Docker CI pool] + E[Public ci-fleet engine] -->|pinned engine commit| C[Private configuration] + C -->|pool policy and capacity budget| A[Controller at site A] + C -->|pool policy and capacity budget| B[Controller at site B] + P[Authorized project repositories] -->|all independent jobs| G[GitHub runner group] + A --> G + B --> G + G --> R[Ephemeral Docker workers] P -->|approved image digest| D[Development hosts] - P -->|manual approval + image digest| X[Production hosts] - S[GitHub Environments / host secret store] -. secret values .-> P + P -->|manual approval and same digest| X[Production hosts] + S[GitHub Environments or host secret store] -. secret values .-> P classDef public fill:#dff4ff,stroke:#1570a6,color:#102a43 classDef private fill:#fff3cd,stroke:#9a6700,color:#3d2b00 class E public - class C,P,R,D,X,S private + class C,A,B,P,G,R,D,X,S private ``` ## Start a private organization configuration 1. Create a **private** repository from this public template. -2. Clone it and initialize the first project: +2. Clone it and initialize the first project and controller: ```bash - ./scripts/init.sh --organization your-org --project your-app + ./scripts/init.sh \ + --organization your-org \ + --project your-app \ + --controller ci-01 \ + --location primary-site \ + --capacity-budget 1 \ + --max-runners 1 \ + --engine-ref ``` 3. Edit `fleet.json` to add the organization's real logical mappings. @@ -35,23 +46,77 @@ flowchart LR ./scripts/validate.sh --strict ``` -5. Configure secret **values** in GitHub Environments or the deployment host's secret manager. The repository stores only names such as `DEPLOY_AUTH`. +5. Configure secret **values** in GitHub Environments, root-owned host files, or an external secret manager. The repository stores only names such as `DEPLOY_AUTH`. -The initializer refuses to replace a configured file unless `--force` is explicit. Run `./scripts/init.sh --help` for repository, registry, label, and output options. +The initializer refuses to replace a configured file unless `--force` is explicit. Run `./scripts/init.sh --help` for repository, registry, runner-group, controller, location, capacity, resource, and output options. + +## Schema v3: Git-authored controller desired state + +`fleet.json` is the reviewed authority for logical controller state. Each entry in `controllers` has a unique ID and declares: + +- its runner pool and logical location; +- whether it is `active`, `drained`, or `disabled`; +- its unique GitHub scale-set name; +- an `experimental`, `stable`, or `retiring` lifecycle; +- the full reviewed ci-fleet commit SHA it runs; +- a zero managed minimum and reviewed maximum runner capacity; +- CPU and memory available to each ephemeral runner. + +The controller ID is how a target host selects its declaration. A location is a non-sensitive logical slug such as `primary-site` or `remote-site`, never an address. Runtime-generated configuration and credentials remain host-local. + +### Pool capacity is infrastructure policy + +Each runner pool has a `capacity_budget`. The validator totals the maximum capacity of every active or drained controller assigned to the pool and rejects overcommit. Drained capacity remains reserved so an undrain cannot silently exceed the reviewed budget. Disabled controllers do not reserve capacity. + +Application repositories do not encode the number of available workers. They submit all independent tasks and shards. Do not use GitHub Actions `strategy.max-parallel` to model fleet size; controllers and the private configuration decide how many jobs run simultaneously. An application may limit concurrency only for a separately documented external-system constraint, not worker availability. + +This separation lets one infrastructure change add, remove, drain, or resize controllers without editing every project workflow. + +## Target-host installation and adoption + +The public ci-fleet engine owns the host installer. Its intended interface consumes one logical controller from a pinned private configuration revision: + +```bash +sudo ./scripts/install-worker-controller.sh \ + --config-repo example-org/example-fleet-config \ + --controller example-ci-01 \ + --ref \ + --install +``` + +Use `--adopt` instead of `--install` to bring an existing controller under Git-authored desired state. The engine contract also provides `--check`, `--upgrade`, `--rollback`, and `--uninstall` modes. + +The command runs on the target Linux Docker machine. It validates the pinned configuration, renders host-local runtime state, preserves root-owned secrets, drains before disruptive changes, verifies a recoverable checkpoint, installs maintenance services, checks health, and reports drift without exposing credentials. OpenClaw or another agent may invoke it, but no agent is required. + +The engine-side implementation is `scripts/install-worker-controller.sh` in the parent ci-fleet repository. Its accepted scope is isolated ordinary-CI fleet hosts under reviewed schema-v3 desired state; this configuration repository never installs a controller by itself. + +## Drain, retire, and delete a controller host + +Host retirement is an explicit reviewed transition: + +1. Change the controller state to `drained` and merge the private configuration change. +2. Converge the host and verify that it accepts no new work and has no active runner. +3. Remove only fleet-owned residue and verify replacement capacity. +4. Unregister its scale set and revoke that host's credentials. +5. Change the declaration to `disabled` or remove it in a later reviewed change. +6. Delete or repurpose the machine according to the installation's declared infrastructure policy. + +Deleting one generic controller must not require application workflow changes. Legacy project-specific hosts should remain only until CI, promotion, and deployment no longer reference them. ## Hard rules -- Public repositories never receive access to the trusted self-hosted runner pool. +- Public repositories never receive direct access to the trusted self-hosted runner pool. - Every project publishes `scripts/ci/plan.json` and implements `./scripts/ci/run.sh --shard INDEX/TOTAL` in its own Docker-defined test environment. - `./scripts/ci/run.sh fast` and `full` remain aggregate developer commands; fleet scheduling expands their named tasks across available workers. - Every matrix job has a five-minute hard timeout, while expected test payload targets four minutes or less to reserve startup and reporting time. +- Application workflows submit all independent jobs; infrastructure configuration alone controls worker capacity. - CI runner pools and deployment host groups are separate trust roles. - Production deployment is manual and requires GitHub Environment approval. -- Reusable workflows and third-party actions are pinned to immutable commits. +- Controller engine revisions, reusable workflows, and third-party actions are pinned to immutable commits. - Configuration contains logical identifiers only. Secret values, private host details, and credentials never enter Git. - Promoted artifacts are container image digests; production does not rebuild a different image. -`fleet.schema.json` provides editor completion and structural documentation. `scripts/validate.py` is the authoritative dependency-free policy check, including relationships JSON Schema cannot express clearly. +`fleet.schema.json` provides editor completion and structural documentation. `scripts/validate.py` is the authoritative dependency-free policy check, including cross-object relationships JSON Schema cannot express clearly. ## Five-minute parallelism contract @@ -61,9 +126,9 @@ Projects divide their total test-minutes into independent named tasks and determ flowchart LR P[plan.json] --> M[GitHub matrix] M --> A[lint] - M --> B[unit 1/4] - M --> C[unit 2/4] - M --> D[integration 1/3] + M --> B[unit 1 of 4] + M --> C[unit 2 of 4] + M --> D[integration 1 of 3] M --> E[other independent shards] ``` @@ -73,21 +138,22 @@ Adding workers reduces wall-clock time only while independent shards remain queu | Path | Purpose | |---|---| -| `fleet.json` | Fictional, valid reference configuration to initialize or replace | +| `fleet.json` | Fictional, valid schema-v3 configuration with one controller | | `fleet.schema.json` | JSON Schema draft 2020-12 editor contract | -| `scripts/init.sh` | Safe first-project initializer | -| `scripts/validate.sh` | Structural, policy, and secret-boundary validation | +| `scripts/init.sh` | Safe first-project and first-controller initializer | +| `scripts/validate.sh` | Structural, relational, capacity, and secret-boundary validation | | `scripts/test_policy.py` | Regression tests proving unsafe configurations fail closed | -| `examples/multi-host/fleet.json` | Fictional two-project, multi-host topology | +| `examples/multi-host/fleet.json` | Fictional two-project, two-location controller topology | | `SECURITY.md` | Secret handling and vulnerability reporting | | `AGENTS.md` | Non-negotiable rules for humans and coding agents | ## Public and private boundary -| Safe in this public template | Belongs in the private config repo | Belongs only in a secret store | +| Safe in this public template | Belongs in the private config repo | Belongs only outside Git | |---|---|---| -| Schema, validator, fictional examples | Real repository names and logical host-group names | Tokens, passwords, private keys | -| Standard CI entrypoint names | Environment policy and allowed repository lists | Host addresses and SSH material | -| Reusable workflow references | Required secret **names** | `.env` contents and app credentials | +| Schema, validator, fictional examples | Real repository names and runner-group policy | Tokens, passwords, private keys | +| Standard CI entrypoint names | Logical controller IDs and locations | Host addresses, VM IDs, SSH material | +| Controller state and lifecycle vocabulary | Capacity budgets and per-runner limits | Rendered runtime configuration | +| Reusable engine interface | Required secret **names** | Secret values and application credentials | The public engine and this template use the [Unlicense](LICENSE). See [THIRD_PARTY_NOTICES.md](THIRD_PARTY_NOTICES.md) before copying third-party material into a derived repository. diff --git a/templates/config-repository/examples/multi-host/fleet.json b/templates/config-repository/examples/multi-host/fleet.json index 603ee21f..4d7acece 100644 --- a/templates/config-repository/examples/multi-host/fleet.json +++ b/templates/config-repository/examples/multi-host/fleet.json @@ -1,6 +1,6 @@ { "$schema": "../../fleet.schema.json", - "schema_version": 2, + "schema_version": 3, "organization": { "slug": "sample-company", "registry": "ghcr.io/sample-company", @@ -9,10 +9,42 @@ }, "runner_pools": { "trusted-ci": { + "runner_group": "sample-trusted-ci", "routing_labels": ["docker-ci"], "allowed_repositories": ["sample-company/api", "sample-company/web"], "public_repositories": false, - "max_concurrent_jobs": 4 + "capacity_budget": 6, + "job_submission_policy": "all-independent-jobs" + } + }, + "controllers": { + "sample-ci-primary": { + "pool": "trusted-ci", + "location": "primary-site", + "state": "active", + "scale_set_name": "sample-ci-primary", + "lifecycle": "stable", + "engine_ref": "254b98a10ea200e39f19bbe75ab9a822de2d5966", + "min_runners": 0, + "max_runners": 4, + "runner_resources": { + "cpu_cores": 4, + "memory_mib": 8192 + } + }, + "sample-ci-remote": { + "pool": "trusted-ci", + "location": "remote-site", + "state": "active", + "scale_set_name": "sample-ci-remote", + "lifecycle": "stable", + "engine_ref": "254b98a10ea200e39f19bbe75ab9a822de2d5966", + "min_runners": 0, + "max_runners": 2, + "runner_resources": { + "cpu_cores": 2, + "memory_mib": 4096 + } } }, "host_groups": { diff --git a/templates/config-repository/fleet.json b/templates/config-repository/fleet.json index 0a3177aa..bf3ccea4 100644 --- a/templates/config-repository/fleet.json +++ b/templates/config-repository/fleet.json @@ -1,6 +1,6 @@ { "$schema": "./fleet.schema.json", - "schema_version": 2, + "schema_version": 3, "organization": { "slug": "example-org", "registry": "ghcr.io/example-org", @@ -9,10 +9,28 @@ }, "runner_pools": { "trusted-ci": { + "runner_group": "example-trusted-ci", "routing_labels": ["docker-ci"], "allowed_repositories": ["example-org/example-app"], "public_repositories": false, - "max_concurrent_jobs": 1 + "capacity_budget": 1, + "job_submission_policy": "all-independent-jobs" + } + }, + "controllers": { + "example-ci-01": { + "pool": "trusted-ci", + "location": "example-site-a", + "state": "active", + "scale_set_name": "example-ci-01", + "lifecycle": "experimental", + "engine_ref": "254b98a10ea200e39f19bbe75ab9a822de2d5966", + "min_runners": 0, + "max_runners": 1, + "runner_resources": { + "cpu_cores": 2, + "memory_mib": 4096 + } } }, "host_groups": { diff --git a/templates/config-repository/fleet.schema.json b/templates/config-repository/fleet.schema.json index 9dd012e8..a8f0716c 100644 --- a/templates/config-repository/fleet.schema.json +++ b/templates/config-repository/fleet.schema.json @@ -4,10 +4,10 @@ "title": "ci-fleet organization configuration", "type": "object", "additionalProperties": false, - "required": ["schema_version", "organization", "runner_pools", "host_groups", "environments", "projects"], + "required": ["schema_version", "organization", "runner_pools", "controllers", "host_groups", "environments", "projects"], "properties": { "$schema": {"type": "string"}, - "schema_version": {"const": 2}, + "schema_version": {"const": 3}, "organization": { "type": "object", "additionalProperties": false, @@ -15,41 +15,80 @@ "properties": { "slug": {"type": "string", "pattern": "^[a-z0-9][a-z0-9-]{0,38}$"}, "registry": {"type": "string", "pattern": "^[a-z0-9.-]+/[a-z0-9._/-]+$"}, - "delivery_engine": {"type": "string", "pattern": "^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$"}, + "delivery_engine": {"const": "RandomDevelopment/ci-fleet"}, "workflow_ref_policy": {"const": "immutable-commit"} } }, "runner_pools": { "type": "object", "minProperties": 1, + "propertyNames": {"$ref": "#/$defs/slug"}, "additionalProperties": {"$ref": "#/$defs/runner_pool"} }, + "controllers": { + "type": "object", + "minProperties": 1, + "propertyNames": {"$ref": "#/$defs/slug"}, + "additionalProperties": {"$ref": "#/$defs/controller"} + }, "host_groups": { "type": "object", "minProperties": 1, + "propertyNames": {"$ref": "#/$defs/slug"}, "additionalProperties": {"$ref": "#/$defs/host_group"} }, "environments": { "type": "object", "minProperties": 1, + "propertyNames": {"$ref": "#/$defs/slug"}, "additionalProperties": {"$ref": "#/$defs/environment"} }, "projects": { "type": "object", "minProperties": 1, + "propertyNames": {"$ref": "#/$defs/slug"}, "additionalProperties": {"$ref": "#/$defs/project"} } }, "$defs": { + "slug": {"type": "string", "pattern": "^[a-z0-9][a-z0-9-]{0,62}$"}, + "repository": {"type": "string", "pattern": "^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$"}, "runner_pool": { "type": "object", "additionalProperties": false, - "required": ["routing_labels", "allowed_repositories", "public_repositories", "max_concurrent_jobs"], + "required": ["runner_group", "routing_labels", "allowed_repositories", "public_repositories", "capacity_budget", "job_submission_policy"], "properties": { - "routing_labels": {"type": "array", "minItems": 1, "uniqueItems": true, "items": {"type": "string", "pattern": "^[a-z0-9][a-z0-9-]{0,62}$"}}, - "allowed_repositories": {"type": "array", "minItems": 1, "uniqueItems": true, "items": {"type": "string", "pattern": "^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$"}}, + "runner_group": {"$ref": "#/$defs/slug"}, + "routing_labels": {"type": "array", "minItems": 1, "uniqueItems": true, "items": {"$ref": "#/$defs/slug"}}, + "allowed_repositories": {"type": "array", "minItems": 1, "uniqueItems": true, "items": {"$ref": "#/$defs/repository"}}, "public_repositories": {"const": false}, - "max_concurrent_jobs": {"type": "integer", "minimum": 1} + "capacity_budget": {"type": "integer", "minimum": 1}, + "job_submission_policy": {"const": "all-independent-jobs"} + } + }, + "controller": { + "type": "object", + "additionalProperties": false, + "required": ["pool", "location", "state", "scale_set_name", "lifecycle", "engine_ref", "min_runners", "max_runners", "runner_resources"], + "properties": { + "pool": {"$ref": "#/$defs/slug"}, + "location": {"$ref": "#/$defs/slug"}, + "state": {"enum": ["active", "drained", "disabled"]}, + "scale_set_name": {"$ref": "#/$defs/slug"}, + "lifecycle": {"enum": ["experimental", "stable", "retiring"]}, + "engine_ref": {"type": "string", "pattern": "^(?!0{40}$)[0-9a-f]{40}$"}, + "min_runners": {"const": 0}, + "max_runners": {"type": "integer", "minimum": 1}, + "runner_resources": {"$ref": "#/$defs/runner_resources"} + } + }, + "runner_resources": { + "type": "object", + "additionalProperties": false, + "required": ["cpu_cores", "memory_mib"], + "properties": { + "cpu_cores": {"type": "integer", "minimum": 1}, + "memory_mib": {"type": "integer", "minimum": 512} } }, "host_group": { @@ -77,7 +116,7 @@ "additionalProperties": false, "required": ["repository", "image", "ci_pool", "ci_contract", "deployments"], "properties": { - "repository": {"type": "string", "pattern": "^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$"}, + "repository": {"$ref": "#/$defs/repository"}, "image": {"type": "string", "pattern": "^[a-z0-9.-]+/[a-z0-9._/-]+$"}, "ci_pool": {"type": "string"}, "ci_contract": { diff --git a/templates/config-repository/scripts/init.py b/templates/config-repository/scripts/init.py index 0a761d61..3d3ac48b 100755 --- a/templates/config-repository/scripts/init.py +++ b/templates/config-repository/scripts/init.py @@ -5,14 +5,25 @@ import argparse import json +import os import re import subprocess +import tempfile from pathlib import Path ROOT = Path(__file__).resolve().parents[1] ORG_SLUG = re.compile(r"^[a-z0-9][a-z0-9-]{0,38}$") -PROJECT_SLUG = re.compile(r"^[a-z0-9][a-z0-9-]{0,62}$") +SLUG = re.compile(r"^[a-z0-9][a-z0-9-]{0,62}$") +COMMIT_SHA = re.compile(r"^[0-9a-f]{40}$") + + + +def positive_integer(value: str) -> int: + parsed = int(value) + if parsed < 1: + raise argparse.ArgumentTypeError("must be a positive integer") + return parsed def parse_args() -> argparse.Namespace: @@ -22,6 +33,14 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--repository", help="owner/repository; defaults to ORGANIZATION/PROJECT") parser.add_argument("--registry", help="registry namespace; defaults to ghcr.io/ORGANIZATION") parser.add_argument("--runner-label", default="docker-ci", help="capability label for the shared CI pool") + parser.add_argument("--runner-group", default="trusted-ci", help="logical GitHub runner-group slug") + parser.add_argument("--controller", default="ci-01", help="logical controller ID and scale-set name") + parser.add_argument("--location", default="primary-site", help="logical location slug; never an address") + parser.add_argument("--capacity-budget", type=positive_integer, default=1, help="maximum capacity reserved by the pool") + parser.add_argument("--max-runners", type=positive_integer, default=1, help="initial controller maximum") + parser.add_argument("--runner-cpu-cores", type=positive_integer, default=2, help="CPU cores available to each runner") + parser.add_argument("--runner-memory-mib", type=positive_integer, default=4096, help="memory available to each runner") + parser.add_argument("--engine-ref", required=True, help="reviewed full ci-fleet commit SHA") parser.add_argument("--output", type=Path, default=ROOT / "fleet.json", help="output configuration path") parser.add_argument("--force", action="store_true", help="replace an existing non-example output file") return parser.parse_args() @@ -35,10 +54,21 @@ def main() -> int: args = parse_args() if not ORG_SLUG.fullmatch(args.organization): fail("--organization must be a lowercase GitHub organization slug") - if not PROJECT_SLUG.fullmatch(args.project): - fail("--project must be a lowercase slug") - if not PROJECT_SLUG.fullmatch(args.runner_label): - fail("--runner-label must be a lowercase slug") + for option, value in { + "--project": args.project, + "--runner-label": args.runner_label, + "--runner-group": args.runner_group, + "--controller": args.controller, + "--location": args.location, + }.items(): + if not SLUG.fullmatch(value): + fail(f"{option} must be a lowercase slug") + if not COMMIT_SHA.fullmatch(args.engine_ref) or args.engine_ref == "0" * 40: + fail("--engine-ref must be a nonzero full lowercase commit SHA") + if args.max_runners > args.capacity_budget: + fail("--max-runners must not exceed --capacity-budget") + if args.runner_memory_mib < 512: + fail("--runner-memory-mib must be at least 512") repository = args.repository or f"{args.organization}/{args.project}" registry = (args.registry or f"ghcr.io/{args.organization}").rstrip("/") @@ -54,7 +84,7 @@ def main() -> int: config = { "$schema": str((ROOT / "fleet.schema.json").resolve()) if output.parent != ROOT else "./fleet.schema.json", - "schema_version": 2, + "schema_version": 3, "organization": { "slug": args.organization, "registry": registry, @@ -63,10 +93,28 @@ def main() -> int: }, "runner_pools": { "trusted-ci": { + "runner_group": args.runner_group, "routing_labels": [args.runner_label], "allowed_repositories": [repository], "public_repositories": False, - "max_concurrent_jobs": 1, + "capacity_budget": args.capacity_budget, + "job_submission_policy": "all-independent-jobs", + } + }, + "controllers": { + args.controller: { + "pool": "trusted-ci", + "location": args.location, + "state": "active", + "scale_set_name": args.controller, + "lifecycle": "experimental", + "engine_ref": args.engine_ref, + "min_runners": 0, + "max_runners": args.max_runners, + "runner_resources": { + "cpu_cores": args.runner_cpu_cores, + "memory_mib": args.runner_memory_mib, + }, } }, "host_groups": { @@ -108,14 +156,23 @@ def main() -> int: }, } output.parent.mkdir(parents=True, exist_ok=True) - output.write_text(json.dumps(config, indent=2) + "\n", encoding="utf-8") - - subprocess.run( - [str(ROOT / "scripts" / "validate.sh"), "--strict", "--skip-path-scan", "--config", str(output)], - check=True, - ) + descriptor, temporary_name = tempfile.mkstemp(prefix=f".{output.name}.", dir=output.parent, text=True) + temporary = Path(temporary_name) + try: + with os.fdopen(descriptor, "w", encoding="utf-8") as handle: + handle.write(json.dumps(config, indent=2) + "\n") + handle.flush() + os.fsync(handle.fileno()) + subprocess.run( + [str(ROOT / "scripts" / "validate.sh"), "--strict", "--skip-path-scan", "--config", str(temporary)], + check=True, + ) + os.chmod(temporary, 0o644) + os.replace(temporary, output) + finally: + temporary.unlink(missing_ok=True) print(f"Initialized {output}") - print("Next: edit logical mappings, configure GitHub Environments, and keep every secret value outside Git.") + print("Next: review controller capacity, configure GitHub policy, and keep every secret value outside Git.") return 0 diff --git a/templates/config-repository/scripts/scan_committed_secrets.py b/templates/config-repository/scripts/scan_committed_secrets.py new file mode 100755 index 00000000..cd191118 --- /dev/null +++ b/templates/config-repository/scripts/scan_committed_secrets.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +"""Fail when tracked files contain high-confidence credential material.""" + +from __future__ import annotations + +import argparse +import re +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +PATTERN = re.compile( + rb"BEGIN (?:[A-Z0-9]+ )*PRIVATE KEY|" + rb"github_pat_[A-Za-z0-9_]{20,}|" + rb"gh[opusr]_[A-Za-z0-9]{20,}|" + rb"AKIA[0-9A-Z]{16}|" + rb"(?:postgres|mysql|mongodb(?:\+srv)?|redis)://[^\s/:]+:[^\s/@]+@" +) +assert all(PATTERN.search(b"BEGIN " + kind + b"PRIVATE KEY") for kind in (b"", b"RSA ", b"DSA ", b"EC ", b"OPENSSH ", b"ENCRYPTED ")) +assert all(PATTERN.search(prefix + b"x" * 20) for prefix in (b"gho_", b"ghp_", b"ghr_", b"ghs_", b"ghu_")) +assert PATTERN.search(b"AKIA" + b"A" * 16) +assert PATTERN.search(b"mysql" + b"://fixture-user:***@example.invalid/database") + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--repository", type=Path, default=ROOT) + parser.add_argument("--commit", help="immutable commit whose blobs should be scanned") + return parser.parse_args() + + +def main() -> int: + args = parse_args() + repository = args.repository.resolve() + if args.commit is not None and not re.fullmatch(r"[0-9a-f]{40}", args.commit): + print("--commit must be a full lowercase commit SHA", file=sys.stderr) + return 2 + listing = ["git", "-C", str(repository)] + listing += ["ls-tree", "-rz", "--name-only", args.commit] if args.commit else ["ls-files", "-z"] + tracked = subprocess.run( + listing, + check=True, + stdout=subprocess.PIPE, + ).stdout.split(b"\0") + findings: list[str] = [] + for raw in tracked: + if not raw: + continue + relative = raw.decode("utf-8") + if args.commit is None: + data = (repository / relative).read_bytes() + else: + data = subprocess.run( + ["git", "-C", str(repository), "cat-file", "blob", f"{args.commit}:{relative}"], + check=True, + stdout=subprocess.PIPE, + ).stdout + for match in PATTERN.finditer(data): + line = data.count(b"\n", 0, match.start()) + 1 + findings.append(f"{relative}:{line}") + if findings: + print("possible committed secret detected:", file=sys.stderr) + print("\n".join(findings), file=sys.stderr) + return 1 + print("OK: no high-confidence secret material in tracked files") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/templates/config-repository/scripts/test_policy.py b/templates/config-repository/scripts/test_policy.py index 630b519b..7dc83468 100755 --- a/templates/config-repository/scripts/test_policy.py +++ b/templates/config-repository/scripts/test_policy.py @@ -5,10 +5,14 @@ import copy import json +import re +import subprocess +import sys +import tempfile import unittest from pathlib import Path -from validate import Validation, scan_secret_material, validate_config +from validate import Validation, load_json, scan_secret_material, scan_tree_path_list, validate_config ROOT = Path(__file__).resolve().parents[1] @@ -18,6 +22,20 @@ def reference_config() -> dict: return json.loads((ROOT / "fleet.json").read_text(encoding="utf-8")) +def contract_schema() -> dict: + return json.loads((ROOT / "fleet.schema.json").read_text(encoding="utf-8")) + + +def schema_accepts_engine_ref(value: str) -> bool: + pattern = contract_schema()["$defs"]["controller"]["properties"]["engine_ref"]["pattern"] + return re.fullmatch(pattern, value) is not None + + +def schema_accepts_delivery_engine(value: str) -> bool: + contract = contract_schema()["properties"]["organization"]["properties"]["delivery_engine"] + return contract.get("const") == value + + def errors_for(config: dict, *, strict: bool = False) -> list[str]: validation = Validation() scan_secret_material(config, validation) @@ -29,19 +47,144 @@ def first_project(config: dict) -> dict: return next(iter(config["projects"].values())) +def first_controller(config: dict) -> dict: + return next(iter(config["controllers"].values())) + + class PolicyTests(unittest.TestCase): def assert_rejected(self, config: dict, expected: str, *, strict: bool = False) -> None: errors = errors_for(config, strict=strict) self.assertTrue(any(expected in error for error in errors), errors) + def assert_engine_ref_contract(self, value: str, accepted: bool) -> None: + config = copy.deepcopy(reference_config()) + first_controller(config)["engine_ref"] = value + self.assertEqual(schema_accepts_engine_ref(value), accepted) + self.assertEqual(errors_for(config) == [], accepted) + + def assert_delivery_engine_contract(self, value: str, accepted: bool) -> None: + config = copy.deepcopy(reference_config()) + config["organization"]["delivery_engine"] = value + self.assertEqual(schema_accepts_delivery_engine(value), accepted) + self.assertEqual(errors_for(config) == [], accepted) + def test_reference_configuration_is_valid(self) -> None: self.assertEqual(errors_for(reference_config()), []) + def test_multi_host_multi_location_configuration_is_valid(self) -> None: + config = json.loads((ROOT / "examples" / "multi-host" / "fleet.json").read_text(encoding="utf-8")) + self.assertEqual(errors_for(config), []) + + def test_schema_version_two_is_rejected(self) -> None: + config = copy.deepcopy(reference_config()) + config["schema_version"] = 2 + self.assert_rejected(config, "must equal 3") + def test_public_repository_access_is_rejected(self) -> None: config = copy.deepcopy(reference_config()) config["runner_pools"]["trusted-ci"]["public_repositories"] = True self.assert_rejected(config, "trusted private repositories") + def test_capacity_overcommit_is_rejected(self) -> None: + config = copy.deepcopy(reference_config()) + overcommit = config["runner_pools"]["trusted-ci"]["capacity_budget"] + 1 + first_controller(config)["max_runners"] = overcommit + self.assert_rejected(config, f"must cover {overcommit} runners") + + def test_drained_capacity_remains_reserved(self) -> None: + config = copy.deepcopy(reference_config()) + overcommit = config["runner_pools"]["trusted-ci"]["capacity_budget"] + 1 + first_controller(config)["state"] = "drained" + first_controller(config)["max_runners"] = overcommit + self.assert_rejected(config, f"must cover {overcommit} runners") + + def test_disabled_capacity_is_not_reserved(self) -> None: + config = copy.deepcopy(reference_config()) + first_controller(config)["state"] = "disabled" + first_controller(config)["max_runners"] = 100 + self.assertEqual(errors_for(config), []) + + def test_duplicate_scale_set_is_rejected(self) -> None: + config = copy.deepcopy(reference_config()) + duplicate = copy.deepcopy(first_controller(config)) + duplicate["location"] = "example-site-b" + config["controllers"]["example-ci-02"] = duplicate + config["runner_pools"]["trusted-ci"]["capacity_budget"] = 2 + self.assert_rejected(config, "scale_set_name: must be unique") + + def test_scale_set_must_include_controller_id(self) -> None: + config = copy.deepcopy(reference_config()) + first_controller(config)["scale_set_name"] = "other-scale" + self.assert_rejected(config, "must include the controller ID") + + def test_routing_label_must_not_equal_scale_set(self) -> None: + config = copy.deepcopy(reference_config()) + config["runner_pools"]["trusted-ci"]["routing_labels"] = [first_controller(config)["scale_set_name"]] + self.assert_rejected(config, "must not equal a controller scale-set name") + + def test_controller_pool_must_exist(self) -> None: + config = copy.deepcopy(reference_config()) + first_controller(config)["pool"] = "missing" + self.assert_rejected(config, "must reference a declared runner pool") + + def test_controller_pool_must_be_a_string(self) -> None: + config = copy.deepcopy(reference_config()) + first_controller(config)["pool"] = ["trusted-ci"] + self.assert_rejected(config, "must reference a declared runner pool") + + def test_delivery_engine_repository_is_fixed_in_schema_and_semantics(self) -> None: + self.assert_delivery_engine_contract("RandomDevelopment/ci-fleet", True) + + def test_delivery_engine_rejects_another_repository(self) -> None: + self.assert_delivery_engine_contract("attacker/engine", False) + + def test_delivery_engine_rejects_url_form(self) -> None: + self.assert_delivery_engine_contract("https://github.com/RandomDevelopment/ci-fleet", False) + + def test_delivery_engine_rejects_credential_form(self) -> None: + self.assert_delivery_engine_contract("user:password@RandomDevelopment/ci-fleet", False) + + def test_controller_address_is_rejected(self) -> None: + config = copy.deepcopy(reference_config()) + first_controller(config)["ip_address"] = "192.0.2.10" + self.assert_rejected(config, "host-local infrastructure details are forbidden") + + def test_full_lowercase_engine_ref_passes_schema_and_semantics(self) -> None: + self.assert_engine_ref_contract("1" * 40, True) + + def test_uppercase_engine_ref_fails_schema_and_semantics(self) -> None: + self.assert_engine_ref_contract("A" * 40, False) + + def test_short_engine_ref_fails_schema_and_semantics(self) -> None: + self.assert_engine_ref_contract("1" * 39, False) + + def test_malformed_engine_ref_fails_schema_and_semantics(self) -> None: + self.assert_engine_ref_contract("g" * 40, False) + + def test_zero_engine_ref_fails_schema_and_semantics(self) -> None: + self.assert_engine_ref_contract("0" * 40, False) + + def test_active_controller_minimum_must_be_zero(self) -> None: + config = copy.deepcopy(reference_config()) + first_controller(config)["min_runners"] = 1 + self.assert_rejected(config, "managed prewarmed runners are not supported") + + def test_drained_controller_minimum_must_be_zero(self) -> None: + config = copy.deepcopy(reference_config()) + first_controller(config)["state"] = "drained" + first_controller(config)["min_runners"] = 1 + self.assert_rejected(config, "managed prewarmed runners are not supported") + + def test_application_capacity_control_is_rejected(self) -> None: + config = copy.deepcopy(reference_config()) + first_project(config)["ci_contract"]["max_parallel"] = 1 + self.assert_rejected(config, "unknown keys: max_parallel") + + def test_pool_must_submit_all_independent_jobs(self) -> None: + config = copy.deepcopy(reference_config()) + config["runner_pools"]["trusted-ci"]["job_submission_policy"] = "max-parallel" + self.assert_rejected(config, "leave capacity control to infrastructure") + def test_automatic_production_is_rejected(self) -> None: config = copy.deepcopy(reference_config()) config["environments"]["production"]["automatic"] = True @@ -59,7 +202,7 @@ def test_repository_must_be_in_pool_allowlist(self) -> None: def test_embedded_credential_url_is_rejected(self) -> None: config = copy.deepcopy(reference_config()) - config["organization"]["database_url"] = "postgres://user:password@db.example.invalid/app" + config["organization"]["database_url"] = "post" + "gres://user:***@db.example.invalid/app" self.assert_rejected(config, "probable secret material") def test_secret_value_key_is_rejected(self) -> None: @@ -67,6 +210,46 @@ def test_secret_value_key_is_rejected(self) -> None: config["environments"]["development"]["token"] = "not-a-real-token" self.assert_rejected(config, "secret values are forbidden") + def test_host_local_environment_paths_are_rejected(self) -> None: + validation = Validation() + with tempfile.TemporaryDirectory() as directory: + path_list = Path(directory) / "paths" + path_list.write_bytes(b"host.env\0ci-fleet.env\0nested/host.env\0nested/ci-fleet.env\0") + scan_tree_path_list(path_list, validation) + self.assertEqual(len(validation.errors), 4, validation.errors) + self.assertTrue(all("secret-bearing files are forbidden" in error for error in validation.errors), validation.errors) + + def test_template_ci_scans_committed_file_contents(self) -> None: + scanner = ROOT / "scripts" / "scan_committed_secrets.py" + self.assertTrue(scanner.is_file()) + workflow = (ROOT / ".github" / "workflows" / "validate.yml").read_text(encoding="utf-8") + self.assertIn("python3 scripts/scan_committed_secrets.py", workflow) + with tempfile.TemporaryDirectory() as directory: + repository = Path(directory) + subprocess.run(["git", "init", "-q", str(repository)], check=True) + for relative in ("scripts/scan_committed_secrets.py", "scripts/validate.sh"): + path = repository / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("ghp_" + "x" * 20 + "\n", encoding="utf-8") + subprocess.run(["git", "-C", str(repository), "add", "."], check=True) + result = subprocess.run( + [sys.executable, str(scanner), "--repository", str(repository)], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + self.assertNotEqual(result.returncode, 0) + self.assertIn("scripts/scan_committed_secrets.py:1", result.stderr) + self.assertIn("scripts/validate.sh:1", result.stderr) + + def test_duplicate_json_controller_id_is_rejected(self) -> None: + validation = Validation() + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "duplicate.json" + path.write_text('{"controllers":{"ci-01":{},"ci-01":{}}}', encoding="utf-8") + self.assertIsNone(load_json(path, validation)) + self.assertTrue(any("duplicate object key: ci-01" in error for error in validation.errors), validation.errors) + def test_strict_mode_rejects_unchanged_example(self) -> None: config = copy.deepcopy(reference_config()) project = first_project(config) diff --git a/templates/config-repository/scripts/validate.py b/templates/config-repository/scripts/validate.py index f644f657..5b70b92f 100755 --- a/templates/config-repository/scripts/validate.py +++ b/templates/config-repository/scripts/validate.py @@ -17,6 +17,7 @@ REPOSITORY = re.compile(r"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$") IMAGE = re.compile(r"^[a-z0-9.-]+/[a-z0-9._/-]+$") SECRET_NAME = re.compile(r"^[A-Z][A-Z0-9_]*$") +COMMIT_SHA = re.compile(r"^[0-9a-f]{40}$") HIGH_CONFIDENCE_SECRET_PATTERNS = ( re.compile(r"-----BEGIN [A-Z ]*PRIVATE KEY-----"), re.compile(r"github_pat_[A-Za-z0-9_]{20,}"), @@ -36,7 +37,23 @@ "secret_value", "token", } -FORBIDDEN_FILENAMES = re.compile(r"(?:^|/)\.env(?:\..+)?$|\.(?:key|pem|p12|pfx)$", re.IGNORECASE) +FORBIDDEN_INFRASTRUCTURE_KEYS = { + "backup_id", + "backup_snapshot", + "backup_storage", + "disk_storage", + "host_address", + "hostname", + "ip", + "ip_address", + "proxmox_vmid", + "ssh_host", + "ssh_password", + "ssh_private_key", + "vm_id", + "vmid", +} +FORBIDDEN_FILENAMES = re.compile(r"(?:^|/)(?:\.env(?:\..+)?|host\.env|ci-fleet\.env)$|\.(?:key|pem|p12|pfx)$", re.IGNORECASE) FORBIDDEN_DIRECTORIES = {"credentials", "private", "secrets"} @@ -64,12 +81,22 @@ def exact_keys(self, value: Any, path: str, required: set[str], optional: set[st def load_json(path: Path, validation: Validation) -> Any: + def reject_duplicate_keys(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + value: dict[str, Any] = {} + for key, child in pairs: + if key in value: + raise ValueError(f"duplicate object key: {key}") + value[key] = child + return value + try: - return json.loads(path.read_text(encoding="utf-8")) + return json.loads(path.read_text(encoding="utf-8"), object_pairs_hook=reject_duplicate_keys) except FileNotFoundError: validation.errors.append(f"{path}: file not found") except json.JSONDecodeError as exc: validation.errors.append(f"{path}:{exc.lineno}:{exc.colno}: invalid JSON: {exc.msg}") + except ValueError as exc: + validation.errors.append(f"{path}: invalid JSON: {exc}") return None @@ -88,10 +115,15 @@ def scan_secret_material(config: Any, validation: Validation) -> None: def scan_keys(value: Any, path: str = "$") -> None: if isinstance(value, dict): for key, child in value.items(): - if key.lower() in FORBIDDEN_SECRET_KEYS: + normalized = key.lower() + if normalized in FORBIDDEN_SECRET_KEYS: validation.errors.append( f"{path}.{key}: secret values are forbidden; store only an uppercase secret name" ) + if normalized in FORBIDDEN_INFRASTRUCTURE_KEYS: + validation.errors.append( + f"{path}.{key}: host-local infrastructure details are forbidden in Git-authored policy" + ) scan_keys(child, f"{path}.{key}") elif isinstance(value, list): for index, child in enumerate(value): @@ -120,12 +152,41 @@ def scan_forbidden_paths(repo_root: Path, validation: Validation) -> None: validation.errors.append(f"{relative_text}: secret-bearing files are forbidden") +def scan_tree_path_list(path_list: Path, validation: Validation) -> None: + try: + raw_paths = path_list.read_bytes().split(b"\0") + except OSError as error: + validation.errors.append(f"{path_list}: cannot read tree path list: {error}") + return + for raw_path in raw_paths: + if not raw_path: + continue + try: + relative_text = raw_path.decode("utf-8") + except UnicodeDecodeError: + validation.errors.append("repository tree contains a non-UTF-8 path") + continue + parts = Path(relative_text).parts + if any(part.lower() in FORBIDDEN_DIRECTORIES for part in parts[:-1]): + validation.errors.append(f"{relative_text}: secret-bearing directory names are forbidden") + elif FORBIDDEN_FILENAMES.search(relative_text): + validation.errors.append(f"{relative_text}: secret-bearing files are forbidden") + + def validate_config(config: Any, validation: Validation, strict: bool) -> None: - required_top = {"schema_version", "organization", "runner_pools", "host_groups", "environments", "projects"} + required_top = { + "schema_version", + "organization", + "runner_pools", + "controllers", + "host_groups", + "environments", + "projects", + } if not validation.exact_keys(config, "$", required_top, {"$schema"}): return - validation.require(config.get("schema_version") == 2, "$.schema_version", "must equal 2") + validation.require(config.get("schema_version") == 3, "$.schema_version", "must equal 3") organization = config.get("organization") organization_keys = {"slug", "registry", "delivery_engine", "workflow_ref_policy"} @@ -136,6 +197,7 @@ def validate_config(config: Any, validation: Validation, strict: bool) -> None: validation.require(isinstance(slug, str) and bool(ORG_SLUG.fullmatch(slug)), "$.organization.slug", "must be a lowercase GitHub organization slug") validation.require(isinstance(registry, str) and bool(IMAGE.fullmatch(registry)), "$.organization.registry", "must be a registry namespace such as ghcr.io/acme") validation.require(isinstance(engine, str) and bool(REPOSITORY.fullmatch(engine)), "$.organization.delivery_engine", "must be an owner/repository name") + validation.require(engine == "RandomDevelopment/ci-fleet", "$.organization.delivery_engine", "must use the fixed reviewed public engine repository") validation.require(organization.get("workflow_ref_policy") == "immutable-commit", "$.organization.workflow_ref_policy", "must equal immutable-commit") if strict: validation.require(slug != "example-org", "$.organization.slug", "replace the example organization before use") @@ -144,13 +206,24 @@ def validate_config(config: Any, validation: Validation, strict: bool) -> None: if not isinstance(pools, dict) or not pools: validation.errors.append("$.runner_pools: must be a non-empty object") pools = {} + pool_capacity: dict[str, int] = {} for name, pool in pools.items(): path = f"$.runner_pools.{name}" - validation.require(bool(SLUG.fullmatch(name)), path, "pool name must be a lowercase slug") - if not validation.exact_keys(pool, path, {"routing_labels", "allowed_repositories", "public_repositories", "max_concurrent_jobs"}): + validation.require(isinstance(name, str) and bool(SLUG.fullmatch(name)), path, "pool name must be a lowercase slug") + pool_keys = { + "runner_group", + "routing_labels", + "allowed_repositories", + "public_repositories", + "capacity_budget", + "job_submission_policy", + } + if not validation.exact_keys(pool, path, pool_keys): continue + runner_group = pool.get("runner_group") labels = pool.get("routing_labels") repos = pool.get("allowed_repositories") + validation.require(isinstance(runner_group, str) and bool(SLUG.fullmatch(runner_group)), f"{path}.runner_group", "must be a lowercase logical runner-group slug") validation.require(isinstance(labels, list) and bool(labels), f"{path}.routing_labels", "must be a non-empty list") if isinstance(labels, list): validation.require(len(labels) == len(set(labels)), f"{path}.routing_labels", "must contain unique labels") @@ -163,8 +236,80 @@ def validate_config(config: Any, validation: Validation, strict: bool) -> None: for index, repository in enumerate(repos): validation.require(isinstance(repository, str) and bool(REPOSITORY.fullmatch(repository)), f"{path}.allowed_repositories[{index}]", "must be owner/repository") validation.require(pool.get("public_repositories") is False, f"{path}.public_repositories", "must be false; this fleet is for trusted private repositories") - jobs = pool.get("max_concurrent_jobs") - validation.require(type(jobs) is int and jobs > 0, f"{path}.max_concurrent_jobs", "must be a positive integer") + budget = pool.get("capacity_budget") + validation.require(type(budget) is int and budget > 0, f"{path}.capacity_budget", "must be a positive infrastructure capacity budget") + if type(budget) is int and budget > 0: + pool_capacity[name] = budget + validation.require(pool.get("job_submission_policy") == "all-independent-jobs", f"{path}.job_submission_policy", "must submit every independent job and leave capacity control to infrastructure") + + controllers = config.get("controllers") + if not isinstance(controllers, dict) or not controllers: + validation.errors.append("$.controllers: must be a non-empty object") + controllers = {} + reserved_capacity = {name: 0 for name in pools} + scale_sets: dict[str, str] = {} + controller_keys = { + "pool", + "location", + "state", + "scale_set_name", + "lifecycle", + "engine_ref", + "min_runners", + "max_runners", + "runner_resources", + } + for name, controller in controllers.items(): + path = f"$.controllers.{name}" + validation.require(isinstance(name, str) and bool(SLUG.fullmatch(name)), path, "controller ID must be a unique lowercase slug") + if not validation.exact_keys(controller, path, controller_keys): + continue + pool_name = controller.get("pool") + location = controller.get("location") + state = controller.get("state") + scale_set = controller.get("scale_set_name") + lifecycle = controller.get("lifecycle") + engine_ref = controller.get("engine_ref") + minimum = controller.get("min_runners") + maximum = controller.get("max_runners") + validation.require(isinstance(pool_name, str) and pool_name in pools, f"{path}.pool", "must reference a declared runner pool") + validation.require(isinstance(location, str) and bool(SLUG.fullmatch(location)), f"{path}.location", "must be a logical location slug, never an address") + validation.require(state in {"active", "drained", "disabled"}, f"{path}.state", "must be active, drained, or disabled") + validation.require(isinstance(scale_set, str) and bool(SLUG.fullmatch(scale_set)), f"{path}.scale_set_name", "must be a lowercase scale-set slug") + if isinstance(scale_set, str) and isinstance(name, str): + validation.require(name in scale_set, f"{path}.scale_set_name", "must include the controller ID required by managed preflight") + if isinstance(scale_set, str): + if scale_set in scale_sets: + validation.errors.append(f"{path}.scale_set_name: must be unique; also used by {scale_sets[scale_set]}") + else: + scale_sets[scale_set] = name + validation.require(lifecycle in {"experimental", "stable", "retiring"}, f"{path}.lifecycle", "must be experimental, stable, or retiring") + validation.require(isinstance(engine_ref, str) and bool(COMMIT_SHA.fullmatch(engine_ref)) and engine_ref != "0" * 40, f"{path}.engine_ref", "must be a nonzero full lowercase commit SHA") + validation.require(type(minimum) is int and minimum >= 0, f"{path}.min_runners", "must be a non-negative integer") + validation.require(type(maximum) is int and maximum > 0, f"{path}.max_runners", "must be a positive integer") + if type(minimum) is int and type(maximum) is int: + validation.require(minimum <= maximum, f"{path}.min_runners", "must not exceed max_runners") + validation.require(minimum == 0, f"{path}.min_runners", "must be zero because managed prewarmed runners are not supported") + resources = controller.get("runner_resources") + if validation.exact_keys(resources, f"{path}.runner_resources", {"cpu_cores", "memory_mib"}): + cpu = resources.get("cpu_cores") + memory = resources.get("memory_mib") + validation.require(type(cpu) is int and cpu > 0, f"{path}.runner_resources.cpu_cores", "must be a positive integer") + validation.require(type(memory) is int and memory >= 512, f"{path}.runner_resources.memory_mib", "must be at least 512 MiB") + if isinstance(pool_name, str) and pool_name in pools and state != "disabled" and type(maximum) is int and maximum > 0: + reserved_capacity[pool_name] += maximum + + for pool_name, pool in pools.items(): + labels = pool.get("routing_labels") if isinstance(pool, dict) else None + if isinstance(labels, list): + for index, label in enumerate(labels): + if isinstance(label, str): + validation.require(label not in scale_sets, f"$.runner_pools.{pool_name}.routing_labels[{index}]", "must not equal a controller scale-set name") + + for name, reserved in reserved_capacity.items(): + budget = pool_capacity.get(name) + if budget is not None: + validation.require(reserved <= budget, f"$.runner_pools.{name}.capacity_budget", f"must cover {reserved} runners reserved by active or drained controllers") groups = config.get("host_groups") if not isinstance(groups, dict) or not groups: @@ -214,8 +359,8 @@ def validate_config(config: Any, validation: Validation, strict: bool) -> None: pool_name = project.get("ci_pool") validation.require(isinstance(repository, str) and bool(REPOSITORY.fullmatch(repository)), f"{path}.repository", "must be owner/repository") validation.require(isinstance(image, str) and bool(IMAGE.fullmatch(image)), f"{path}.image", "must be a container image path without a mutable tag") - validation.require(pool_name in pools, f"{path}.ci_pool", "must reference a declared runner pool") - if pool_name in pools and isinstance(pools[pool_name].get("allowed_repositories"), list): + validation.require(isinstance(pool_name, str) and pool_name in pools, f"{path}.ci_pool", "must reference a declared runner pool") + if isinstance(pool_name, str) and pool_name in pools and isinstance(pools[pool_name].get("allowed_repositories"), list): validation.require(repository in pools[pool_name]["allowed_repositories"], f"{path}.repository", "must be explicitly allowed by its CI pool") contract = project.get("ci_contract") contract_path = f"{path}.ci_contract" @@ -254,6 +399,7 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--config", type=Path, default=ROOT / "fleet.json", help="configuration file to validate") parser.add_argument("--strict", action="store_true", help="reject unchanged example values") parser.add_argument("--skip-path-scan", action="store_true", help="skip repository path checks (for external fixtures)") + parser.add_argument("--tree-paths", type=Path, help="NUL-delimited committed paths to scan instead of the local template tree") return parser.parse_args() @@ -267,7 +413,9 @@ def main() -> int: if config is not None: scan_secret_material(config, validation) validate_config(config, validation, args.strict) - if not args.skip_path_scan: + if args.tree_paths is not None: + scan_tree_path_list(args.tree_paths, validation) + elif not args.skip_path_scan: scan_forbidden_paths(ROOT, validation) if validation.errors: