diff --git a/docs/GITHUB-APP-SETUP.md b/docs/GITHUB-APP-SETUP.md index 7ada9610..5605470a 100644 --- a/docs/GITHUB-APP-SETUP.md +++ b/docs/GITHUB-APP-SETUP.md @@ -44,11 +44,259 @@ Nothing else. No `write` on contents, no actions, no administration. If the controller ever needs more, that is a reviewed design change, not a settings tweak. -## 3. Generate the private key +## 3. Generate and transfer the private key On the app page: Private keys → Generate a private key. GitHub downloads one -PEM. Store it only on the controller host, root-owned `0600`. It is never -committed, printed, or copied elsewhere — see [SECRETS.md](SECRETS.md). +PEM to the management workstation. Treat that download as a temporary copy: +transfer it over an authenticated, encrypted channel directly to the final +root-owned path on the controller. Do not stage it in a shared directory or +send it through chat, email, or a repository. + +For an initial installation, the destination may use the conventional active +path because no controller key exists yet. Set `PEM_DEST` explicitly when a +different installation path is wanted: + +```bash +PEM="$HOME/Downloads/YOUR-APP.private-key.pem" +CONTROLLER=root@CONTROLLER_HOST +ACTIVE_PEM="" +PEM_DEST=${PEM_DEST:-"/etc/ci-fleet/secrets/github-app.pem"} +``` + +For rotation, replace the last two assignments with the exact active path from +`CI_FLEET_GITHUB_APP_PRIVATE_KEY_FILE` and a new key-specific filename. Never +use the active path as the rotation destination: + +```bash +ACTIVE_PEM="/etc/ci-fleet/secrets/github-app.pem" +PEM_DEST="/etc/ci-fleet/secrets/github-app-ROTATION-ID.pem" +``` + +Lifecycle commands use canonical absolute paths so path comparisons and exact +deletion cannot change meaning. Before rotating, revoking, or directly retiring +an existing non-canonical path +(for example one containing `..`, repeated separators, or a symlinked parent), +validate it as shell-safe and resolve it on the controller—not the workstation: + +```bash +[[ $ACTIVE_PEM =~ ^/[A-Za-z0-9._/-]+$ ]] || exit 1 +# ACTIVE_PEM is shell-safe above and intentionally expanded client-side. +# shellcheck disable=SC2029 +ACTIVE_PEM=$(ssh "$CONTROLLER" "readlink -f -- \"$ACTIVE_PEM\"") || exit 1 +``` + +Update controller `host.env` to that canonical result, then reconcile and verify +healthy convergence before continuing. Record the result as `ACTIVE_PEM`. + +Before either transfer workflow, reject the active path and key itself as the +replacement. This preflight runs before importing a manager-backed version: + +```bash +[[ $PEM_DEST =~ ^/[A-Za-z0-9._/-]+$ ]] || exit 1 +set -o pipefail +replacement_pubkey_sha=$(openssl pkey -in "$PEM" -pubout -outform DER | \ + sha256sum) || exit 1 +replacement_pubkey_sha=${replacement_pubkey_sha%% *} +[[ "$replacement_pubkey_sha" =~ ^[0-9a-f]{64}$ ]] || exit 1 +if [[ -n "$ACTIVE_PEM" ]]; then + [[ $ACTIVE_PEM =~ ^/[A-Za-z0-9._/-]+$ ]] || exit 1 + [[ "$PEM_DEST" != "$ACTIVE_PEM" ]] || exit 1 + # ACTIVE_PEM is shell-safe above and intentionally expanded client-side. + # shellcheck disable=SC2029 + active_pubkey_sha=$(ssh "$CONTROLLER" \ + "bash -o pipefail -c 'openssl pkey -in \"$ACTIVE_PEM\" -pubout -outform DER | sha256sum'") || exit 1 + active_pubkey_sha=${active_pubkey_sha%% *} + [[ "$active_pubkey_sha" =~ ^[0-9a-f]{64}$ ]] || exit 1 + [[ "$replacement_pubkey_sha" != "$active_pubkey_sha" ]] || exit 1 + unset active_pubkey_sha +fi +``` + +The SSH workflow below is only for a host-local destination. For a +secret-manager-backed destination, do not write into the materialized mount. +Instead, use that manager's authenticated import/version operation to create a +new inactive version from `$PEM`, materialize it at a distinct canonical +`PEM_DEST`, and run the following verification. Activate that version only in +step 2 of the rotation procedure. If verification fails, retain the download, +remove only the new inactive version through the manager, and leave the active +version and path untouched: + +```bash +# PEM_DEST is canonical and intentionally expanded client-side. +# shellcheck disable=SC2029 +if + [[ $PEM_DEST =~ ^/[A-Za-z0-9._/-]+$ ]] && + ssh "$CONTROLLER" \ + "test \"\$(realpath -m -- \"$PEM_DEST\")\" = \"$PEM_DEST\"" && + ssh "$CONTROLLER" " + secure_pem_ancestors() { + dir=\"${PEM_DEST%/*}\"; + test -n \"\$dir\" || dir=/; + while :; do + test \"\$(stat -c '%U' -- \"\$dir\")\" = root || return 1; + test -z \"\$(find \"\$dir\" -maxdepth 0 -perm /022 -print -quit)\" || return 1; + test \"\$dir\" != / || break; + dir=\${dir%/*}; + test -n \"\$dir\" || dir=/; + done; + } + test \"\$(stat -c '%F' -- \"$PEM_DEST\")\" = 'regular file' && + secure_pem_ancestors + " && + local_sha=$(sha256sum -- "$PEM") && + local_sha=${local_sha%% *} && + [[ "$local_sha" =~ ^[0-9a-f]{64}$ ]] && + remote_sha=$(ssh "$CONTROLLER" "sha256sum -- \"$PEM_DEST\"") && + remote_sha=${remote_sha%% *} && + [[ "$remote_sha" =~ ^[0-9a-f]{64}$ ]] && + test "$local_sha" = "$remote_sha" && + remote_pubkey_sha=$(ssh "$CONTROLLER" \ + "bash -o pipefail -c 'openssl pkey -in \"$PEM_DEST\" -pubout -outform DER | sha256sum'") && + remote_pubkey_sha=${remote_pubkey_sha%% *} && + [[ "$remote_pubkey_sha" =~ ^[0-9a-f]{64}$ ]] && + test "$replacement_pubkey_sha" = "$remote_pubkey_sha" && + ssh "$CONTROLLER" \ + "test \"\$(stat -c '%U:%G' -- \"$PEM_DEST\")\" = 'root:root'" && + ssh "$CONTROLLER" \ + "test \"\$(stat -c '%a' -- \"$PEM_DEST\")\" = '600'" +then + rm -f -- "$PEM" || exit 1 + unset replacement_pubkey_sha remote_pubkey_sha +else + printf 'manager import verification failed; retained download: %s\n' \ + "$PEM" >&2 + exit 1 +fi +``` + +Manager layouts whose materialized file or any containing directory fails +these checks are unsupported until secured. + +For a host-local destination, run the transfer and verification sequence below. +The path +validation makes it safe to quote `PEM_DEST` in the remote shell command, the +rotation guard rejects the active path, and an atomic hard-link creation refuses +any existing destination node before the key can appear at that path: + +```bash +valid_pem_path() { + [[ $1 =~ ^/[A-Za-z0-9._/-]+$ ]] +} +valid_pem_path "$PEM_DEST" || exit 1 +if [[ -n "$ACTIVE_PEM" && "$PEM_DEST" == "$ACTIVE_PEM" ]]; then + printf 'refusing to overwrite active PEM: %s\n' "$ACTIVE_PEM" >&2 + exit 1 +fi +PEM_DIR=${PEM_DEST%/*} +[[ -n "$PEM_DIR" ]] || PEM_DIR=/ +TRANSFER_ID=$(< /proc/sys/kernel/random/uuid) || exit 1 +PEM_MARKER="$PEM_DIR/.ci-fleet-transfer-$TRANSFER_ID" +[[ $PEM_MARKER =~ ^/[A-Za-z0-9._/-]+$ ]] || exit 1 +[[ -z "$ACTIVE_PEM" || "$PEM_MARKER" != "$ACTIVE_PEM" ]] || exit 1 + +# PEM_DIR and PEM_DEST are validated above and intentionally expanded locally. +# shellcheck disable=SC2029 +if +ssh "$CONTROLLER" \ + "test \"\$(realpath -m -- \"$PEM_DEST\")\" = \"$PEM_DEST\"" && +ssh "$CONTROLLER" " + secure_pem_ancestors() { + dir=\"$PEM_DIR\"; + while :; do + test \"\$(stat -c '%U' -- \"\$dir\")\" = root || return 1; + test -z \"\$(find \"\$dir\" -maxdepth 0 -perm /022 -print -quit)\" || return 1; + test \"\$dir\" != / || break; + dir=\${dir%/*}; + test -n \"\$dir\" || dir=/; + done; + } + { test -d \"$PEM_DIR\" || install -d -m 0700 \"$PEM_DIR\"; } && + secure_pem_ancestors && + umask 077 && + tmp=\$(mktemp -- \"$PEM_DIR/.github-app-key.XXXXXX\") && + trap 'status=\$?; + if [ \"\$status\" -ne 0 ] && + [ -e \"$PEM_MARKER\" ] && [ \"$PEM_MARKER\" -ef \"\$tmp\" ]; then + if [ -e \"$PEM_DEST\" ] && [ \"$PEM_DEST\" -ef \"$PEM_MARKER\" ]; then + rm -f -- \"$PEM_DEST\"; + fi; + rm -f -- \"$PEM_MARKER\"; + fi; + rm -f -- \"\$tmp\"; + exit \"\$status\"' 0 && + cat >\"\$tmp\" && + chown root:root \"\$tmp\" && + chmod 0600 \"\$tmp\" && + ln -T -- \"\$tmp\" \"$PEM_MARKER\" && + ln -T -- \"\$tmp\" \"$PEM_DEST\" && + rm -f -- \"\$tmp\" && + trap - 0 +" <"$PEM" && + local_sha=$(sha256sum -- "$PEM") && + local_sha=${local_sha%% *} && + [[ "$local_sha" =~ ^[0-9a-f]{64}$ ]] && + remote_sha=$(ssh "$CONTROLLER" "sha256sum -- \"$PEM_DEST\"") && + remote_sha=${remote_sha%% *} && + [[ "$remote_sha" =~ ^[0-9a-f]{64}$ ]] && + test "$local_sha" = "$remote_sha" && + remote_pubkey_sha=$(ssh "$CONTROLLER" \ + "bash -o pipefail -c 'openssl pkey -in \"$PEM_DEST\" -pubout -outform DER | sha256sum'") && + remote_pubkey_sha=${remote_pubkey_sha%% *} && + [[ "$remote_pubkey_sha" =~ ^[0-9a-f]{64}$ ]] && + test "$replacement_pubkey_sha" = "$remote_pubkey_sha" && + ssh "$CONTROLLER" \ + "test \"\$(stat -c '%U:%G' -- \"$PEM_DEST\")\" = 'root:root'" && + ssh "$CONTROLLER" \ + "test \"\$(stat -c '%a' -- \"$PEM_DEST\")\" = '600'" && + ssh "$CONTROLLER" \ + "test \"$PEM_MARKER\" -ef \"$PEM_DEST\"" +then + if ! ssh "$CONTROLLER" \ + "if test -e \"$PEM_MARKER\"; then + test \"$PEM_MARKER\" -ef \"$PEM_DEST\" && rm -f -- \"$PEM_MARKER\"; + fi"; then + printf 'verified transfer; retry idempotent marker cleanup before activation: %s\n' \ + "$PEM_MARKER" >&2 + exit 1 + fi + if rm -f -- "$PEM"; then + unset PEM local_sha remote_sha remote_pubkey_sha replacement_pubkey_sha \ + TRANSFER_ID PEM_MARKER + else + printf 'verified transfer, but could not delete downloaded PEM: %s\n' \ + "$PEM" >&2 + exit 1 + fi +else + if ! ssh "$CONTROLLER" \ + "if test -e \"$PEM_MARKER\" && test \"$PEM_MARKER\" -ef \"$PEM_DEST\"; then + rm -f -- \"$PEM_DEST\" \"$PEM_MARKER\"; + elif test -e \"$PEM_MARKER\"; then rm -f -- \"$PEM_MARKER\"; fi"; then + printf 'remote ownership cleanup failed; retain and retry marker %s for destination %s\n' \ + "$PEM_MARKER" "$PEM_DEST" >&2 + unset local_sha remote_sha TRANSFER_ID + exit 1 + fi + printf 'transfer verification failed; retained downloaded PEM: %s\n' "$PEM" >&2 + unset local_sha remote_sha TRANSFER_ID PEM_MARKER + exit 1 +fi +``` + +Use an equivalent privileged SSH workflow if direct root login is disabled. +If transfer, checksum, owner, or mode verification fails, the sequence stops. +It removes `PEM_DEST` only when the per-transfer hard-link marker proves that +this invocation created that inode, including after an ambiguous SSH result; +pre-existing destinations are preserved. The active controller PEM remains +untouched and the downloaded replacement is retained for diagnosis or a safe +retry. If marker cleanup itself returns an ambiguous SSH result, rerun only its +idempotent cleanup command before activation; do not rerun the transfer. Do not +revoke the old GitHub key. +After success, remove any other copy from the browser download location, trash, +sync, and temporary storage according to the workstation's secure-erasure +policy; plain `rm` may not erase data from snapshots, SSDs, or copy-on-write +storage. The only retained copy may be the controller file or an approved +secret manager — see [SECRETS.md](SECRETS.md). ## 4. Install the app @@ -75,26 +323,37 @@ The controller exchanges a short-lived JWT signed with the PEM for an installation token at runtime (`scripts/github-app-token.sh`). No token is stored. -## 6. Verify +## 6. Pre-install verification from the reviewed checkout -On the controller host: +Before the first managed installation, `/opt/ci-fleet/manager/current` and the +installed-state file do not exist. On the controller host, run the token helper +from the exact reviewed engine checkout that will be installed: ```bash -sudo /opt/ci-fleet/manager/current/scripts/github-app-token.sh \ - --env-file /etc/ci-fleet/host.env +sudo /PATH/TO/REVIEWED/ci-fleet/scripts/github-app-token.sh \ + --env-file /etc/ci-fleet/host.env >/dev/null ``` -Prints nothing secret; exit 0 means JWT signing and token exchange work. -Then a check-only reconcile validates the full fetch path without applying -anything: +The helper writes the installation token to stdout, so the redirection is +mandatory. Exit 0 means JWT signing and token exchange work; it does not prove +that an installed controller can reconcile. Continue with the managed install +workflow before using an installed-manager command. + +## 7. Post-install remote-reconciliation verification + +Only after a successful managed install, validate the complete fetch and +installed-state path without applying anything: ```bash -sudo /opt/ci-fleet/manager/current/scripts/remote-reconcile.sh --check-only +sudo /opt/ci-fleet/manager/current/scripts/remote-reconcile.sh \ + --check-only --installed-ref ``` -`CHECK_OK` means the app can read the desired-state repository. A 403 or -"Repository not found" means the installation lacks the repository or the -`contents: read` grant — see Troubleshooting. +`RECONCILE CONVERGED` means the app can read and validate the installed desired-state +commit and the controller is converged. The reconcile script consumes its +token internally; it does not print the token. A 403 or "Repository not found" +means the installation lacks the repository or the `contents: read` grant — +see Troubleshooting. ## Troubleshooting @@ -104,9 +363,222 @@ sudo /opt/ci-fleet/manager/current/scripts/remote-reconcile.sh --check-only | 403 on content API after granting permission | permission change saved on the app but not yet accepted on the installation — reopen the installation page and approve the pending permission request | | 401 on token exchange | wrong client ID, installation ID, or PEM path in `host.env` | -## Rotation and removal - -- New controller: new app. Do not share one app across controllers. -- Rotate: generate a new key, update the PEM path, delete the old key. -- Retire: uninstall the app from the organization. The host keeps no usable - credential. +## Key rotation: activate and verify before revocation + +New controller: new app. Do not share one app across controllers. + +1. Generate a new key and use the rotation assignments and secure-transfer + procedure in section 3. `PEM_DEST` must differ from `ACTIVE_PEM`; the + transfer must not overwrite either path. Continue only after every transfer + and verification command succeeds and the downloaded workstation copy is + removed. +2. Before changing `host.env`, record the exact value of `ACTIVE_PEM` for + rollback and old-file removal. Then update + `CI_FLEET_GITHUB_APP_PRIVATE_KEY_FILE` in `/etc/ci-fleet/host.env` to the + exact value of `PEM_DEST`. +3. Verify that the new key can mint a token, always suppressing token stdout: + + ```bash + sudo /opt/ci-fleet/manager/current/scripts/github-app-token.sh \ + --env-file /etc/ci-fleet/host.env >/dev/null + ``` + + If token generation fails, immediately restore + `CI_FLEET_GITHUB_APP_PRIVATE_KEY_FILE` in `host.env` to the exact recorded + `ACTIVE_PEM`, verify token generation with the old key, and run normal + reconciliation. Stop; retain both PEMs and do not revoke the old GitHub key. + +4. Run a normal reconciliation, not `--check-only`. The host-configuration + drift forces the installer upgrade path, recreates the controller with the + new PEM mount, and runs its post-activation health check: + + ```bash + sudo /opt/ci-fleet/manager/current/scripts/remote-reconcile.sh + sudo /opt/ci-fleet/current/scripts/healthcheck.sh || exit 1 + sudo /opt/ci-fleet/manager/current/scripts/remote-reconcile.sh \ + --check-only --installed-ref + ``` + + Stop if reconciliation does not report `RECONCILE_OK`, the health check is + not `healthy` for an `active` controller or `maintenance` for a controller + whose reviewed desired state is `drained` or `disabled`, or the final check + does not report `RECONCILE CONVERGED`. Warning and unhealthy results always fail. + Restore + `CI_FLEET_GITHUB_APP_PRIVATE_KEY_FILE` to the exact value of `ACTIVE_PEM` + and reconcile again; do not revoke the old key or remove either PEM until + rollback is healthy and converged. +5. Before revoking the old key, verify that the newest complete rollback + checkpoint references `PEM_DEST`, not `ACTIVE_PEM`: + + ```bash + valid_pem_path() { + [[ $1 =~ ^/[A-Za-z0-9._/-]+$ ]] && + [[ $(realpath -m -- "$1") == "$1" ]] + } + PEM_DEST=$(sudo grep -E '^CI_FLEET_GITHUB_APP_PRIVATE_KEY_FILE=' \ + /etc/ci-fleet/host.env | cut -d= -f2-) + valid_pem_path "$PEM_DEST" || exit 1 + LATEST_CHECKPOINT=$(sudo find /var/lib/ci-fleet/checkpoints \ + -mindepth 2 -maxdepth 2 -type f -name .complete \ + ! -path '/var/lib/ci-fleet/checkpoints/.checkpoint.staging.*/*' \ + -printf '%T@ %h\n' | sort -nr | awk 'NR == 1 {print $2}') + [[ -n "$LATEST_CHECKPOINT" ]] || exit 1 + sudo grep -Fx -- \ + "CI_FLEET_GITHUB_APP_PRIVATE_KEY_FILE=$PEM_DEST" \ + "$LATEST_CHECKPOINT/ci-fleet.env" >/dev/null || exit 1 + ``` + + The rotation itself checkpoints the pre-rotation environment. If this gate + fails, retain the old GitHub key and old PEM until a subsequent reviewed + controller mutation creates and validates a checkpoint based on the new + path. Do not revoke a key still required by the latest rollback checkpoint. + +## Old-key revocation + +Only after every activation check above succeeds: + +1. In the current shell, read the new active destination from `host.env` and + reassign `ACTIVE_PEM` to the exact old path recorded before activation. + Put it in exactly one array: `OLD_LOCAL_PEMS` for host-local storage or + `OLD_MANAGED_PEMS` for secret-manager-backed storage. Remove manager-backed + material through that manager and verify it is absent before running this + block. The block resolves host-local symlinks, validates every exact path, + and removes only host-local material; do not use a wildcard: + + ```bash + valid_pem_path() { + [[ $1 =~ ^/[A-Za-z0-9._/-]+$ ]] && + [[ $(realpath -m -- "$1") == "$1" ]] + } + PEM_DEST=$(sudo grep -E '^CI_FLEET_GITHUB_APP_PRIVATE_KEY_FILE=' \ + /etc/ci-fleet/host.env | cut -d= -f2-) + ACTIVE_PEM="/etc/ci-fleet/secrets/OLD-GITHUB-APP-KEY.pem" + OLD_LOCAL_PEMS=("$ACTIVE_PEM") + OLD_MANAGED_PEMS=() + active_classifications=0 + for pem in "${OLD_LOCAL_PEMS[@]}" "${OLD_MANAGED_PEMS[@]}"; do + [[ $pem =~ ^/[A-Za-z0-9._/-]+$ ]] || exit 1 + if [[ "$pem" == "$ACTIVE_PEM" ]]; then + active_classifications=$((active_classifications + 1)) + fi + done + ((active_classifications == 1)) || exit 1 + PEM_DEST_BACKING=$(sudo readlink -f -- "$PEM_DEST") || exit 1 + valid_pem_path "$PEM_DEST_BACKING" || exit 1 + RESOLVED_OLD_LOCAL_PEMS=() + for pem in "${OLD_LOCAL_PEMS[@]}"; do + backing=$(sudo readlink -f -- "$pem") || exit 1 + valid_pem_path "$backing" || exit 1 + [[ "$backing" != "$PEM_DEST_BACKING" ]] || exit 1 + RESOLVED_OLD_LOCAL_PEMS+=("$backing") + [[ "$pem" == "$backing" ]] || RESOLVED_OLD_LOCAL_PEMS+=("$pem") + done + OLD_LOCAL_PEMS=("${RESOLVED_OLD_LOCAL_PEMS[@]}") + valid_pem_path "$PEM_DEST" || exit 1 + for pem in "${OLD_LOCAL_PEMS[@]}" "${OLD_MANAGED_PEMS[@]}"; do + [[ $pem =~ ^/[A-Za-z0-9._/-]+$ ]] || exit 1 + pem_backing=$(sudo realpath -m -- "$pem") || exit 1 + [[ "$pem_backing" != "$PEM_DEST_BACKING" ]] || exit 1 + done + for pem in "${OLD_MANAGED_PEMS[@]}"; do + sudo test ! -e "$pem" || exit 1 + done + for pem in "${OLD_LOCAL_PEMS[@]}"; do + sudo rm -f -- "$pem" || exit 1 + done + unset active_classifications backing pem_backing PEM_DEST_BACKING \ + RESOLVED_OLD_LOCAL_PEMS \ + OLD_LOCAL_PEMS OLD_MANAGED_PEMS + ``` + +2. On the GitHub App settings page, under **Private keys**, delete/revoke the + old key. Do not revoke it unless step 1 completed. +3. Remove any old workstation or temporary copies under the applicable secure + erasure policy. Keep the new key and its configured path unchanged. + +## Controller retirement and PEM removal + +Retirement is not complete when only the App installation is removed: + +1. Drain the controller through reviewed desired state and verify zero managed + runners and zero effective capacity. +2. Uninstall the App from the organization to invalidate installation access. +3. On the GitHub App settings page, delete/revoke **every** private key for this + controller's app; delete the dedicated app itself if it will not be reused. +4. Before uninstalling, read the configured destination while `host.env` still + exists and classify every retained rotation path explicitly. Revoke or + unmount each secret-manager-backed path through that manager first and + verify that it is absent; never pass a manager-owned path to `rm`. + `LOCAL_PEMS` must contain only host-local files. The uninstaller deliberately + preserves `/etc/ci-fleet/host.env` and `/etc/ci-fleet/secrets`: + + ```bash + safe_pem_path() { + [[ $1 =~ ^/[A-Za-z0-9._/-]+$ ]] + } + PEM_DEST=$(sudo grep -E '^CI_FLEET_GITHUB_APP_PRIVATE_KEY_FILE=' \ + /etc/ci-fleet/host.env | cut -d= -f2-) + safe_pem_path "$PEM_DEST" || exit 1 + [[ $(sudo realpath -m -- "$PEM_DEST") == "$PEM_DEST" ]] || exit 1 + # Put PEM_DEST and every old path in exactly one array; never use a wildcard. + LOCAL_PEMS=("/etc/ci-fleet/secrets/HOST-LOCAL-KEY.pem") + MANAGED_PEMS=("/run/secret-manager/MANAGER-BACKED-KEY") + # Resolve every host-local path before persisting the retry inventory. + RESOLVED_LOCAL_PEMS=() + for pem in "${LOCAL_PEMS[@]}"; do + backing=$(sudo readlink -f -- "$pem") || exit 1 + safe_pem_path "$backing" || exit 1 + RESOLVED_LOCAL_PEMS+=("$backing") + [[ "$pem" == "$backing" ]] || RESOLVED_LOCAL_PEMS+=("$pem") + done + LOCAL_PEMS=("${RESOLVED_LOCAL_PEMS[@]}") + PEM_INVENTORY=/etc/ci-fleet/retired-pem-paths + configured_classifications=0 + for pem in "${LOCAL_PEMS[@]}" "${MANAGED_PEMS[@]}"; do + safe_pem_path "$pem" || exit 1 + [[ $(sudo realpath -m -- "$pem") != \ + $(sudo realpath -m -- "$PEM_INVENTORY") ]] || exit 1 + if [[ "$pem" == "$PEM_DEST" ]]; then + configured_classifications=$((configured_classifications + 1)) + fi + done + ((configured_classifications == 1)) || exit 1 + for pem in "${MANAGED_PEMS[@]}"; do + sudo test ! -e "$pem" || exit 1 + done + if ((${#LOCAL_PEMS[@]})); then + printf '%s\n' "${LOCAL_PEMS[@]}" | \ + sudo install -T -m 0600 /dev/stdin "$PEM_INVENTORY" + else + sudo install -T -m 0600 /dev/null "$PEM_INVENTORY" + fi || exit 1 + + remove_retired_pems() { + for pem in "${LOCAL_PEMS[@]}"; do + sudo rm -f -- "$pem" || return 1 + done + } + if sudo /opt/ci-fleet/manager/current/scripts/install-worker-controller.sh \ + --uninstall && + remove_retired_pems && + sudo rm -f -- /etc/ci-fleet/host.env && + sudo rm -f -- "$PEM_INVENTORY"; then + unset PEM_DEST PEM_INVENTORY LOCAL_PEMS MANAGED_PEMS RESOLVED_LOCAL_PEMS + else + printf 'retirement cleanup failed; retained %s and host.env\n' \ + "$PEM_INVENTORY" >&2 + exit 1 + fi + ``` + + Replace or repeat the examples for every exact rotation path used by this + app. If classification, manager cleanup, extraction, or validation fails, + stop before uninstalling + or removing `host.env`. On later cleanup failure, rebuild `LOCAL_PEMS` + from the retained root-owned inventory before retrying exact-path removal; + the inventory is deleted only after every PEM and `host.env` are removed. +5. Remove remaining management-workstation, temporary, and backup copies + according to their retention and secure-erasure policies. If + the retired storage cannot guarantee file-level erasure (for example SSD, + snapshot, or copy-on-write media), destroy the encrypted volume or its + encryption key before disposal. diff --git a/scripts/test_quickstart.py b/scripts/test_quickstart.py index 55a82a24..de74c0c2 100644 --- a/scripts/test_quickstart.py +++ b/scripts/test_quickstart.py @@ -1,7 +1,9 @@ #!/usr/bin/env python3 +import re from pathlib import Path -raw_quickstart = (Path(__file__).resolve().parents[1] / "docs" / "QUICKSTART.md").read_text() +repo_root = Path(__file__).resolve().parents[1] +raw_quickstart = (repo_root / "docs" / "QUICKSTART.md").read_text() quickstart = " ".join(raw_quickstart.split()) required = ( @@ -16,4 +18,241 @@ assert quickstart.index("Cancel every queued job") < quickstart.index("3. Authorize the repository") assert "PROJECT_PREFIX=" not in raw_quickstart assert "managed controller managed controller" not in quickstart -print("quickstart_contract=PASS") + +app_setup = (repo_root / "docs" / "GITHUB-APP-SETUP.md").read_text() +bash_blocks = "\n".join(re.findall(r"```bash\n(.*?)\n\s*```", app_setup, re.S)) +bash_commands = re.sub(r"\\\n\s*", " ", bash_blocks).splitlines() +token_commands = [ + command for command in bash_commands if "scripts/github-app-token.sh" in command +] +assert len(token_commands) == 2, f"expected two documented token-helper calls, found {len(token_commands)}" +assert all(">/dev/null" in command for command in token_commands), ( + "token-helper stdout must be redirected" +) +assert app_setup.index("## Key rotation: activate and verify before revocation") < app_setup.index( + "## Old-key revocation" +) + +rotation = re.search( + r'ACTIVE_PEM="(/etc/ci-fleet/secrets/[^"\n]+)"\n' + r'PEM_DEST="(/etc/ci-fleet/secrets/[^"\n]+)"', + app_setup, +) +assert rotation and rotation[1] != rotation[2], "rotation destination must differ from active PEM" + +transfer = app_setup[ + app_setup.index("valid_pem_path() {") : app_setup.index( + "Use an equivalent privileged SSH workflow" + ) +] +active_guard = '[[ -n "$ACTIVE_PEM" && "$PEM_DEST" == "$ACTIVE_PEM" ]]' +assert "valid_pem_path \"$PEM_DEST\" || exit 1" in transfer +assert '[[ -n "$PEM_DIR" ]] || PEM_DIR=/' in transfer +assert 'PEM_MARKER="$PEM_DIR/.ci-fleet-transfer-$TRANSFER_ID"' in transfer +assert 'PEM_MARKER="$PEM_DEST' not in transfer +assert "install -d -m 0700" in transfer and "$PEM_DIR" in transfer +assert "secure_pem_ancestors()" in transfer +assert "stat -c '%U'" in transfer and "-perm /022" in transfer +assert "dir=\\${dir%/*}" in transfer +assert "^/[A-Za-z0-9._/-]+$" in transfer +assert 'realpath -m -- \\"$PEM_DEST\\"' in transfer +assert active_guard in transfer +assert transfer.index(active_guard) < transfer.index('ssh "$CONTROLLER"') +assert "mktemp --" in transfer and 'cat >\\"\\$tmp\\"' in transfer +hard_link = 'ln -T -- \\"\\$tmp\\" \\"$PEM_DEST\\"' +assert hard_link in transfer +assert transfer.index("mktemp --") < transfer.index('cat >\\"\\$tmp\\"') +assert transfer.index('cat >\\"\\$tmp\\"') < transfer.index(hard_link) +assert "set -C" not in transfer +marker_link = 'ln -T -- \\"\\$tmp\\" \\"$PEM_MARKER\\"' +assert transfer.index(marker_link) < transfer.index(hard_link) +trap_body = transfer[transfer.index("trap 'status=") : transfer.index("exit \\\"\\$status\\\"' 0")] +assert trap_body.index('\\"$PEM_MARKER\\" -ef \\"\\$tmp\\"') < trap_body.index( + 'rm -f -- \\"\\$tmp\\"' +) +for command in ("sha256sum --", "stat -c '%U:%G'", "stat -c '%a'"): + assert any(command in line and "$PEM_DEST" in line for line in transfer.splitlines()), ( + f"transfer does not use configured destination: {command}" + ) + +main_then = transfer.index("\nthen\n if ! ssh", transfer.index("stat -c '%a'")) +verification = transfer[transfer.index('if\nssh "$CONTROLLER"') : main_then] +for check in ( + '[[ "$local_sha" =~ ^[0-9a-f]{64}$ ]] &&\n', + '[[ "$remote_sha" =~ ^[0-9a-f]{64}$ ]] &&\n', + 'test "$local_sha" = "$remote_sha" &&\n', + "= 'root:root'\" &&\n", + "= '600'\" &&\n", +): + assert check in verification, f"download deletion is not gated by: {check}" + +delete_download = transfer[ + transfer.index('if rm -f -- "$PEM"; then') : transfer.index( + "else\n if ! ssh" + ) +] +delete_success, delete_failure = delete_download.split("else", 1) +assert "unset PEM" in delete_success +assert '"$PEM" >&2' in delete_failure and "exit 1" in delete_failure +assert 'rm -f -- "$PEM"' not in transfer[transfer.index("transfer verification failed") :] +verification_failure = transfer[transfer.index("else\n if ! ssh") :] +assert 'test \\"$PEM_MARKER\\" -ef \\"$PEM_DEST\\"' in verification_failure +assert 'rm -f -- \\"$PEM_DEST\\" \\"$PEM_MARKER\\"' in verification_failure +assert "remote ownership cleanup failed; retain and retry marker" in verification_failure +assert '"$PEM_MARKER" "$PEM_DEST" >&2' in verification_failure +cleanup_failure = verification_failure[ + verification_failure.index("remote ownership cleanup failed") : + verification_failure.index("transfer verification failed") +] +assert "unset local_sha remote_sha TRANSFER_ID\n" in cleanup_failure +assert "PEM_MARKER" not in cleanup_failure.split("unset", 1)[1] +assert 'elif test -e \\"$PEM_MARKER\\"; then rm -f -- \\"$PEM_MARKER\\"' in verification_failure +assert "per-transfer hard-link marker proves" in app_setup +assert "pre-existing destinations are preserved" in app_setup +verification_ack = transfer[transfer.index("stat -c '%a'") : main_then] +assert 'test \\"$PEM_MARKER\\" -ef \\"$PEM_DEST\\"' in verification_ack +assert 'rm -f -- \\"$PEM_MARKER\\"' not in verification_ack +marker_cleanup = transfer[main_then : transfer.index('if rm -f -- "$PEM"')] +assert 'if test -e \\"$PEM_MARKER\\"; then' in marker_cleanup +assert "retry idempotent marker cleanup before activation" in marker_cleanup +assert "secret-manager-backed destination" in app_setup +assert "authenticated import/version operation" in app_setup +assert "remove only the new inactive version through the manager" in app_setup +manager_import = app_setup.index("authenticated import/version operation") +preflight_guard = '[[ "$PEM_DEST" != "$ACTIVE_PEM" ]] || exit 1' +assert app_setup.index(preflight_guard) < manager_import +assert app_setup.index('replacement_pubkey_sha=$(openssl pkey') < manager_import +assert '[[ "$replacement_pubkey_sha" != "$active_pubkey_sha" ]] || exit 1' in app_setup +preflight = app_setup[app_setup.index("Before either transfer workflow") : manager_import] +assert preflight.index('replacement_pubkey_sha=$(openssl pkey') < preflight.index( + 'if [[ -n "$ACTIVE_PEM" ]]' +) +assert "bash -o pipefail -c 'openssl pkey" in preflight +assert "unset replacement_pubkey_sha" not in preflight +assert app_setup.count('test "$replacement_pubkey_sha" = "$remote_pubkey_sha"') == 2 +manager_workflow = app_setup[ + app_setup.index("secret-manager-backed destination, do not") : app_setup.index( + "For a host-local destination" + ) +] +assert manager_workflow.index('[[ $PEM_DEST =~ ^/[A-Za-z0-9._/-]+$ ]]') < manager_workflow.index( + 'ssh "$CONTROLLER"' +) +manager_condition = manager_workflow[ + manager_workflow.index("if\n") : manager_workflow.index("\nthen\n") +] +manager_ancestor_gate = manager_condition.index("secure_pem_ancestors\n") +regular_file_gate = ( + "test \\\"\\$(stat -c '%F' -- \\\"$PEM_DEST\\\")\\\" = 'regular file' &&" +) +assert regular_file_gate in manager_condition +assert "stat -c '%U'" in manager_condition and "-perm /022" in manager_condition +assert ( + "dir=\\${dir%/*}" in manager_condition + and 'test \\\"\\$dir\\\" != / || break' in manager_condition +) +assert manager_ancestor_gate < manager_condition.index('local_sha=$(sha256sum -- "$PEM")') +assert manager_workflow.index("secure_pem_ancestors\n") < manager_workflow.index('rm -f -- "$PEM"') +manager_failure = manager_workflow[manager_workflow.index("else\n") :] +assert "manager import verification failed" in manager_failure and "exit 1" in manager_failure +assert "unsupported until secured" in manager_workflow +manager_gate = app_setup.index("secure_pem_ancestors\n", manager_import) +rotation_start = app_setup.index("## Key rotation: activate and verify before revocation") +rotation_token = app_setup.index("scripts/github-app-token.sh", rotation_start) +assert manager_gate < rotation_start < rotation_token +assert 'ACTIVE_PEM=$(ssh "$CONTROLLER" "readlink -f -- \\"$ACTIVE_PEM\\"")' in app_setup +assert "Update controller `host.env` to that canonical result" in app_setup +assert "rotating, revoking, or directly retiring" in app_setup + +for use in ( + "exact value of `PEM_DEST`", + "exact value of `ACTIVE_PEM`", +): + assert use in app_setup, f"configured PEM destination contract missing: {use}" + +rotation = app_setup[ + app_setup.index("## Key rotation: activate and verify before revocation") : + app_setup.index("## Old-key revocation") +] +assert "`healthy` for an `active` controller" in rotation +assert "`maintenance`" in rotation and "`drained` or `disabled`" in rotation +assert "If token generation fails, immediately restore" in rotation +assert "run normal\n reconciliation" in rotation +assert "`RECONCILE CONVERGED`" in rotation +health_gate = "sudo /opt/ci-fleet/current/scripts/healthcheck.sh || exit 1" +assert rotation.index(health_gate) < rotation.index("--check-only --installed-ref") +checkpoint_match = '"CI_FLEET_GITHUB_APP_PRIVATE_KEY_FILE=$PEM_DEST"' +assert checkpoint_match in rotation +checkpoint_query = "LATEST_CHECKPOINT=$(sudo find" +checkpoint_destination = "PEM_DEST=$(sudo grep -E '^CI_FLEET_GITHUB_APP_PRIVATE_KEY_FILE='" +assert rotation.index(checkpoint_destination) < rotation.index(checkpoint_query) +assert rotation.index('valid_pem_path "$PEM_DEST" || exit 1') < rotation.index( + checkpoint_query +) +assert "! -path '/var/lib/ci-fleet/checkpoints/.checkpoint.staging.*/*'" in rotation +assert "retain the old GitHub key and old PEM" in rotation + +revocation = app_setup[ + app_setup.index("## Old-key revocation") : app_setup.index( + "## Controller retirement and PEM removal" + ) +] +read_destination = "PEM_DEST=$(sudo grep -E '^CI_FLEET_GITHUB_APP_PRIVATE_KEY_FILE='" +validate_paths = 'for pem in "${OLD_LOCAL_PEMS[@]}" "${OLD_MANAGED_PEMS[@]}"; do' +distinct_paths = '[[ "$pem_backing" != "$PEM_DEST_BACKING" ]] || exit 1' +remove_old_pem = 'sudo rm -f -- "$pem" || exit 1' +assert 'ACTIVE_PEM="/etc/ci-fleet/secrets/OLD-GITHUB-APP-KEY.pem"' in revocation +assert 'PEM_DEST_BACKING=$(sudo readlink -f -- "$PEM_DEST") || exit 1' in revocation +assert 'backing=$(sudo readlink -f -- "$pem") || exit 1' in revocation +assert '[[ "$backing" != "$PEM_DEST_BACKING" ]] || exit 1' in revocation +assert 'RESOLVED_OLD_LOCAL_PEMS+=("$backing")' in revocation +assert 'RESOLVED_OLD_LOCAL_PEMS+=("$pem")' in revocation +assert 'OLD_LOCAL_PEMS=("${RESOLVED_OLD_LOCAL_PEMS[@]}")' in revocation +assert 'OLD_MANAGED_PEMS=()' in revocation +assert "((active_classifications == 1)) || exit 1" in revocation +assert revocation.index("((active_classifications == 1)) || exit 1") < revocation.index( + 'OLD_LOCAL_PEMS=("${RESOLVED_OLD_LOCAL_PEMS[@]}")' +) +old_manager_absent = 'for pem in "${OLD_MANAGED_PEMS[@]}"; do' +assert "^/[A-Za-z0-9._/-]+$" in revocation +assert revocation.index(read_destination) < revocation.index(validate_paths) +assert revocation.index(validate_paths) < revocation.index(distinct_paths) +assert revocation.index(distinct_paths) < revocation.index(old_manager_absent) +assert revocation.index(old_manager_absent) < revocation.index(remove_old_pem) +assert revocation.index(remove_old_pem) < revocation.index( + "On the GitHub App settings page" +) + +retirement = app_setup[app_setup.index("## Controller retirement and PEM removal") :] +validate_paths = 'safe_pem_path "$pem" || exit 1' +uninstall = "scripts/install-worker-controller.sh \\\n --uninstall &&" +remove_pem = 'sudo rm -f -- "$pem" || return 1' +persist_inventory = 'sudo install -T -m 0600 /dev/stdin "$PEM_INVENTORY"' +remove_host_env = "sudo rm -f -- /etc/ci-fleet/host.env &&" +remove_inventory = 'sudo rm -f -- "$PEM_INVENTORY"; then' +classify_destination = 'configured_classifications=$((configured_classifications + 1))' +manager_absent = 'sudo test ! -e "$pem" || exit 1' +inventory_distinct = '$(sudo realpath -m -- "$PEM_INVENTORY") ]] || exit 1' +assert 'LOCAL_PEMS=(' in retirement and 'MANAGED_PEMS=(' in retirement +assert 'backing=$(sudo readlink -f -- "$pem") || exit 1' in retirement +assert 'RESOLVED_LOCAL_PEMS+=("$backing")' in retirement +assert 'RESOLVED_LOCAL_PEMS+=("$pem")' in retirement +assert 'LOCAL_PEMS=("${RESOLVED_LOCAL_PEMS[@]}")' in retirement +assert "^/[A-Za-z0-9._/-]+$" in retirement +assert '[[ $(sudo realpath -m -- "$PEM_DEST") == "$PEM_DEST" ]] || exit 1' in retirement +assert '$(sudo realpath -m -- "$PEM_INVENTORY")' in retirement +assert retirement.index(read_destination) < retirement.index(validate_paths) +assert retirement.index(validate_paths) < retirement.index(inventory_distinct) +assert retirement.index(inventory_distinct) < retirement.index(classify_destination) +assert "((configured_classifications == 1)) || exit 1" in retirement +assert retirement.index(classify_destination) < retirement.index(manager_absent) +assert retirement.index(manager_absent) < retirement.index(persist_inventory) +assert retirement.index(persist_inventory) < retirement.index(uninstall) +assert remove_pem in retirement +assert 'for pem in "${LOCAL_PEMS[@]}"; do' in retirement +assert retirement.index(uninstall) < retirement.index("remove_retired_pems &&") +assert retirement.index("remove_retired_pems &&") < retirement.index(remove_host_env) +assert retirement.index(remove_host_env) < retirement.index(remove_inventory) +retirement_failure = retirement[retirement.index("retirement cleanup failed") :] +assert '"$PEM_INVENTORY" >&2' in retirement_failure and "exit 1" in retirement_failure +print("documentation_contract=PASS")