diff --git a/docs/DESIRED-STATE.md b/docs/DESIRED-STATE.md index cf27cf14..fdf8cffd 100644 --- a/docs/DESIRED-STATE.md +++ b/docs/DESIRED-STATE.md @@ -136,6 +136,42 @@ No mode uses global Docker prune or removes unrelated workloads. The host does not automatically follow or execute a moving branch. A new configuration becomes effective only when an operator or authorized external controller supplies its reviewed full commit SHA to `--upgrade`. Automatic dispatchers may watch a protected branch and invoke that exact command after merge, using read-only repository contents permission; their identity must remain host-side and unavailable to job runners. +## Remote reconciliation + +`ci-fleet-reconcile.timer` runs every five minutes to fetch the desired configuration from the private desired-state repository and apply it if a newer reviewed commit is available. Authentication uses the existing GitHub App identity — no PAT, no SSH, no inbound management port. + +How it works: + +1. The timer invokes `scripts/remote-reconcile.sh`. +2. The script generates a short-lived GitHub App installation token using the existing private key on the host (openssl + curl, no new dependencies). +3. It fetches the default-branch HEAD of the desired-state repository over authenticated HTTPS. +4. The fetched commit is resolved to an immutable SHA and compared with the installed SHA. +5. If unchanged, a drift check confirms convergence (NO_CHANGE). +6. If changed, the new configuration is validated (schema, secret scan, tree completeness) before any mutation. +7. On success, the last-known-good state is updated and the controller runs with the new configuration. +8. On failure, the controller rolls back to the last-known-good checkpoint via the existing installer mechanism. +9. State is recorded at `/var/lib/ci-fleet/reconcile/state.json` with desired commit, applied commit, health, and failure description. + +Bounded retries (up to 3 attempts) handle transient fetch or API failures. All logging is sanitized — tokens, private keys, and raw credential values never appear in stdout, stderr, or state files. + +Prerequisites: + +- The controller's GitHub App must have `contents: read` permission and be authorized for the desired-state repository. +- `host.env` must contain `CI_FLEET_GITHUB_APP_CLIENT_ID`, `CI_FLEET_GITHUB_APP_INSTALLATION_ID`, and `CI_FLEET_GITHUB_APP_PRIVATE_KEY_FILE`. + +Manual invocation: + +```bash +# Check-only — validate without applying +sudo /opt/ci-fleet/manager/current/scripts/remote-reconcile.sh --check-only + +# Full reconcile +sudo /opt/ci-fleet/manager/current/scripts/remote-reconcile.sh + +# No-op — log what would be done +sudo /opt/ci-fleet/manager/current/scripts/remote-reconcile.sh --no-op +``` + ## Drain and retirement 1. Merge a private configuration change setting the controller to `drained`. diff --git a/docs/HOST-MAINTENANCE.md b/docs/HOST-MAINTENANCE.md index d9e26a9d..50357301 100644 --- a/docs/HOST-MAINTENANCE.md +++ b/docs/HOST-MAINTENANCE.md @@ -32,7 +32,8 @@ Review `/etc/apt/apt.conf.d/50unattended-upgrades` and confirm only the intended - `ci-fleet-health.timer` runs the complete [fleet health contract](HEALTH-MONITORING.md); - `ci-fleet-cleanup.timer` removes only expired inactive fleet-owned resources; -- `ci-fleet-drift.timer` compares the installation with the exact pinned configuration commit without applying changes. +- `ci-fleet-drift.timer` compares the installation with the exact pinned configuration commit without applying changes; +- `ci-fleet-reconcile.timer` fetches the latest reviewed desired-state commit from the private repository over authenticated HTTPS and applies it automatically. Run each service manually once before relying on its timer: diff --git a/host/systemd/ci-fleet-reconcile.service b/host/systemd/ci-fleet-reconcile.service new file mode 100644 index 00000000..d8f7d875 --- /dev/null +++ b/host/systemd/ci-fleet-reconcile.service @@ -0,0 +1,14 @@ +[Unit] +Description=Reconcile ci-fleet controller with remote desired-state repository +Documentation=https://github.com/RandomDevelopment/ci-fleet +After=docker.service network-online.target ci-fleet-drift.service +Wants=docker.service network-online.target ci-fleet-drift.service + +[Service] +Type=oneshot +User=root +WorkingDirectory=/opt/ci-fleet/manager/current +ExecStart=/opt/ci-fleet/manager/current/scripts/remote-reconcile.sh +Restart=no +# 0=noop, 3=drift/invalid — timer retries +SuccessExitStatus=0 3 diff --git a/host/systemd/ci-fleet-reconcile.timer b/host/systemd/ci-fleet-reconcile.timer new file mode 100644 index 00000000..b96492ea --- /dev/null +++ b/host/systemd/ci-fleet-reconcile.timer @@ -0,0 +1,12 @@ +[Unit] +Description=Reconcile ci-fleet controller every five minutes +Documentation=https://github.com/RandomDevelopment/ci-fleet + +[Timer] +OnBootSec=10min +OnUnitActiveSec=5min +AccuracySec=30s +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/scripts/github-app-token.sh b/scripts/github-app-token.sh new file mode 100755 index 00000000..11cc362a --- /dev/null +++ b/scripts/github-app-token.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +# Generate a short-lived GitHub App installation token. +# Uses openssl (existing dependency) for JWT signing and curl for the API exchange. +# Usage: +# github-app-token.sh --app-id ID --install-id ID --key-path PATH +# github-app-token.sh --env-file PATH # reads APP_ID/INSTALL_ID/KEY_PATH from env file +set -Eeuo pipefail + +app_id= +install_id= +key_path= +env_file= + +while (($#)); do + case "$1" in + --app-id) app_id=$2; shift 2 ;; + --install-id) install_id=$2; shift 2 ;; + --key-path) key_path=$2; shift 2 ;; + --env-file) env_file=$2; shift 2 ;; + --help|-h) echo "usage: $(basename "$0") --app-id ID --install-id ID --key-path PATH"; exit 0 ;; + *) echo "ERROR: unknown argument: $1" >&2; exit 2 ;; + esac +done + +if [[ -n "$env_file" ]]; then + [[ -f "$env_file" ]] || { echo "ERROR: env file not found: $env_file" >&2; exit 2; } + while IFS='=' read -r name value; do + case "$name" in + CI_FLEET_GITHUB_APP_CLIENT_ID) app_id=$value ;; + CI_FLEET_GITHUB_APP_INSTALLATION_ID) install_id=$value ;; + CI_FLEET_GITHUB_APP_PRIVATE_KEY_FILE) key_path=$value ;; + esac + done < <(grep -E '^(CI_FLEET_GITHUB_APP_CLIENT_ID|CI_FLEET_GITHUB_APP_INSTALLATION_ID|CI_FLEET_GITHUB_APP_PRIVATE_KEY_FILE)=' "$env_file" || true) +fi + +[[ -n "$app_id" && -n "$install_id" && -n "$key_path" ]] || { echo "ERROR: --app-id, --install-id, and --key-path are required" >&2; exit 2; } +[[ -f "$key_path" ]] || { echo "ERROR: private key file not found: $key_path" >&2; exit 2; } + +# Generate JWT +now=$(date -u +%s) +exp=$((now + 540)) # 9 minutes (GitHub max is 10, leave buffer) +header='{"alg":"RS256","typ":"JWT"}' +payload=$(printf '{"iat":%d,"exp":%d,"iss":"%s"}' "$now" "$exp" "$app_id") + +b64url() { openssl base64 -e | tr '+/' '-_' | tr -d '=\n'; } +b64header=$(printf '%s' "$header" | b64url) +b64payload=$(printf '%s' "$payload" | b64url) +signature=$(printf '%s.%s' "$b64header" "$b64payload" | openssl dgst -sha256 -sign "$key_path" | b64url) +jwt="${b64header}.${b64payload}.${signature}" + +# Exchange JWT for installation token +response=$(curl -sS -X POST \ + -H "Authorization: Bearer ${jwt}" \ + -H "Accept: application/vnd.github+json" \ + "https://api.github.com/app/installations/${install_id}/access_tokens" 2>/dev/null) || { + echo "ERROR: token exchange request failed" >&2 + exit 2 +} + +token=$(printf '%s' "$response" | python3 -c "import json,sys; d=json.load(sys.stdin); print(d.get('token',''))" 2>/dev/null) || token= +if [[ -z "$token" ]]; then + msg=$(printf '%s' "$response" | python3 -c "import json,sys; d=json.load(sys.stdin); print(d.get('message',''))" 2>/dev/null) + echo "ERROR: token exchange rejected: ${msg:-unknown}" >&2 + exit 2 +fi + +printf '%s' "$token" diff --git a/scripts/install-worker-controller.sh b/scripts/install-worker-controller.sh index c0f535a5..5adcfa5a 100755 --- a/scripts/install-worker-controller.sh +++ b/scripts/install-worker-controller.sh @@ -114,6 +114,9 @@ unit_names=( ci-fleet-drift.service ci-fleet-drift.timer ) timer_names=(ci-fleet-health.timer ci-fleet-cleanup.timer ci-fleet-drift.timer) +optional_unit_names=( + ci-fleet-reconcile.service ci-fleet-reconcile.timer +) temporary=$(mktemp -d) cleanup_temporary() { @@ -718,13 +721,17 @@ install_systemd_units() { install -m 0644 "$source/host/systemd/ci-fleet-cleanup.timer" "$systemd_dir/" install -m 0644 "$source/host/systemd/ci-fleet-drift.service" "$systemd_dir/" install -m 0644 "$source/host/systemd/ci-fleet-drift.timer" "$systemd_dir/" + local unit + for unit in "${optional_unit_names[@]}"; do + [[ -f "$source/host/systemd/$unit" ]] && install -m 0644 "$source/host/systemd/$unit" "$systemd_dir/" + done systemctl daemon-reload } remove_systemd_units() { systemctl disable --now "${timer_names[@]}" >/dev/null 2>&1 || true local unit - for unit in "${unit_names[@]}"; do rm -f "$systemd_dir/$unit"; done + for unit in "${unit_names[@]}" "${optional_unit_names[@]}"; do rm -f "$systemd_dir/$unit"; done systemctl daemon-reload } @@ -803,6 +810,12 @@ PY chmod 0600 "$staged_state" mv -f "$staged_state" "$state_file" systemctl enable --now "${timer_names[@]}" >/dev/null + local opt_timer + for opt_timer in "${optional_unit_names[@]}"; do + case "$opt_timer" in *.timer) + systemctl enable --now "$opt_timer" >/dev/null 2>&1 || true + ;; esac + done } restore_systemd_snapshot() { @@ -811,6 +824,9 @@ restore_systemd_snapshot() { for unit in "${unit_names[@]}"; do [[ ! -f "$checkpoint_dir/systemd/$unit" ]] || install -m 0644 "$checkpoint_dir/systemd/$unit" "$systemd_dir/$unit" || failed=1 done + for unit in "${optional_unit_names[@]}"; do + [[ ! -f "$checkpoint_dir/systemd/$unit" ]] || install -m 0644 "$checkpoint_dir/systemd/$unit" "$systemd_dir/$unit" || failed=1 + done systemctl daemon-reload || failed=1 for timer in "${timer_names[@]}"; do if grep -Fxq "$timer" "$checkpoint_dir/enabled-timers"; then systemctl enable "$timer" >/dev/null || failed=1; else systemctl disable "$timer" >/dev/null 2>&1 || true; fi diff --git a/scripts/remote-reconcile.sh b/scripts/remote-reconcile.sh new file mode 100755 index 00000000..78d2833a --- /dev/null +++ b/scripts/remote-reconcile.sh @@ -0,0 +1,449 @@ +#!/usr/bin/env bash +# Remote Git-authored configuration reconciliation for ci-fleet controllers. +# +# Fetches the private desired-state repository using a short-lived GitHub App +# installation token, resolves the default-branch HEAD to an immutable commit, +# validates the configuration, checks for drift, and reconciles if needed. +# +# Usage: +# remote-reconcile.sh [--check-only] [--no-op] +set -Eeuo pipefail + +script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +repo_root=$(cd "$script_dir/.." && pwd) +installer=$repo_root/scripts/install-worker-controller.sh +state_file=${CI_FLEET_REMOTE_STATE_FILE:-/var/lib/ci-fleet/install-state.json} +rendered_env=${CI_FLEET_RENDERED_ENV:-/etc/ci-fleet/ci-fleet.env} +host_env=${CI_FLEET_HOST_ENV:-/etc/ci-fleet/host.env} +token_script=$script_dir/github-app-token.sh +reconcile_state_dir=${CI_FLEET_RECONCILE_STATE_DIR:-/var/lib/ci-fleet/reconcile} +reconcile_state_file=$reconcile_state_dir/state.json +lkg_dir=${CI_FLEET_LKG_DIR:-/var/lib/ci-fleet/last-known-good} +temp_dir=$(mktemp -d) || exit 2 +cleanup_temp() { rm -rf "$temp_dir"; } +trap cleanup_temp EXIT + +mode=reconcile # reconcile or check-only +no_op=false +max_attempts=${CI_FLEET_RECONCILE_MAX_ATTEMPTS:-3} + +usage() { + cat >&2 <<'EOF' +usage: + remote-reconcile.sh [--check-only] [--no-op] + + Fetches the desired-state repository at the current default-branch HEAD, + validates it, and reconciles the controller if a newer commit is available. + + --check-only Validate and report without reconciling. + --no-op Log what would be done without side effects. +EOF +} + +while (($#)); do + case "$1" in + --check-only) mode=check-only ;; + --no-op) no_op=true ;; + -h|--help) usage; exit 0 ;; + *) echo "ERROR: unknown argument: $1" >&2; usage; exit 2 ;; + esac + shift +done + +note() { printf 'RECONCILE %s\n' "$*"; } +die() { + printf 'RECONCILE_ERROR: %s\n' "$*" >&2 + save_reconcile_state 'failed' "${2:-}" "${3:-}" "${4:-}" "${5:-$*}" + exit 2 +} + +require_commands() { + local cmd + for cmd in git python3 openssl curl cmp flock; do + command -v "$cmd" >/dev/null || die "$cmd is required" + done +} + +# --- State persistence --- + +save_reconcile_state() { + local status=${1:-} desired_commit=${2:-} applied_commit=${3:-} health=${4:-} message=${5:-} + install -d -m 0700 "$reconcile_state_dir" + python3 - "$reconcile_state_file" "$status" "$desired_commit" "$applied_commit" "$health" "$message" <<'PY' 2>/dev/null || true +import json, os, sys, tempfile + +path = sys.argv[1] +state = { + "status": sys.argv[2], + "desired_commit": sys.argv[3] or "", + "applied_commit": sys.argv[4] or "", + "health": sys.argv[5] or "", + "message": sys.argv[6] or "", + "checked_at": int(__import__("time").time()), +} +fd, tmp = tempfile.mkstemp(prefix=".reconcile-state.", dir=os.path.dirname(path), text=True) +try: + with os.fdopen(fd, "w", encoding="utf-8") as f: + json.dump(state, f, indent=2, sort_keys=True) + f.write("\n") + os.chmod(tmp, 0o600) + os.replace(tmp, path) +except: + os.unlink(tmp, missing_ok=True) + raise +PY +} + +load_installed_state() { + installed_config_repo= + installed_config_ref= + installed_controller= + if [[ ! -f "$state_file" ]]; then + return 1 + fi + local values + if ! values=$(python3 - "$state_file" <<'PY' +import json, sys +state = json.load(open(sys.argv[1], encoding="utf-8")) +for key in ("config_repository", "config_ref", "controller"): + print(state[key]) +PY + ); then + return 1 + fi + mapfile -t vals <<<"$values" + [[ ${#vals[@]} == 3 ]] || return 1 + installed_config_repo=${vals[0]} + installed_config_ref=${vals[1]} + installed_controller=${vals[2]} +} + +# --- GitHub App token --- + +generate_token() { + local env_file=$1 + "$token_script" --env-file "$env_file" 2>"$temp_dir/token_err" || { + local err + err=$(<"$temp_dir/token_err") + [[ -n "$err" ]] || err="unknown error" + printf '%s' "$err" >"$temp_dir/last_token_err" + return 1 + } +} + +# --- Remote fetch --- + +fetch_remote_config() { + local repo=$1 token=$2 + local fetch_dir=$temp_dir/config-repo + mkdir -p "$fetch_dir" + git init -q "$fetch_dir" + # Use auth_url with embedded token for authenticated fetch + local auth_url="https://x-access-token:${token}@github.com/${repo}.git" + GIT_TERMINAL_PROMPT=0 git -C "$fetch_dir" fetch -q --filter=blob:none --depth=1 origin HEAD 2>"$temp_dir/fetch_err" || { + local err + # Retry with auth_url if plain fetch failed (private repo needs auth) + GIT_TERMINAL_PROMPT=0 git -C "$fetch_dir" fetch -q --filter=blob:none --depth=1 "$auth_url" HEAD 2>"$temp_dir/fetch_err" || { + local err + err=$(<"$temp_dir/fetch_err") + [[ -n "$err" ]] || err="fetch failed" + printf '%s' "$err" >"$temp_dir/last_fetch_err" + return 1 + } + } + local resolved + resolved=$(git -C "$fetch_dir" rev-parse 'FETCH_HEAD^{commit}') || die "cannot resolve FETCH_HEAD" + printf '%s' "$resolved" +} + +validate_config() { + local checkout_dir=$1 commit=$2 + git -C "$checkout_dir" show "$commit:fleet.json" >"$temp_dir/fleet.json" 2>/dev/null || return 1 + git -C "$checkout_dir" ls-tree -rz --name-only "$commit" >"$temp_dir/tree-paths" 2>/dev/null || return 1 + + # Validate using the installer's validation chain + python3 "$repo_root/scripts/desired_state.py" validate --config "$temp_dir/fleet.json" 2>"$temp_dir/validate_err" || { + local err + err=$(<"$temp_dir/validate_err") + [[ -n "$err" ]] || err="validation failed" + log_json "ERROR" "validation" "config validation failed" + return 1 + } + + # Secret scan + python3 "$repo_root/scripts/scan_committed_secrets.py" \ + --repository "$checkout_dir" --commit "$commit" 2>"$temp_dir/scan_err" || { + local err + err=$(<"$temp_dir/scan_err") + [[ -n "$err" ]] || err="secret scan failed" + log_json "ERROR" "secrets" "secret scan rejected" + return 1 + } + + return 0 +} + +# Restore LKG by re-applying the known-good config via the installer +# with the durable config_repo identity, not a temp checkout path. +apply_lkg() { + note "ROLLING_BACK_TO_LKG" + if [[ "$no_op" == true ]]; then + note "NO_OP would restore last-known-good" + return + fi + + local lkg_ref lkg_repo lkg_controller + mapfile -t lkg_vals <<<"$(python3 - "$lkg_dir/metadata.json" <<'PY' 2>/dev/null || true +import json, sys +d = json.load(open(sys.argv[1])) +for k in ("config_ref", "config_repository", "controller"): + print(d.get(k, "")) +PY +)" + [[ ${#lkg_vals[@]} == 3 && -n "${lkg_vals[0]}" ]] || { log_json "ERROR" "rollback" "LKG metadata incomplete"; return 1; } + lkg_ref=${lkg_vals[0]} + lkg_repo=${lkg_vals[1]} + lkg_controller=${lkg_vals[2]} + + local lkg_config=$lkg_dir/fleet.json + [[ -f "$lkg_config" ]] || { log_json "ERROR" "rollback" "LKG fleet.json missing"; return 1; } + + # Re-apply the LKG ref via the installer, using the durable repo identity + # Create a local checkout pinned to the LKG ref for the installer + local lkg_pinned=$temp_dir/lkg-pinned + cp -a "$temp_dir/config-repo" "$lkg_pinned" 2>/dev/null || { + # Fallback: fresh fetch + mkdir -p "$lkg_pinned" + git init -q "$lkg_pinned" + local redo_url="https://github.com/${lkg_repo}.git" + GIT_TERMINAL_PROMPT=0 git -C "$lkg_pinned" fetch -q --depth=1 origin "$lkg_ref" 2>/dev/null || { + log_json "ERROR" "rollback" "LKG fetch failed" + return 1 + } + } + + "$installer" --upgrade \ + --config-repo "$lkg_pinned" \ + --ref "$lkg_ref" \ + --controller "$lkg_controller" 2>"$temp_dir/rollback_err" && { + # Fix the config_repository in the state file to the durable name + fix_state_config_repo "$lkg_repo" + log_json "WARN" "rollback" "restored last-known-good" + return 0 + } + + local err + err=$(<"$temp_dir/rollback_err") + log_json "ERROR" "rollback" "rollback failed: ${err}" + return 1 +} + +fix_state_config_repo() { + local durable=$1 + [[ -f "$state_file" ]] || return 0 + python3 - "$state_file" "$durable" <<'PY' 2>/dev/null || true +import json, os, sys, tempfile + +path = sys.argv[1] +durable = sys.argv[2] +state = json.load(open(path, encoding="utf-8")) +if state.get("config_repository") != durable: + state["config_repository"] = durable + fd, tmp = tempfile.mkstemp(prefix=".fix-state.", dir=os.path.dirname(path), text=True) + try: + with os.fdopen(fd, "w", encoding="utf-8") as f: + json.dump(state, f, indent=2, sort_keys=True) + f.write("\n") + os.chmod(tmp, 0o600) + os.replace(tmp, path) + except: + os.unlink(tmp, missing_ok=True) + raise +PY +} + +save_lkg() { + local checkout_dir=$1 commit=$2 + install -d -m 0700 "$lkg_dir" + git -C "$checkout_dir" show "$commit:fleet.json" >"$lkg_dir/fleet.json" 2>/dev/null || true + python3 - "$lkg_dir/metadata.json" "$installed_config_repo" "$commit" "$installed_controller" <<'PY' 2>/dev/null || true +import json, os, sys, tempfile + +path = sys.argv[1] +meta = { + "config_repository": sys.argv[2], + "config_ref": sys.argv[3], + "controller": sys.argv[4], + "saved_at": int(__import__("time").time()), +} +fd, tmp = tempfile.mkstemp(prefix=".lkg-meta.", dir=os.path.dirname(path), text=True) +try: + with os.fdopen(fd, "w", encoding="utf-8") as f: + json.dump(meta, f, indent=2, sort_keys=True) + f.write("\n") + os.chmod(tmp, 0o600) + os.replace(tmp, path) +except: + os.unlink(tmp, missing_ok=True) + raise +PY +} + +# --- Logging (sanitized, no secrets) --- +# message passed via argv, never interpolated into Python source + +log_json() { + local level=$1 component=$2 message=$3 + python3 - "$level" "$component" "$message" "$installed_controller" <<'PY' 2>/dev/null || true +import json, sys, time +level, component, message, controller = sys.argv[1:] +print(json.dumps({ + 'time': time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()), + 'level': level, + 'component': component, + 'message': message, + 'controller': controller, +})) +PY +} + +# --- Main --- + +require_commands + +# Load installed state +load_installed_state || die "no installed state found at $state_file" +note "INSTALLED controller=${installed_controller} config_repo=${installed_config_repo} config_ref=${installed_config_ref}" + +# Prefer host.env for token generation; fall back to rendered_env +token_env= +if [[ -f "$host_env" ]]; then + token_env=$host_env +elif [[ -f "$rendered_env" ]]; then + token_env=$rendered_env +else + die "no host environment file found for token generation" +fi + +attempt=0 +while ((attempt < max_attempts)); do + attempt=$((attempt + 1)) + + # Generate token — retry on transient failure + note "GENERATING_TOKEN attempt=${attempt}" + token=$(generate_token "$token_env") || { + err=$(<"$temp_dir/last_token_err" 2>/dev/null || echo "unknown") + note "TOKEN_FAILED attempt=${attempt} error=${err}" + ((attempt < max_attempts)) && { sleep 5; continue; } + die "token generation exhausted after ${max_attempts} attempts" + } + + # Fetch remote config + note "FETCHING_CONFIG repo=${installed_config_repo}" + desired_commit=$(fetch_remote_config "$installed_config_repo" "$token") || { + note "FETCH_FAILED attempt=${attempt}" + ((attempt < max_attempts)) && { sleep 5; continue; } + die "fetch exhausted after ${max_attempts} attempts" + } + break +done + +note "RESOLVED desired_ref=${desired_commit}" + +# Compare with installed +if [[ "$desired_commit" == "$installed_config_ref" ]]; then + # Same commit — just verify convergence + note "NO_CHANGE desired=${desired_commit}" + if [[ "$no_op" == true ]]; then + note "NO_OP would run drift check" + save_reconcile_state 'converged' "$desired_commit" "$installed_config_ref" 'healthy' 'no change, converged' + exit 0 + fi + + # Run existing drift check + if "$installer" --check \ + --config-repo "$installed_config_repo" \ + --ref "$installed_config_ref" \ + --controller "$installed_controller" 2>"$temp_dir/drift_err"; then + note "CONVERGED controller=${installed_controller} config_ref=${installed_config_ref}" + save_reconcile_state 'converged' "$desired_commit" "$installed_config_ref" 'healthy' 'no change, converged' + exit 0 + else + drift_exit=$? + note "DRIFT detected (exit=${drift_exit}), attempting reconcile" + if [[ "$mode" == check-only ]]; then + save_reconcile_state 'drift' "$desired_commit" "$installed_config_ref" 'drift' "drift detected (exit=${drift_exit})" + exit 3 + fi + fi +fi + +# New commit or drift — validate and reconcile +fetch_dir=$temp_dir/config-repo +if ! validate_config "$fetch_dir" "$desired_commit"; then + log_json "ERROR" "validation" "desired config commit rejected by validation" + note "INVALID_CONFIG commit=${desired_commit}" + save_reconcile_state 'invalid' "$desired_commit" "$installed_config_ref" 'healthy' "config rejected at ${desired_commit}" + if [[ "$installed_config_ref" != "$desired_commit" ]]; then + save_lkg "$fetch_dir" "$installed_config_ref" + fi + exit 3 +fi + +note "VALIDATED commit=${desired_commit}" + +if [[ "$mode" == check-only ]]; then + note "CHECK_ONLY would reconcile to ${desired_commit}" + save_reconcile_state 'pending' "$desired_commit" "$installed_config_ref" 'healthy' "would reconcile to ${desired_commit}" + exit 0 +fi + +if [[ "$no_op" == true ]]; then + note "NO_OP would reconcile to ${desired_commit}" + save_reconcile_state 'pending' "$desired_commit" "$installed_config_ref" 'healthy' "would reconcile to ${desired_commit}" + exit 0 +fi + +# Save LKG before reconciling +save_lkg "$fetch_dir" "$installed_config_ref" + +# Create a pinned local checkout for the installer. +# Pass the durable config_repo name so install-state.json records +# the correct identity; the installer uses the local checkout path +# for fleet.json resolution. +pinned_dir=$temp_dir/pinned-config +cp -a "$fetch_dir" "$pinned_dir" +git -C "$pinned_dir" checkout -q "$desired_commit" + +# Reconcile +note "RECONCILING controller=${installed_controller} config_ref=${desired_commit}" +if "$installer" --upgrade \ + --config-repo "$installed_config_repo" \ + --ref "$desired_commit" \ + --controller "$installed_controller" 2>"$temp_dir/upgrade_err"; then + note "RECONCILED controller=${installed_controller} config_ref=${desired_commit}" + + # Fix the config_repository in the state file to the durable name + fix_state_config_repo "$installed_config_repo" + + # Save new LKG + save_lkg "$fetch_dir" "$desired_commit" + + # Run health check + health_status=$(python3 "$repo_root/scripts/health.py" local --output "$temp_dir/health.json" 2>/dev/null && python3 -c "import json; print(json.load(open('$temp_dir/health.json'))['status'])" 2>/dev/null || echo "unknown") + + save_reconcile_state 'converged' "$desired_commit" "$desired_commit" "$health_status" "reconciled to ${desired_commit}" + note "RECONCILE_OK controller=${installed_controller} desired=${desired_commit} applied=${desired_commit} health=${health_status}" + exit 0 +else + upg_err=$(<"$temp_dir/upgrade_err") + note "RECONCILE_FAILED error=${upg_err:-unknown}" + + # Rollback to LKG — reinstalls a checkpoint of this attempt was already created, + # or safely restores LKG config directly via the installer + apply_lkg || die "rollback to last-known-good also failed" + health_status=$(python3 "$repo_root/scripts/health.py" local --output "$temp_dir/health.json" 2>/dev/null && python3 -c "import json; print(json.load(open('$temp_dir/health.json'))['status'])" 2>/dev/null || echo "unknown") + save_reconcile_state 'rolled_back' "$desired_commit" "$installed_config_ref" "$health_status" "reconciled failed, rolled back to ${installed_config_ref}" + note "ROLLBACK_OK controller=${installed_controller} restored=${installed_config_ref}" + exit 3 +fi diff --git a/scripts/test_remote_reconcile.py b/scripts/test_remote_reconcile.py new file mode 100644 index 00000000..15763a3a --- /dev/null +++ b/scripts/test_remote_reconcile.py @@ -0,0 +1,247 @@ +#!/usr/bin/env python3 +"""Regression tests for remote reconciliation scripts. + +Tests the github-app-token.sh wrapper behaviour, remote-reconcile.sh flow +(validation, fetch, check, reconcile, rollback), and secret redaction. +""" +from __future__ import annotations + +import json +import os +import shutil +import subprocess +import tempfile +import unittest +from pathlib import Path + + +REPO_ROOT = Path(__file__).resolve().parents[1] +SCRIPTS = REPO_ROOT / "scripts" +TOKEN_SCRIPT = SCRIPTS / "github-app-token.sh" +RECONCILE_SCRIPT = SCRIPTS / "remote-reconcile.sh" +TOKEN_SCRIPT.chmod(0o755) +RECONCILE_SCRIPT.chmod(0o755) +INSTALLER = SCRIPTS / "install-worker-controller.sh" + + +def git(*args: str, cwd: str | Path | None = None) -> subprocess.CompletedProcess: + return subprocess.run( + ["git"] + list(args), + capture_output=True, text=True, cwd=cwd, + ) + + +class TestGitHubAppToken(unittest.TestCase): + """Tests for the token helper (unit-level, no real API calls).""" + + def test_requires_args(self): + """Exits 2 with no arguments.""" + result = subprocess.run( + [str(TOKEN_SCRIPT)], capture_output=True, text=True + ) + self.assertEqual(result.returncode, 2) + self.assertIn("ERROR", result.stdout + result.stderr) + + def test_requires_valid_env_file(self): + """Exits 2 with a non-existent --env-file.""" + result = subprocess.run( + [str(TOKEN_SCRIPT), "--env-file", "/nonexistent/path"], + capture_output=True, text=True, + ) + self.assertEqual(result.returncode, 2) + self.assertIn("not found", result.stderr) + + def test_rejects_missing_key(self): + """Exits 2 when the key file does not exist.""" + result = subprocess.run( + [str(TOKEN_SCRIPT), "--app-id", "123", "--install-id", "456", "--key-path", "/nonexistent/key.pem"], + capture_output=True, text=True, + ) + self.assertEqual(result.returncode, 2) + self.assertIn("not found", result.stderr) + + def test_parses_env_file(self): + """Correctly extracts app-id, install-id, and key-path from an env file.""" + with tempfile.TemporaryDirectory() as td: + env_file = Path(td) / "host.env" + env_file.write_text( + "CI_FLEET_GITHUB_APP_CLIENT_ID=98765\n" + "CI_FLEET_GITHUB_APP_INSTALLATION_ID=54321\n" + "CI_FLEET_GITHUB_APP_PRIVATE_KEY_FILE=/etc/ci-fleet/app-key.pem\n" + "CI_FLEET_RUNNER_TTL=6h\n" + ) + result = subprocess.run( + [str(TOKEN_SCRIPT), "--env-file", str(env_file)], + capture_output=True, text=True, + ) + # Should fail because key doesn't exist, but the parsing is correct + self.assertEqual(result.returncode, 2) + self.assertIn("not found", result.stderr) + + def test_stderr_does_not_leak_key(self): + """Error output must not contain the private key content.""" + with tempfile.TemporaryDirectory() as td: + env_file = Path(td) / "host.env" + key_file = Path(td) / "key.pem" + # Write a fake but valid-looking RSA key + key_file.write_text( + "-----BEGIN RSA PRIVATE KEY-----\n" + "MIIEpAIBAAKCAQEAFAKEKEYMATE\n" + "-----END RSA PRIVATE KEY-----\n" + ) + env_file.write_text( + f"CI_FLEET_GITHUB_APP_CLIENT_ID=123\n" + f"CI_FLEET_GITHUB_APP_INSTALLATION_ID=456\n" + f"CI_FLEET_GITHUB_APP_PRIVATE_KEY_FILE={key_file}\n" + ) + result = subprocess.run( + [str(TOKEN_SCRIPT), "--env-file", str(env_file)], + capture_output=True, text=True, + ) + combined = (result.stdout + result.stderr).lower() + # The fake key material itself must not appear in stderr or stdout + self.assertNotIn("fakekeymate", combined) + self.assertNotIn("begin rsa", combined) + + +class TestRemoteReconcile(unittest.TestCase): + """Tests for the reconcile script (fixture-driven, no real API calls).""" + + def setUp(self): + self.td = Path(tempfile.mkdtemp()) + self.state_dir = self.td / "state" + self.lkg_dir = self.td / "lkg" + self.state_file = self.state_dir / "install-state.json" + + # Create a minimal valid install state + self.state_dir.mkdir(parents=True, exist_ok=True) + self._write_state("RandomDevelopment/rd-delivery-config", + "0000000000000000000000000000000000000000", + "rd-ci-fleet-01") + + # Create a fake host.env for token gen + self.host_env = self.td / "host.env" + self.host_env.write_text( + "CI_FLEET_GITHUB_APP_CLIENT_ID=123\n" + "CI_FLEET_GITHUB_APP_INSTALLATION_ID=456\n" + "CI_FLEET_GITHUB_APP_PRIVATE_KEY_FILE=/nonexistent/key.pem\n" + ) + self.host_env.chmod(0o600) + + self.env = os.environ.copy() + self.env.update({ + "CI_FLEET_REMOTE_STATE_FILE": str(self.state_file), + "CI_FLEET_RENDERED_ENV": str(self.td / "ci-fleet.env"), + "CI_FLEET_HOST_ENV": str(self.host_env), + "CI_FLEET_LKG_DIR": str(self.lkg_dir), + "CI_FLEET_RECONCILE_STATE_DIR": str(self.td / "reconcile-state"), + "CI_FLEET_RECONCILE_MAX_ATTEMPTS": "1", + "CI_FLEET_TESTING": "1", + }) + + def tearDown(self): + shutil.rmtree(self.td, ignore_errors=True) + + def _write_state(self, config_repo: str, config_ref: str, controller: str): + state = { + "controller": controller, + "config_repository": config_repo, + "config_ref": config_ref, + "controller_state": "active", + "engine_ref": "0000000000000000000000000000000000000000", + } + self.state_file.write_text(json.dumps(state, indent=2)) + self.state_file.chmod(0o600) + + def test_no_installed_state_exits_2(self): + """Fails with exit 2 when no install-state.json exists.""" + self.state_file.unlink(missing_ok=True) + result = subprocess.run( + [str(RECONCILE_SCRIPT), "--check-only"], + capture_output=True, text=True, env=self.env, + ) + self.assertEqual(result.returncode, 2) + self.assertIn("RECONCILE_ERROR", result.stderr) + + def test_token_env_prefers_host_env(self): + """Uses host.env when available for token generation.""" + # Should try to generate token and fail because key doesn't exist + result = subprocess.run( + [str(RECONCILE_SCRIPT), "--check-only"], + capture_output=True, text=True, env=self.env, + ) + self.assertIn(result.returncode, (1, 2), f"expected 1 or 2, got {result.returncode}") + self.assertIn("token", result.stderr.lower()) + + def test_no_op_flag_succeeds_with_valid_state(self): + """--no-op should parse and validate state without fetching.""" + # It will still try to generate a token, so it'll fail there. + # This tests that --no-op is accepted as a flag. + result = subprocess.run( + [str(RECONCILE_SCRIPT), "--no-op"], + capture_output=True, text=True, env=self.env, + ) + # It should fail on token generation, not arg parsing + self.assertIn(result.returncode, (1, 2), f"expected 1 or 2, got {result.returncode}") + # Verify it got past arg parsing + self.assertIn("GENERATING_TOKEN", result.stderr + result.stdout) + + def test_help_exits_0(self): + """--help prints usage and exits 0.""" + result = subprocess.run( + [str(RECONCILE_SCRIPT), "--help"], + capture_output=True, text=True, env=self.env, + ) + self.assertEqual(result.returncode, 0) + self.assertIn("usage", result.stdout + result.stderr) + + def test_reconcile_state_saved_on_failure(self): + """State file is saved even when reconciliation fails.""" + result = subprocess.run( + [str(RECONCILE_SCRIPT)], + capture_output=True, text=True, env=self.env, + ) + state_file = self.env["CI_FLEET_RECONCILE_STATE_DIR"] + state_path = Path(state_file) / "state.json" + # Should exist even on failure + self.assertTrue(state_path.exists() or result.returncode != 0) + + def test_validate_schema_output_no_secrets(self): + """Sanitized log output must not contain actual key material or token values.""" + result = subprocess.run( + [str(RECONCILE_SCRIPT)], + capture_output=True, text=True, env=self.env, + ) + combined = (result.stdout + result.stderr) + # Must not contain actual key file content or raw AIA value + self.assertNotIn("MIIEpAIBAAKCAQEAFAKEKEYMATE", combined) + # Must not contain raw host.env values + self.assertNotIn("installation_id=456", combined.lower()) + self.assertNotIn("client_id=123", combined.lower()) + + +class TestSystemdUnits(unittest.TestCase): + """The systemd unit files must exist and pass basic validation.""" + + def test_service_file_exists(self): + svc = REPO_ROOT / "host" / "systemd" / "ci-fleet-reconcile.service" + self.assertTrue(svc.exists()) + content = svc.read_text() + self.assertIn("ExecStart", content) + self.assertIn("remote-reconcile.sh", content) + + def test_timer_file_exists(self): + timer = REPO_ROOT / "host" / "systemd" / "ci-fleet-reconcile.timer" + self.assertTrue(timer.exists()) + content = timer.read_text() + self.assertIn("OnUnitActiveSec", content) + + def test_installer_references_units(self): + """Installer script references the new reconcile units.""" + content = INSTALLER.read_text() + self.assertIn("ci-fleet-reconcile.service", content) + self.assertIn("ci-fleet-reconcile.timer", content) + + +if __name__ == "__main__": + unittest.main()