diff --git a/host/systemd/ci-fleet-reconcile.timer b/host/systemd/ci-fleet-reconcile.timer index b96492ea..b5ca500b 100644 --- a/host/systemd/ci-fleet-reconcile.timer +++ b/host/systemd/ci-fleet-reconcile.timer @@ -3,7 +3,7 @@ Description=Reconcile ci-fleet controller every five minutes Documentation=https://github.com/RandomDevelopment/ci-fleet [Timer] -OnBootSec=10min +OnActiveSec=2min OnUnitActiveSec=5min AccuracySec=30s Persistent=true diff --git a/scripts/install-worker-controller.sh b/scripts/install-worker-controller.sh index 5adcfa5a..c9629153 100755 --- a/scripts/install-worker-controller.sh +++ b/scripts/install-worker-controller.sh @@ -629,7 +629,7 @@ make_checkpoint() { printf '%s\n' "$target" >"$checkpoint_dir/manager-target" chmod 0600 "$checkpoint_dir/manager-target" fi - for unit in "${unit_names[@]}"; do + for unit in "${unit_names[@]}" "${optional_unit_names[@]}"; do [[ ! -f "$systemd_dir/$unit" ]] || install -m 0644 "$systemd_dir/$unit" "$checkpoint_dir/systemd/$unit" done : >"$checkpoint_dir/enabled-timers" @@ -638,6 +638,13 @@ make_checkpoint() { if systemctl is-enabled --quiet "$timer" 2>/dev/null; then printf '%s\n' "$timer" >>"$checkpoint_dir/enabled-timers"; fi if systemctl is-active --quiet "$timer" 2>/dev/null; then printf '%s\n' "$timer" >>"$checkpoint_dir/active-timers"; fi done + local opt_name + for opt_name in "${optional_unit_names[@]}"; do + case "$opt_name" in *.timer) + if systemctl is-enabled --quiet "$opt_name" 2>/dev/null; then printf '%s\n' "$opt_name" >>"$checkpoint_dir/enabled-timers"; fi + if systemctl is-active --quiet "$opt_name" 2>/dev/null; then printf '%s\n' "$opt_name" >>"$checkpoint_dir/active-timers"; fi + ;; esac + done chmod 0600 "$checkpoint_dir/enabled-timers" "$checkpoint_dir/active-timers" : >"$checkpoint_dir/.complete" chmod 0600 "$checkpoint_dir/.complete" @@ -731,6 +738,9 @@ install_systemd_units() { remove_systemd_units() { systemctl disable --now "${timer_names[@]}" >/dev/null 2>&1 || true local unit + for unit in "${optional_unit_names[@]}"; do + case "$unit" in *.timer) systemctl disable --now "$unit" >/dev/null 2>&1 || true ;; esac + done for unit in "${unit_names[@]}" "${optional_unit_names[@]}"; do rm -f "$systemd_dir/$unit"; done systemctl daemon-reload } @@ -813,7 +823,14 @@ PY local opt_timer for opt_timer in "${optional_unit_names[@]}"; do case "$opt_timer" in *.timer) - systemctl enable --now "$opt_timer" >/dev/null 2>&1 || true + # Only enable remote reconciliation timers when config is + # identified as an OWNER/REPO (not a local checkout path) + if [[ "$config_identity" == *"/"* && "$config_identity" != "/"* ]]; then + systemctl enable --now "$opt_timer" >/dev/null 2>&1 || true + else + # Local checkout path — disable and stop any previously enabled timer + systemctl disable --now "$opt_timer" >/dev/null 2>&1 || true + fi ;; esac done } @@ -832,6 +849,13 @@ restore_systemd_snapshot() { if grep -Fxq "$timer" "$checkpoint_dir/enabled-timers"; then systemctl enable "$timer" >/dev/null || failed=1; else systemctl disable "$timer" >/dev/null 2>&1 || true; fi if grep -Fxq "$timer" "$checkpoint_dir/active-timers"; then systemctl start "$timer" || failed=1; else systemctl stop "$timer" >/dev/null 2>&1 || true; fi done + local opt_name + for opt_name in "${optional_unit_names[@]}"; do + case "$opt_name" in *.timer) + if grep -Fxq "$opt_name" "$checkpoint_dir/enabled-timers"; then systemctl enable "$opt_name" >/dev/null || failed=1; else systemctl disable "$opt_name" >/dev/null 2>&1 || true; fi + if grep -Fxq "$opt_name" "$checkpoint_dir/active-timers"; then systemctl start "$opt_name" || failed=1; else systemctl stop "$opt_name" >/dev/null 2>&1 || true; fi + ;; esac + done return "$failed" } diff --git a/scripts/remote-reconcile.sh b/scripts/remote-reconcile.sh index 79264d57..a953becc 100755 --- a/scripts/remote-reconcile.sh +++ b/scripts/remote-reconcile.sh @@ -163,6 +163,7 @@ validate_config() { git -C "$checkout_dir" ls-tree -rz --name-only "$commit" >"$temp_dir/tree-paths" 2>/dev/null || return 1 # Validate using the installer's validation chain + # Also run the template validator with --strict + --tree-paths (like installer does) python3 "$repo_root/scripts/desired_state.py" validate --config "$temp_dir/fleet.json" 2>"$temp_dir/validate_err" || { local err err=$(<"$temp_dir/validate_err") @@ -170,6 +171,14 @@ validate_config() { log_json "ERROR" "validation" "config validation failed" return 1 } + python3 "$repo_root/templates/config-repository/scripts/validate.py" \ + --config "$temp_dir/fleet.json" --strict --tree-paths "$temp_dir/tree-paths" 2>"$temp_dir/strict_err" || { + local err + err=$(<"$temp_dir/strict_err") + [[ -n "$err" ]] || err="strict validation failed" + log_json "ERROR" "validation" "strict validation rejected" + return 1 + } # Secret scan python3 "$repo_root/scripts/scan_committed_secrets.py" \ @@ -209,30 +218,40 @@ PY local lkg_config=$lkg_dir/fleet.json [[ -f "$lkg_config" ]] || { log_json "ERROR" "rollback" "LKG fleet.json missing"; return 1; } - # Re-apply the LKG ref via the installer, using the durable repo identity - # Create a local checkout pinned to the LKG ref for the installer + # Re-apply the LKG ref via the installer + # Create a checkout containing the LKG ref (may differ from fetched HEAD) local lkg_pinned=$temp_dir/lkg-pinned - cp -a "$temp_dir/config-repo" "$lkg_pinned" 2>/dev/null || { - # Fallback: fresh fetch - mkdir -p "$lkg_pinned" - git init -q "$lkg_pinned" - git -C "$lkg_pinned" remote add origin "https://github.com/${lkg_repo}.git" - GIT_TERMINAL_PROMPT=0 git -C "$lkg_pinned" fetch -q --depth=1 origin "$lkg_ref" 2>/dev/null || { + mkdir -p "$lkg_pinned" + git init -q "$lkg_pinned" + # Use the reconciliation token for authenticated fetch + local lkg_token + lkg_token=$(cat "$temp_dir/reconcile-token" 2>/dev/null || echo "") + if [[ -n "$lkg_token" ]]; then + GIT_TERMINAL_PROMPT=0 git -C "$lkg_pinned" fetch -q --depth=1 \ + "https://x-access-token:${lkg_token}@github.com/${lkg_repo}.git" "$lkg_ref" 2>"$temp_dir/lkg_fetch_err" || { log_json "ERROR" "rollback" "LKG fetch failed" return 1 } - } + else + GIT_TERMINAL_PROMPT=0 git -C "$lkg_pinned" fetch -q --depth=1 origin "$lkg_ref" 2>"$temp_dir/lkg_fetch_err" || { + log_json "ERROR" "rollback" "LKG fetch failed" + return 1 + } + fi + git -C "$lkg_pinned" checkout -q FETCH_HEAD + release_lock "$installer" --upgrade \ --config-repo "$lkg_pinned" \ --ref "$lkg_ref" \ --controller "$lkg_controller" 2>"$temp_dir/rollback_err" && { + acquire_lock # Fix the config_repository in the state file to the durable name fix_state_config_repo "$lkg_repo" log_json "WARN" "rollback" "restored last-known-good" return 0 } - + acquire_lock local err err=$(<"$temp_dir/rollback_err") log_json "ERROR" "rollback" "rollback failed: ${err}" @@ -263,6 +282,39 @@ if state.get("config_repository") != durable: PY } +fix_rendered_env_config_repo() { + local durable=$1 + [[ -f "$rendered_env" ]] || return 0 + python3 - "$rendered_env" "$durable" <<'PY' 2>/dev/null || true +import os, sys, tempfile +path = sys.argv[1] +durable = sys.argv[2] +with open(path, encoding="utf-8") as f: + lines = f.readlines() +changed = False +for i, line in enumerate(lines): + if line.startswith("CI_FLEET_CONFIG_REPOSITORY="): + val = line.split("=", 1)[1].strip() + if val != durable: + lines[i] = f"CI_FLEET_CONFIG_REPOSITORY={durable}\n" + changed = True + break +if not changed: + raise SystemExit(0) +fd, tmp = tempfile.mkstemp(prefix=".fix-env.", dir=os.path.dirname(path), text=True) +try: + with os.fdopen(fd, "w", encoding="utf-8") as f: + f.writelines(lines) + f.flush() + os.fsync(f.fileno()) + os.chmod(tmp, 0o600) + os.replace(tmp, path) +except: + os.unlink(tmp, missing_ok=True) + raise +PY +} + save_lkg() { local checkout_dir=$1 commit=$2 install -d -m 0700 "$lkg_dir" @@ -308,10 +360,32 @@ print(json.dumps({ PY } +release_lock() { flock -u 9 2>/dev/null || true; } +acquire_lock() { flock -n 9 2>/dev/null || die "cannot re-acquire installer lock"; } + +# --- Health (with rendered env) --- + +run_health_check() { + local output=$1 + ( + set -a + # shellcheck disable=SC1090 + [[ ! -f "$rendered_env" ]] || . "$rendered_env" + set +a + python3 "$repo_root/scripts/health.py" local --output "$output" 2>/dev/null + ) && python3 -c "import json; print(json.load(open('$output'))['status'])" 2>/dev/null || echo "unknown" +} + # --- Main --- require_commands +# Serialize with installer mutations — share the installer's lock +lock_file=${CI_FLEET_INSTALLER_LOCK:-/run/ci-fleet-installer.lock} +install -d -m 0755 "$(dirname "$lock_file")" +exec 9>"$lock_file" +flock -n 9 || die "another reconcile or installer is already running" + # Load installed state load_installed_state || die "no installed state found at $state_file" note "INSTALLED controller=${installed_controller} config_repo=${installed_config_repo} config_ref=${installed_config_ref}" @@ -338,6 +412,7 @@ while ((attempt < max_attempts)); do ((attempt < max_attempts)) && { sleep 5; continue; } die "token generation exhausted after ${max_attempts} attempts" } + printf '%s' "$token" >"$temp_dir/reconcile-token" # Fetch remote config note "FETCHING_CONFIG repo=${installed_config_repo}" @@ -361,21 +436,35 @@ if [[ "$desired_commit" == "$installed_config_ref" ]]; then exit 0 fi - # Run existing drift check - if "$installer" --check \ - --config-repo "$installed_config_repo" \ - --ref "$installed_config_ref" \ - --controller "$installed_controller" 2>"$temp_dir/drift_err"; then + # Run drift check using the fetched local checkout + local_pinned=$temp_dir/config-repo + if [[ -d "$local_pinned/.git" ]]; then + release_lock + if "$installer" --check \ + --config-repo "$local_pinned" \ + --ref "$installed_config_ref" \ + --controller "$installed_controller" 2>"$temp_dir/drift_err"; then + acquire_lock + note "CONVERGED controller=${installed_controller} config_ref=${installed_config_ref}" + save_reconcile_state 'converged' "$desired_commit" "$installed_config_ref" 'healthy' 'no change, converged' + exit 0 + fi + acquire_lock + fi + # Same commit + drift = check controller health + # If controller is unhealthy, reconcile; otherwise converge with drift note + if [[ "$mode" == check-only ]]; then + save_reconcile_state 'drift' "$desired_commit" "$installed_config_ref" 'drift' 'internal drift detected' + exit 3 + fi + # Full mode: run health check to decide if reconciliation is needed + controller_running=$(docker inspect --format '{{.State.Status}}' "ci-fleet-controller-1" 2>/dev/null || echo "missing") + if [[ "$controller_running" != "running" ]]; then + note "DRIFT with unhealthy controller, falling through to reconcile" + else note "CONVERGED controller=${installed_controller} config_ref=${installed_config_ref}" - save_reconcile_state 'converged' "$desired_commit" "$installed_config_ref" 'healthy' 'no change, converged' + save_reconcile_state 'converged' "$desired_commit" "$installed_config_ref" 'drift' 'no commit change; internal drift tracked by drift timer' exit 0 - else - drift_exit=$? - note "DRIFT detected (exit=${drift_exit}), attempting reconcile" - if [[ "$mode" == check-only ]]; then - save_reconcile_state 'drift' "$desired_commit" "$installed_config_ref" 'drift' "drift detected (exit=${drift_exit})" - exit 3 - fi fi fi @@ -418,32 +507,39 @@ git -C "$pinned_dir" checkout -q "$desired_commit" # Reconcile note "RECONCILING controller=${installed_controller} config_ref=${desired_commit}" +release_lock if "$installer" --upgrade \ - --config-repo "$installed_config_repo" \ + --config-repo "$pinned_dir" \ --ref "$desired_commit" \ --controller "$installed_controller" 2>"$temp_dir/upgrade_err"; then + acquire_lock note "RECONCILED controller=${installed_controller} config_ref=${desired_commit}" - # Fix the config_repository in the state file to the durable name + # Fix config_repository in state file AND rendered env to the durable name fix_state_config_repo "$installed_config_repo" + fix_rendered_env_config_repo "$installed_config_repo" + + # Re-enable reconcile timer (may have been disabled during local-checkout upgrade) + systemctl enable --now ci-fleet-reconcile.timer >/dev/null 2>&1 || true # Save new LKG save_lkg "$fetch_dir" "$desired_commit" # Run health check - health_status=$(python3 "$repo_root/scripts/health.py" local --output "$temp_dir/health.json" 2>/dev/null && python3 -c "import json; print(json.load(open('$temp_dir/health.json'))['status'])" 2>/dev/null || echo "unknown") + health_status=$(run_health_check "$temp_dir/health.json") save_reconcile_state 'converged' "$desired_commit" "$desired_commit" "$health_status" "reconciled to ${desired_commit}" note "RECONCILE_OK controller=${installed_controller} desired=${desired_commit} applied=${desired_commit} health=${health_status}" exit 0 else + acquire_lock upg_err=$(<"$temp_dir/upgrade_err") note "RECONCILE_FAILED error=${upg_err:-unknown}" # Rollback to LKG — reinstalls a checkpoint of this attempt was already created, # or safely restores LKG config directly via the installer apply_lkg || die "rollback to last-known-good also failed" - health_status=$(python3 "$repo_root/scripts/health.py" local --output "$temp_dir/health.json" 2>/dev/null && python3 -c "import json; print(json.load(open('$temp_dir/health.json'))['status'])" 2>/dev/null || echo "unknown") + health_status=$(run_health_check "$temp_dir/health.json") save_reconcile_state 'rolled_back' "$desired_commit" "$installed_config_ref" "$health_status" "reconciled failed, rolled back to ${installed_config_ref}" note "ROLLBACK_OK controller=${installed_controller} restored=${installed_config_ref}" exit 3