fix: make remote reconciliation transactional

scripts/health.py

@@ -151,6 +151,21 @@ def add(check_id: str, severity: str, **details: Any) -> None:
     add("clock", "ok" if snapshot["clock_synchronized"] else "warning")
     backup = snapshot["backup"]
     add("backup", "warning" if backup == "failed" else "ok", state=backup)
+    reconciliation = snapshot.get("reconciliation")
+    if reconciliation:
+        reconciliation_severity = (
+            "ok" if reconciliation["status"] in {"converged", "bootstrap"}
+            else "warning" if reconciliation["status"] in {"missing", "pending", "reconciling"}
+            else "critical"
+        )
+        add(
+            "reconciliation",
+            reconciliation_severity,
+            state=reconciliation["status"],
+            desired_commit=reconciliation["desired_commit"],
+            applied_commit=reconciliation["applied_commit"],
+            reported_health=reconciliation["health"],
+        )
 
     rank = max(({"ok": 0, "warning": 1, "critical": 2}[check["status"]] for check in checks), default=0)
     overall = ("healthy", "warning", "unhealthy")[rank]
@@ -294,6 +309,24 @@ def _backup_state(values: dict[str, str], run: Runner) -> str:
     return "ok" if run([str(path)]).returncode == 0 else "failed"
 
 
+def _reconcile_state(path: Path) -> dict[str, str]:
+    try:
+        value = json.loads(path.read_text())
+    except (OSError, json.JSONDecodeError):
+        return {"status": "missing", "desired_commit": "", "applied_commit": "", "health": ""}
+    if not isinstance(value, dict):
+        return {"status": "invalid", "desired_commit": "", "applied_commit": "", "health": ""}
+    status = value.get("status", "")
+    if not isinstance(status, str) or status not in {"converged", "drift", "invalid", "pending", "reconciling", "rolled_back", "failed"}:
+        status = "invalid"
+    commits = [value.get(name, "") for name in ("desired_commit", "applied_commit")]
+    commits = [commit if isinstance(commit, str) and (not commit or re.fullmatch(r"[0-9a-f]{40}", commit)) else "invalid" for commit in commits]
+    reported_health = value.get("health", "")
+    if not isinstance(reported_health, str) or reported_health not in {"", "healthy", "warning", "unhealthy", "maintenance", "drift", "unknown"}:
+        reported_health = "invalid"
+    return {"status": status, "desired_commit": commits[0], "applied_commit": commits[1], "health": reported_health}
+
+
 def collect_snapshot(values: dict[str, str], *, root: Path = Path("/"), run: Runner = _run) -> dict[str, Any]:
     docker_root = values.get("CI_FLEET_DOCKER_ROOT", "/var/lib/docker")
     available, swap = _memory(root)
@@ -315,11 +348,20 @@ def collect_snapshot(values: dict[str, str], *, root: Path = Path("/"), run: Run
     oom = run(["journalctl", "--dmesg", "--since=-24h", "--grep=Out of memory|Killed process", "--quiet"])
     configured = {"min": int(values.get("CI_FLEET_MIN_RUNNERS", 0)), "max": int(values.get("CI_FLEET_MAX_RUNNERS", 0))}
     timer_ages = {"health": 900, "cleanup": 172800, "drift": 3600}
+    remote_config = bool(re.fullmatch(r"[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+", values.get("CI_FLEET_CONFIG_REPOSITORY", "")))
+    reconciliation = _reconcile_state(root / "var/lib/ci-fleet/reconcile/state.json") if remote_config else None
+    if reconciliation and values.get("CI_FLEET_HEALTH_BOOTSTRAP") == "1":
+        reconciliation["status"] = "bootstrap"
+    if remote_config:
+        timer_ages["reconcile"] = 900
     timers = {name: _unit_state(run, f"ci-fleet-{name}.timer", timer=True, max_age_seconds=age) for name, age in timer_ages.items()}
-    services = {name: _unit_state(run, unit) for name, unit in {
+    service_units = {
         "cleanup": "ci-fleet-cleanup.service",
         "drift": "ci-fleet-drift.service",
-    }.items()}
+    }
+    if remote_config:
+        service_units["reconcile"] = "ci-fleet-reconcile.service"
+    services = {name: _unit_state(run, unit) for name, unit in service_units.items()}
     debian = (root / "etc/debian_version").exists()
     if debian:
         timers["updates"] = _unit_state(run, "apt-daily-upgrade.timer", timer=True, max_age_seconds=172800)
@@ -351,6 +393,7 @@ def collect_snapshot(values: dict[str, str], *, root: Path = Path("/"), run: Run
         "failed_packages": debian and bool(run(["dpkg", "--audit"]).stdout.strip()),
         "clock_synchronized": run(["timedatectl", "show", "--property=NTPSynchronized", "--value"]).stdout.strip() == "yes",
         "backup": _backup_state(values, run),
+        "reconciliation": reconciliation,
     }
 
 

scripts/install-worker-controller.sh

@@ -6,6 +6,7 @@ export PYTHONDONTWRITEBYTECODE=1
 repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)
 mode=
 config_repo=
+config_identity_arg=
 config_ref=
 controller_id=
 host_config_arg=
@@ -21,7 +22,7 @@ usage() {
 usage:
   install-worker-controller.sh --check|--install|--adopt|--upgrade \
     --config-repo OWNER/REPOSITORY|PATH --ref FULL_COMMIT_SHA \
-    --controller CONTROLLER_ID
+    --controller CONTROLLER_ID [--config-identity OWNER/REPOSITORY]
 
   install-worker-controller.sh --rollback
   install-worker-controller.sh --uninstall
@@ -54,6 +55,11 @@ while (($#)); do
       config_repo=$2
       shift 2
       ;;
+    --config-identity)
+      (($# >= 2)) || die '--config-identity requires a value'
+      config_identity_arg=$2
+      shift 2
+      ;;
     --ref)
       (($# >= 2)) || die '--ref requires a value'
       config_ref=$2
@@ -106,7 +112,7 @@ state_file=$state_root/install-state.json
 health_report=$state_root/health/latest.json
 checkpoints_dir=$state_root/checkpoints
 systemd_dir=$(root_path /etc/systemd/system)
-lock_file=$(root_path /run/ci-fleet-installer.lock)
+lock_file=${CI_FLEET_INSTALLER_LOCK:-$(root_path /run/ci-fleet-installer.lock)}
 controller_container=ci-fleet-controller-1
 unit_names=(
   ci-fleet-health.service ci-fleet-health.timer
@@ -145,19 +151,21 @@ validate_common_arguments() {
   if [[ "$config_repo" == *://* || "$config_repo" == *@* ]]; then
     die '--config-repo must not contain a URL or embedded credentials; use OWNER/REPOSITORY or a local path'
   fi
+  [[ -z "$config_identity_arg" || "$config_identity_arg" =~ ^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$ ]] || die '--config-identity must be OWNER/REPOSITORY'
 }
 
 resolve_config() {
   local resolved checkout
   candidate_config=$temporary/fleet.json
   if is_git_checkout "$config_repo"; then
-    config_identity=$(cd "$config_repo" && pwd -P)
-    config_source_checkout=$config_identity
-    resolved=$(git -C "$config_identity" rev-parse "$config_ref^{commit}" 2>/dev/null || true)
+    config_source_checkout=$(cd "$config_repo" && pwd -P)
+    config_identity=${config_identity_arg:-$config_source_checkout}
+    resolved=$(git -C "$config_source_checkout" rev-parse "$config_ref^{commit}" 2>/dev/null || true)
     [[ "$resolved" == "$config_ref" ]] || die 'local configuration repository does not contain the requested commit'
-    git -C "$config_identity" show "$config_ref:fleet.json" >"$candidate_config" || die 'fleet.json is absent at the requested configuration commit'
+    git -C "$config_source_checkout" show "$config_ref:fleet.json" >"$candidate_config" || die 'fleet.json is absent at the requested configuration commit'
     return
   fi
+  [[ -z "$config_identity_arg" ]] || die '--config-identity is valid only with a local Git checkout'
   [[ "$config_repo" =~ ^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$ ]] || die '--config-repo must be OWNER/REPOSITORY or a local Git checkout'
   checkout=$temporary/config-repository
   config_source_checkout=$checkout
@@ -826,10 +834,10 @@ PY
       # Only enable remote reconciliation timers when config is
       # identified as an OWNER/REPO (not a local checkout path)
       if [[ "$config_identity" == *"/"* && "$config_identity" != "/"* ]]; then
-        systemctl enable --now "$opt_timer" >/dev/null 2>&1 || true
-      else
+        systemctl enable --now "$opt_timer" >/dev/null
+      elif [[ -f "$systemd_dir/$opt_timer" ]]; then
         # Local checkout path — disable and stop any previously enabled timer
-        systemctl disable --now "$opt_timer" >/dev/null 2>&1 || true
+        systemctl disable --now "$opt_timer" >/dev/null
       fi
     ;; esac
   done
@@ -1030,9 +1038,15 @@ perform_uninstall() {
 }
 
 require_commands
-install -d -m 0755 "$(dirname "$lock_file")"
-exec 9>"$lock_file"
-flock -n 9 || die 'another ci-fleet installer or drift check is already running'
+if [[ -n ${CI_FLEET_INSTALLER_LOCK_FD:-} ]]; then
+  [[ "$CI_FLEET_INSTALLER_LOCK_FD" == 9 ]] || die 'inherited installer lock must use file descriptor 9'
+  [[ $(readlink -f /proc/self/fd/9 2>/dev/null || true) == $(readlink -m "$lock_file") ]] || die 'inherited installer lock does not match the configured lock file'
+  flock -n 9 || die 'inherited installer lock is unavailable'
+else
+  install -d -m 0755 "$(dirname "$lock_file")"
+  exec 9>"$lock_file"
+  flock -n 9 || die 'another ci-fleet installer or drift check is already running'
+fi
 case "$mode" in
   check|install|adopt|upgrade)
     validate_common_arguments

scripts/remote-reconcile.sh

@@ -215,9 +215,6 @@ PY
   lkg_repo=${lkg_vals[1]}
   lkg_controller=${lkg_vals[2]}
 
-  local lkg_config=$lkg_dir/fleet.json
-  [[ -f "$lkg_config" ]] || { log_json "ERROR" "rollback" "LKG fleet.json missing"; return 1; }
-
   # Re-apply the LKG ref via the installer
   # Create a checkout containing the LKG ref (may differ from fetched HEAD)
   local lkg_pinned=$temp_dir/lkg-pinned
@@ -240,85 +237,23 @@ PY
   fi
   git -C "$lkg_pinned" checkout -q FETCH_HEAD
 
-  release_lock
-  "$installer" --upgrade \
+  CI_FLEET_INSTALLER_LOCK_FD=9 "$installer" --upgrade \
     --config-repo "$lkg_pinned" \
+    --config-identity "$lkg_repo" \
     --ref "$lkg_ref" \
     --controller "$lkg_controller" 2>"$temp_dir/rollback_err" && {
-    acquire_lock
-    # Fix the config_repository in the state file to the durable name
-    fix_state_config_repo "$lkg_repo"
     log_json "WARN" "rollback" "restored last-known-good"
     return 0
   }
-  acquire_lock
   local err
   err=$(<"$temp_dir/rollback_err")
   log_json "ERROR" "rollback" "rollback failed: ${err}"
   return 1
 }
 
-fix_state_config_repo() {
-  local durable=$1
-  [[ -f "$state_file" ]] || return 0
-  python3 - "$state_file" "$durable" <<'PY' 2>/dev/null || true
-import json, os, sys, tempfile
-
-path = sys.argv[1]
-durable = sys.argv[2]
-state = json.load(open(path, encoding="utf-8"))
-if state.get("config_repository") != durable:
-    state["config_repository"] = durable
-    fd, tmp = tempfile.mkstemp(prefix=".fix-state.", dir=os.path.dirname(path), text=True)
-    try:
-        with os.fdopen(fd, "w", encoding="utf-8") as f:
-            json.dump(state, f, indent=2, sort_keys=True)
-            f.write("\n")
-        os.chmod(tmp, 0o600)
-        os.replace(tmp, path)
-    except:
-        os.unlink(tmp, missing_ok=True)
-        raise
-PY
-}
-
-fix_rendered_env_config_repo() {
-  local durable=$1
-  [[ -f "$rendered_env" ]] || return 0
-  python3 - "$rendered_env" "$durable" <<'PY' 2>/dev/null || true
-import os, sys, tempfile
-path = sys.argv[1]
-durable = sys.argv[2]
-with open(path, encoding="utf-8") as f:
-    lines = f.readlines()
-changed = False
-for i, line in enumerate(lines):
-    if line.startswith("CI_FLEET_CONFIG_REPOSITORY="):
-        val = line.split("=", 1)[1].strip()
-        if val != durable:
-            lines[i] = f"CI_FLEET_CONFIG_REPOSITORY={durable}\n"
-            changed = True
-        break
-if not changed:
-    raise SystemExit(0)
-fd, tmp = tempfile.mkstemp(prefix=".fix-env.", dir=os.path.dirname(path), text=True)
-try:
-    with os.fdopen(fd, "w", encoding="utf-8") as f:
-        f.writelines(lines)
-        f.flush()
-        os.fsync(f.fileno())
-    os.chmod(tmp, 0o600)
-    os.replace(tmp, path)
-except:
-    os.unlink(tmp, missing_ok=True)
-    raise
-PY
-}
-
 save_lkg() {
-  local checkout_dir=$1 commit=$2
+  local commit=$1
   install -d -m 0700 "$lkg_dir"
-  git -C "$checkout_dir" show "$commit:fleet.json" >"$lkg_dir/fleet.json" 2>/dev/null || true
   python3 - "$lkg_dir/metadata.json" "$installed_config_repo" "$commit" "$installed_controller" <<'PY' 2>/dev/null || true
 import json, os, sys, tempfile
 
@@ -360,9 +295,6 @@ print(json.dumps({
 PY
 }
 
-release_lock() { flock -u 9 2>/dev/null || true; }
-acquire_lock() { flock -n 9 2>/dev/null || die "cannot re-acquire installer lock"; }
-
 # --- Health (with rendered env) ---
 
 run_health_check() {
@@ -373,7 +305,8 @@ run_health_check() {
     [[ ! -f "$rendered_env" ]] || . "$rendered_env"
     set +a
     python3 "$repo_root/scripts/health.py" local --output "$output" 2>/dev/null
-  ) && python3 -c "import json; print(json.load(open('$output'))['status'])" 2>/dev/null || echo "unknown"
+  ) || true
+  python3 -c "import json; print(json.load(open('$output'))['status'])" 2>/dev/null || echo "unknown"
 }
 
 # --- Main ---
@@ -439,33 +372,22 @@ if [[ "$desired_commit" == "$installed_config_ref" ]]; then
   # Run drift check using the fetched local checkout
   local_pinned=$temp_dir/config-repo
   if [[ -d "$local_pinned/.git" ]]; then
-    release_lock
-    if "$installer" --check \
+    if CI_FLEET_INSTALLER_LOCK_FD=9 "$installer" --check \
       --config-repo "$local_pinned" \
+      --config-identity "$installed_config_repo" \
       --ref "$installed_config_ref" \
       --controller "$installed_controller" 2>"$temp_dir/drift_err"; then
-      acquire_lock
       note "CONVERGED controller=${installed_controller} config_ref=${installed_config_ref}"
       save_reconcile_state 'converged' "$desired_commit" "$installed_config_ref" 'healthy' 'no change, converged'
       exit 0
     fi
-    acquire_lock
   fi
-  # Same commit + drift = check controller health
-  # If controller is unhealthy, reconcile; otherwise converge with drift note
+  # Same commit + drift is still drift: check-only reports it, normal mode repairs it.
   if [[ "$mode" == check-only ]]; then
     save_reconcile_state 'drift' "$desired_commit" "$installed_config_ref" 'drift' 'internal drift detected'
     exit 3
   fi
-  # Full mode: run health check to decide if reconciliation is needed
-  controller_running=$(docker inspect --format '{{.State.Status}}' "ci-fleet-controller-1" 2>/dev/null || echo "missing")
-  if [[ "$controller_running" != "running" ]]; then
-    note "DRIFT with unhealthy controller, falling through to reconcile"
-  else
-    note "CONVERGED controller=${installed_controller} config_ref=${installed_config_ref}"
-    save_reconcile_state 'converged' "$desired_commit" "$installed_config_ref" 'drift' 'no commit change; internal drift tracked by drift timer'
-    exit 0
-  fi
+  note "DRIFT falling through to reconcile"
 fi
 
 # New commit or drift — validate and reconcile
@@ -475,7 +397,7 @@ if ! validate_config "$fetch_dir" "$desired_commit"; then
   note "INVALID_CONFIG commit=${desired_commit}"
   save_reconcile_state 'invalid' "$desired_commit" "$installed_config_ref" 'healthy' "config rejected at ${desired_commit}"
   if [[ "$installed_config_ref" != "$desired_commit" ]]; then
-    save_lkg "$fetch_dir" "$installed_config_ref"
+    save_lkg "$installed_config_ref"
   fi
   exit 3
 fi
@@ -495,44 +417,35 @@ if [[ "$no_op" == true ]]; then
 fi
 
 # Save LKG before reconciling
-save_lkg "$fetch_dir" "$installed_config_ref"
+save_lkg "$installed_config_ref"
 
 # Create a pinned local checkout for the installer.
-# Pass the durable config_repo name so install-state.json records
-# the correct identity; the installer uses the local checkout path
-# for fleet.json resolution.
+# Keep the durable repository identity while the installer reads the fetched checkout.
 pinned_dir=$temp_dir/pinned-config
 cp -a "$fetch_dir" "$pinned_dir"
 git -C "$pinned_dir" checkout -q "$desired_commit"
 
 # Reconcile
 note "RECONCILING controller=${installed_controller} config_ref=${desired_commit}"
-release_lock
-if "$installer" --upgrade \
+save_reconcile_state 'reconciling' "$desired_commit" "$installed_config_ref" 'unknown' "reconciling to ${desired_commit}"
+if CI_FLEET_INSTALLER_LOCK_FD=9 "$installer" --upgrade \
   --config-repo "$pinned_dir" \
+  --config-identity "$installed_config_repo" \
   --ref "$desired_commit" \
   --controller "$installed_controller" 2>"$temp_dir/upgrade_err"; then
-  acquire_lock
   note "RECONCILED controller=${installed_controller} config_ref=${desired_commit}"
 
-  # Fix config_repository in state file AND rendered env to the durable name
-  fix_state_config_repo "$installed_config_repo"
-  fix_rendered_env_config_repo "$installed_config_repo"
-
-  # Re-enable reconcile timer (may have been disabled during local-checkout upgrade)
-  systemctl enable --now ci-fleet-reconcile.timer >/dev/null 2>&1 || true
-
   # Save new LKG
-  save_lkg "$fetch_dir" "$desired_commit"
+  save_lkg "$desired_commit"
 
   # Run health check
+  save_reconcile_state 'converged' "$desired_commit" "$desired_commit" 'unknown' "reconciled to ${desired_commit}; checking health"
   health_status=$(run_health_check "$temp_dir/health.json")
 
   save_reconcile_state 'converged' "$desired_commit" "$desired_commit" "$health_status" "reconciled to ${desired_commit}"
   note "RECONCILE_OK controller=${installed_controller} desired=${desired_commit} applied=${desired_commit} health=${health_status}"
   exit 0
 else
-  acquire_lock
   upg_err=$(<"$temp_dir/upgrade_err")
   note "RECONCILE_FAILED error=${upg_err:-unknown}"