From dcadbb7a677cc5841585efe2c5a54cd2694ac1f7 Mon Sep 17 00:00:00 2001
From: Nickfost <1572453+Nickfost@users.noreply.github.com>
Date: Tue, 28 Jul 2026 00:29:41 -0500
Subject: [PATCH 1/4] fix: make remote reconciliation transactional

---
 scripts/health.py                         |  10 +-
 scripts/install-worker-controller.sh      |  34 +++++--
 scripts/remote-reconcile.sh               | 119 +++-------------------
 scripts/test-install-worker-controller.sh |  18 ++++
 scripts/test_health.py                    |  11 ++
 scripts/test_remote_reconcile.py          |  38 +++++++
 6 files changed, 114 insertions(+), 116 deletions(-)

diff --git a/scripts/health.py b/scripts/health.py
index ab054aa6..1bc4825c 100644
--- a/scripts/health.py
+++ b/scripts/health.py
@@ -315,11 +315,17 @@ def collect_snapshot(values: dict[str, str], *, root: Path = Path("/"), run: Run
     oom = run(["journalctl", "--dmesg", "--since=-24h", "--grep=Out of memory|Killed process", "--quiet"])
     configured = {"min": int(values.get("CI_FLEET_MIN_RUNNERS", 0)), "max": int(values.get("CI_FLEET_MAX_RUNNERS", 0))}
     timer_ages = {"health": 900, "cleanup": 172800, "drift": 3600}
+    remote_config = bool(re.fullmatch(r"[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+", values.get("CI_FLEET_CONFIG_REPOSITORY", "")))
+    if remote_config:
+        timer_ages["reconcile"] = 900
     timers = {name: _unit_state(run, f"ci-fleet-{name}.timer", timer=True, max_age_seconds=age) for name, age in timer_ages.items()}
-    services = {name: _unit_state(run, unit) for name, unit in {
+    service_units = {
         "cleanup": "ci-fleet-cleanup.service",
         "drift": "ci-fleet-drift.service",
-    }.items()}
+    }
+    if remote_config:
+        service_units["reconcile"] = "ci-fleet-reconcile.service"
+    services = {name: _unit_state(run, unit) for name, unit in service_units.items()}
     debian = (root / "etc/debian_version").exists()
     if debian:
         timers["updates"] = _unit_state(run, "apt-daily-upgrade.timer", timer=True, max_age_seconds=172800)
diff --git a/scripts/install-worker-controller.sh b/scripts/install-worker-controller.sh
index c9629153..abaa65d9 100755
--- a/scripts/install-worker-controller.sh
+++ b/scripts/install-worker-controller.sh
@@ -6,6 +6,7 @@ export PYTHONDONTWRITEBYTECODE=1
 repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)
 mode=
 config_repo=
+config_identity_arg=
 config_ref=
 controller_id=
 host_config_arg=
@@ -21,7 +22,7 @@ usage() {
 usage:
   install-worker-controller.sh --check|--install|--adopt|--upgrade \
     --config-repo OWNER/REPOSITORY|PATH --ref FULL_COMMIT_SHA \
-    --controller CONTROLLER_ID
+    --controller CONTROLLER_ID [--config-identity OWNER/REPOSITORY]
 
   install-worker-controller.sh --rollback
   install-worker-controller.sh --uninstall
@@ -54,6 +55,11 @@ while (($#)); do
       config_repo=$2
       shift 2
       ;;
+    --config-identity)
+      (($# >= 2)) || die '--config-identity requires a value'
+      config_identity_arg=$2
+      shift 2
+      ;;
     --ref)
       (($# >= 2)) || die '--ref requires a value'
       config_ref=$2
@@ -145,19 +151,21 @@ validate_common_arguments() {
   if [[ "$config_repo" == *://* || "$config_repo" == *@* ]]; then
     die '--config-repo must not contain a URL or embedded credentials; use OWNER/REPOSITORY or a local path'
   fi
+  [[ -z "$config_identity_arg" || "$config_identity_arg" =~ ^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$ ]] || die '--config-identity must be OWNER/REPOSITORY'
 }
 
 resolve_config() {
   local resolved checkout
   candidate_config=$temporary/fleet.json
   if is_git_checkout "$config_repo"; then
-    config_identity=$(cd "$config_repo" && pwd -P)
-    config_source_checkout=$config_identity
-    resolved=$(git -C "$config_identity" rev-parse "$config_ref^{commit}" 2>/dev/null || true)
+    config_source_checkout=$(cd "$config_repo" && pwd -P)
+    config_identity=${config_identity_arg:-$config_source_checkout}
+    resolved=$(git -C "$config_source_checkout" rev-parse "$config_ref^{commit}" 2>/dev/null || true)
     [[ "$resolved" == "$config_ref" ]] || die 'local configuration repository does not contain the requested commit'
-    git -C "$config_identity" show "$config_ref:fleet.json" >"$candidate_config" || die 'fleet.json is absent at the requested configuration commit'
+    git -C "$config_source_checkout" show "$config_ref:fleet.json" >"$candidate_config" || die 'fleet.json is absent at the requested configuration commit'
     return
   fi
+  [[ -z "$config_identity_arg" ]] || die '--config-identity is valid only with a local Git checkout'
   [[ "$config_repo" =~ ^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$ ]] || die '--config-repo must be OWNER/REPOSITORY or a local Git checkout'
   checkout=$temporary/config-repository
   config_source_checkout=$checkout
@@ -826,10 +834,10 @@ PY
       # Only enable remote reconciliation timers when config is
       # identified as an OWNER/REPO (not a local checkout path)
       if [[ "$config_identity" == *"/"* && "$config_identity" != "/"* ]]; then
-        systemctl enable --now "$opt_timer" >/dev/null 2>&1 || true
+        systemctl enable --now "$opt_timer" >/dev/null
       else
         # Local checkout path — disable and stop any previously enabled timer
-        systemctl disable --now "$opt_timer" >/dev/null 2>&1 || true
+        systemctl disable --now "$opt_timer" >/dev/null
       fi
     ;; esac
   done
@@ -1030,9 +1038,15 @@ perform_uninstall() {
 }
 
 require_commands
-install -d -m 0755 "$(dirname "$lock_file")"
-exec 9>"$lock_file"
-flock -n 9 || die 'another ci-fleet installer or drift check is already running'
+if [[ -n ${CI_FLEET_INSTALLER_LOCK_FD:-} ]]; then
+  [[ "$CI_FLEET_INSTALLER_LOCK_FD" == 9 ]] || die 'inherited installer lock must use file descriptor 9'
+  [[ $(readlink -f /proc/self/fd/9 2>/dev/null || true) == $(readlink -m "$lock_file") ]] || die 'inherited installer lock does not match the configured lock file'
+  flock -n 9 || die 'inherited installer lock is unavailable'
+else
+  install -d -m 0755 "$(dirname "$lock_file")"
+  exec 9>"$lock_file"
+  flock -n 9 || die 'another ci-fleet installer or drift check is already running'
+fi
 case "$mode" in
   check|install|adopt|upgrade)
     validate_common_arguments
diff --git a/scripts/remote-reconcile.sh b/scripts/remote-reconcile.sh
index a953becc..468d7e54 100755
--- a/scripts/remote-reconcile.sh
+++ b/scripts/remote-reconcile.sh
@@ -215,9 +215,6 @@ PY
   lkg_repo=${lkg_vals[1]}
   lkg_controller=${lkg_vals[2]}
 
-  local lkg_config=$lkg_dir/fleet.json
-  [[ -f "$lkg_config" ]] || { log_json "ERROR" "rollback" "LKG fleet.json missing"; return 1; }
-
   # Re-apply the LKG ref via the installer
   # Create a checkout containing the LKG ref (may differ from fetched HEAD)
   local lkg_pinned=$temp_dir/lkg-pinned
@@ -240,85 +237,23 @@ PY
   fi
   git -C "$lkg_pinned" checkout -q FETCH_HEAD
 
-  release_lock
-  "$installer" --upgrade \
+  CI_FLEET_INSTALLER_LOCK_FD=9 "$installer" --upgrade \
     --config-repo "$lkg_pinned" \
+    --config-identity "$lkg_repo" \
     --ref "$lkg_ref" \
     --controller "$lkg_controller" 2>"$temp_dir/rollback_err" && {
-    acquire_lock
-    # Fix the config_repository in the state file to the durable name
-    fix_state_config_repo "$lkg_repo"
     log_json "WARN" "rollback" "restored last-known-good"
     return 0
   }
-  acquire_lock
   local err
   err=$(<"$temp_dir/rollback_err")
   log_json "ERROR" "rollback" "rollback failed: ${err}"
   return 1
 }
 
-fix_state_config_repo() {
-  local durable=$1
-  [[ -f "$state_file" ]] || return 0
-  python3 - "$state_file" "$durable" <<'PY' 2>/dev/null || true
-import json, os, sys, tempfile
-
-path = sys.argv[1]
-durable = sys.argv[2]
-state = json.load(open(path, encoding="utf-8"))
-if state.get("config_repository") != durable:
-    state["config_repository"] = durable
-    fd, tmp = tempfile.mkstemp(prefix=".fix-state.", dir=os.path.dirname(path), text=True)
-    try:
-        with os.fdopen(fd, "w", encoding="utf-8") as f:
-            json.dump(state, f, indent=2, sort_keys=True)
-            f.write("\n")
-        os.chmod(tmp, 0o600)
-        os.replace(tmp, path)
-    except:
-        os.unlink(tmp, missing_ok=True)
-        raise
-PY
-}
-
-fix_rendered_env_config_repo() {
-  local durable=$1
-  [[ -f "$rendered_env" ]] || return 0
-  python3 - "$rendered_env" "$durable" <<'PY' 2>/dev/null || true
-import os, sys, tempfile
-path = sys.argv[1]
-durable = sys.argv[2]
-with open(path, encoding="utf-8") as f:
-    lines = f.readlines()
-changed = False
-for i, line in enumerate(lines):
-    if line.startswith("CI_FLEET_CONFIG_REPOSITORY="):
-        val = line.split("=", 1)[1].strip()
-        if val != durable:
-            lines[i] = f"CI_FLEET_CONFIG_REPOSITORY={durable}\n"
-            changed = True
-        break
-if not changed:
-    raise SystemExit(0)
-fd, tmp = tempfile.mkstemp(prefix=".fix-env.", dir=os.path.dirname(path), text=True)
-try:
-    with os.fdopen(fd, "w", encoding="utf-8") as f:
-        f.writelines(lines)
-        f.flush()
-        os.fsync(f.fileno())
-    os.chmod(tmp, 0o600)
-    os.replace(tmp, path)
-except:
-    os.unlink(tmp, missing_ok=True)
-    raise
-PY
-}
-
 save_lkg() {
-  local checkout_dir=$1 commit=$2
+  local commit=$1
   install -d -m 0700 "$lkg_dir"
-  git -C "$checkout_dir" show "$commit:fleet.json" >"$lkg_dir/fleet.json" 2>/dev/null || true
   python3 - "$lkg_dir/metadata.json" "$installed_config_repo" "$commit" "$installed_controller" <<'PY' 2>/dev/null || true
 import json, os, sys, tempfile
 
@@ -360,9 +295,6 @@ print(json.dumps({
 PY
 }
 
-release_lock() { flock -u 9 2>/dev/null || true; }
-acquire_lock() { flock -n 9 2>/dev/null || die "cannot re-acquire installer lock"; }
-
 # --- Health (with rendered env) ---
 
 run_health_check() {
@@ -373,7 +305,8 @@ run_health_check() {
     [[ ! -f "$rendered_env" ]] || . "$rendered_env"
     set +a
     python3 "$repo_root/scripts/health.py" local --output "$output" 2>/dev/null
-  ) && python3 -c "import json; print(json.load(open('$output'))['status'])" 2>/dev/null || echo "unknown"
+  ) || true
+  python3 -c "import json; print(json.load(open('$output'))['status'])" 2>/dev/null || echo "unknown"
 }
 
 # --- Main ---
@@ -439,33 +372,22 @@ if [[ "$desired_commit" == "$installed_config_ref" ]]; then
   # Run drift check using the fetched local checkout
   local_pinned=$temp_dir/config-repo
   if [[ -d "$local_pinned/.git" ]]; then
-    release_lock
-    if "$installer" --check \
+    if CI_FLEET_INSTALLER_LOCK_FD=9 "$installer" --check \
       --config-repo "$local_pinned" \
+      --config-identity "$installed_config_repo" \
       --ref "$installed_config_ref" \
       --controller "$installed_controller" 2>"$temp_dir/drift_err"; then
-      acquire_lock
       note "CONVERGED controller=${installed_controller} config_ref=${installed_config_ref}"
       save_reconcile_state 'converged' "$desired_commit" "$installed_config_ref" 'healthy' 'no change, converged'
       exit 0
     fi
-    acquire_lock
   fi
-  # Same commit + drift = check controller health
-  # If controller is unhealthy, reconcile; otherwise converge with drift note
+  # Same commit + drift is still drift: check-only reports it, normal mode repairs it.
   if [[ "$mode" == check-only ]]; then
     save_reconcile_state 'drift' "$desired_commit" "$installed_config_ref" 'drift' 'internal drift detected'
     exit 3
   fi
-  # Full mode: run health check to decide if reconciliation is needed
-  controller_running=$(docker inspect --format '{{.State.Status}}' "ci-fleet-controller-1" 2>/dev/null || echo "missing")
-  if [[ "$controller_running" != "running" ]]; then
-    note "DRIFT with unhealthy controller, falling through to reconcile"
-  else
-    note "CONVERGED controller=${installed_controller} config_ref=${installed_config_ref}"
-    save_reconcile_state 'converged' "$desired_commit" "$installed_config_ref" 'drift' 'no commit change; internal drift tracked by drift timer'
-    exit 0
-  fi
+  note "DRIFT falling through to reconcile"
 fi
 
 # New commit or drift — validate and reconcile
@@ -475,7 +397,7 @@ if ! validate_config "$fetch_dir" "$desired_commit"; then
   note "INVALID_CONFIG commit=${desired_commit}"
   save_reconcile_state 'invalid' "$desired_commit" "$installed_config_ref" 'healthy' "config rejected at ${desired_commit}"
   if [[ "$installed_config_ref" != "$desired_commit" ]]; then
-    save_lkg "$fetch_dir" "$installed_config_ref"
+    save_lkg "$installed_config_ref"
   fi
   exit 3
 fi
@@ -495,35 +417,25 @@ if [[ "$no_op" == true ]]; then
 fi
 
 # Save LKG before reconciling
-save_lkg "$fetch_dir" "$installed_config_ref"
+save_lkg "$installed_config_ref"
 
 # Create a pinned local checkout for the installer.
-# Pass the durable config_repo name so install-state.json records
-# the correct identity; the installer uses the local checkout path
-# for fleet.json resolution.
+# Keep the durable repository identity while the installer reads the fetched checkout.
 pinned_dir=$temp_dir/pinned-config
 cp -a "$fetch_dir" "$pinned_dir"
 git -C "$pinned_dir" checkout -q "$desired_commit"
 
 # Reconcile
 note "RECONCILING controller=${installed_controller} config_ref=${desired_commit}"
-release_lock
-if "$installer" --upgrade \
+if CI_FLEET_INSTALLER_LOCK_FD=9 "$installer" --upgrade \
   --config-repo "$pinned_dir" \
+  --config-identity "$installed_config_repo" \
   --ref "$desired_commit" \
   --controller "$installed_controller" 2>"$temp_dir/upgrade_err"; then
-  acquire_lock
   note "RECONCILED controller=${installed_controller} config_ref=${desired_commit}"
 
-  # Fix config_repository in state file AND rendered env to the durable name
-  fix_state_config_repo "$installed_config_repo"
-  fix_rendered_env_config_repo "$installed_config_repo"
-
-  # Re-enable reconcile timer (may have been disabled during local-checkout upgrade)
-  systemctl enable --now ci-fleet-reconcile.timer >/dev/null 2>&1 || true
-
   # Save new LKG
-  save_lkg "$fetch_dir" "$desired_commit"
+  save_lkg "$desired_commit"
 
   # Run health check
   health_status=$(run_health_check "$temp_dir/health.json")
@@ -532,7 +444,6 @@ if "$installer" --upgrade \
   note "RECONCILE_OK controller=${installed_controller} desired=${desired_commit} applied=${desired_commit} health=${health_status}"
   exit 0
 else
-  acquire_lock
   upg_err=$(<"$temp_dir/upgrade_err")
   note "RECONCILE_FAILED error=${upg_err:-unknown}"
 
diff --git a/scripts/test-install-worker-controller.sh b/scripts/test-install-worker-controller.sh
index a6d7e352..539297f9 100755
--- a/scripts/test-install-worker-controller.sh
+++ b/scripts/test-install-worker-controller.sh
@@ -140,6 +140,9 @@ cat >"$fake_bin/systemctl" <<'EOF'
 if [[ "${1:-}" == enable && "${2:-}" == --now && ! -f "${CI_FLEET_ROOT_PREFIX:-}/var/lib/ci-fleet/install-state.json" ]]; then
   exit 98
 fi
+if [[ -n "${FAKE_FAIL_TIMER_ENABLE:-}" && "${1:-}" == enable && "${2:-}" == --now && "$*" == *ci-fleet-reconcile.timer* ]]; then
+  exit 97
+fi
 if [[ -n "${FAKE_DISABLED_TIMER:-}" && ( "${1:-}" == is-enabled || "${1:-}" == is-active ) && $# == 3 && "${3:-}" == "$FAKE_DISABLED_TIMER" ]]; then
   exit 1
 fi
@@ -442,6 +445,21 @@ relative=$(cd "$tmp" && expect_success "$installer" --install --config-repo conf
 grep -Fq 'NO_CHANGE' <<<"$relative" || fail 'relative configuration path was not normalized before drift comparison'
 grep -Fq "CI_FLEET_CONFIG_REPOSITORY=$config_repo" "$root/etc/ci-fleet/ci-fleet.env" || fail 'rendered configuration path is not absolute'
 
+remote_args=(--config-repo "$config_repo" --config-identity fixture-org/fleet-config --controller example-ci-01 --ref "$ref_one")
+export FAKE_FAIL_TIMER_ENABLE=1
+expect_command_failure "$installer" --install "${remote_args[@]}"
+unset FAKE_FAIL_TIMER_ENABLE
+grep -Fq "CI_FLEET_CONFIG_REPOSITORY=$config_repo" "$root/etc/ci-fleet/ci-fleet.env" || fail 'timer activation failure did not restore the local identity'
+expect_success "$installer" --install "${remote_args[@]}" >/dev/null
+grep -Fq 'CI_FLEET_CONFIG_REPOSITORY=fixture-org/fleet-config' "$root/etc/ci-fleet/ci-fleet.env" || fail 'local checkout did not retain its durable repository identity'
+grep -Fq '"config_repository": "fixture-org/fleet-config"' "$install_state" || fail 'install state did not retain the durable repository identity'
+exec 9>"$root/run/ci-fleet-installer.lock"
+flock -n 9 || fail 'fixture could not acquire installer lock'
+expect_success env CI_FLEET_INSTALLER_LOCK_FD=9 "$installer" --check "${remote_args[@]}" >/dev/null
+flock -u 9
+exec 9>&-
+expect_success "$installer" --install "${base_args[@]}" --ref "$ref_one" >/dev/null
+
 printf '\n' >>"$root/etc/ci-fleet/ci-fleet.env"
 printf '\n# drift\n' >>"$root/etc/systemd/system/ci-fleet-health.timer"
 expect_failure 'DRIFT rendered_environment' "$installer" --check "${base_args[@]}" --ref "$ref_one"
diff --git a/scripts/test_health.py b/scripts/test_health.py
index 3dd60ddb..184f3c6a 100644
--- a/scripts/test_health.py
+++ b/scripts/test_health.py
@@ -146,6 +146,17 @@ def run(args):
                 self.assertEqual((snapshot["load_per_cpu"], snapshot["swap_used_percent"]), (3.0, 50))
                 self.assertEqual(set(snapshot["services"]), {"cleanup", "drift"})
                 self.assertEqual(set(snapshot["timers"]), {"health", "cleanup", "drift"})
+                remote = health.collect_snapshot(
+                    {
+                        "CI_FLEET_CONTROLLER_STATE": "disabled",
+                        "CI_FLEET_CONFIG_REPOSITORY": "example/config",
+                        "CI_FLEET_HEALTH_BOOTSTRAP": "1",
+                    },
+                    root=root,
+                    run=run,
+                )
+                self.assertEqual(set(remote["services"]), {"cleanup", "drift", "reconcile"})
+                self.assertEqual(set(remote["timers"]), {"health", "cleanup", "drift", "reconcile"})
                 (root / "etc").mkdir()
                 (root / "etc/debian_version").write_text("13\n")
                 debian = health.collect_snapshot({"CI_FLEET_CONTROLLER_STATE": "disabled", "CI_FLEET_HEALTH_BOOTSTRAP": "1"}, root=root, run=run)
diff --git a/scripts/test_remote_reconcile.py b/scripts/test_remote_reconcile.py
index 43e6d262..02e388f3 100644
--- a/scripts/test_remote_reconcile.py
+++ b/scripts/test_remote_reconcile.py
@@ -244,6 +244,44 @@ def test_installer_references_units(self):
         self.assertIn("ci-fleet-reconcile.service", content)
         self.assertIn("ci-fleet-reconcile.timer", content)
 
+    def test_remote_installer_calls_keep_identity_and_lock(self):
+        """Remote check, upgrade, and rollback keep one locked transaction."""
+        reconcile = RECONCILE_SCRIPT.read_text()
+        installer = INSTALLER.read_text()
+        self.assertNotIn("release_lock", reconcile)
+        self.assertEqual(reconcile.count("CI_FLEET_INSTALLER_LOCK_FD=9"), 3)
+        self.assertEqual(reconcile.count("--config-identity"), 3)
+        self.assertIn("--config-identity)", installer)
+        self.assertIn("CI_FLEET_INSTALLER_LOCK_FD", installer)
+        self.assertIn("/proc/self/fd/9", installer)
+
+    def test_same_commit_drift_is_reconciled(self):
+        """Any failed same-commit check falls through to repair."""
+        content = RECONCILE_SCRIPT.read_text()
+        same_commit = content.split('if [[ "$desired_commit" == "$installed_config_ref" ]]', 1)[1]
+        same_commit = same_commit.split("# New commit or drift", 1)[0]
+        self.assertNotIn("controller_running", same_commit)
+        self.assertNotIn("internal drift tracked by drift timer", same_commit)
+
+    def test_timer_restore_failure_is_not_ignored(self):
+        """A remote activation cannot report success without its timer."""
+        content = INSTALLER.read_text()
+        remote_timer = content.split('if [[ "$config_identity" == *"/"*', 1)[1]
+        remote_timer = remote_timer.split("else", 1)[0]
+        self.assertNotIn("|| true", remote_timer)
+
+    def test_lkg_does_not_depend_on_an_unsaved_fleet_snapshot(self):
+        """Rollback uses the authenticated exact-ref checkout as its source."""
+        content = RECONCILE_SCRIPT.read_text()
+        apply_lkg = content.split("apply_lkg()", 1)[1].split("save_lkg()", 1)[0]
+        self.assertNotIn("LKG fleet.json missing", apply_lkg)
+
+    def test_health_report_is_parsed_for_warning_and_failure_results(self):
+        """Health severity does not discard the report it just wrote."""
+        content = RECONCILE_SCRIPT.read_text()
+        health = content.split("run_health_check()", 1)[1].split("# --- Main ---", 1)[0]
+        self.assertNotIn(") && python3", health)
+
 
 if __name__ == "__main__":
     unittest.main()

From e32d35fa0b1e7e80b13ebe88ae62466575773d41 Mon Sep 17 00:00:00 2001
From: Nickfost <1572453+Nickfost@users.noreply.github.com>
Date: Tue, 28 Jul 2026 01:07:03 -0500
Subject: [PATCH 2/4] fix: preserve optional reconcile compatibility

---
 scripts/install-worker-controller.sh      | 4 ++--
 scripts/test-install-worker-controller.sh | 5 +++--
 scripts/test_remote_reconcile.py          | 2 ++
 3 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/scripts/install-worker-controller.sh b/scripts/install-worker-controller.sh
index abaa65d9..29d8a692 100755
--- a/scripts/install-worker-controller.sh
+++ b/scripts/install-worker-controller.sh
@@ -112,7 +112,7 @@ state_file=$state_root/install-state.json
 health_report=$state_root/health/latest.json
 checkpoints_dir=$state_root/checkpoints
 systemd_dir=$(root_path /etc/systemd/system)
-lock_file=$(root_path /run/ci-fleet-installer.lock)
+lock_file=${CI_FLEET_INSTALLER_LOCK:-$(root_path /run/ci-fleet-installer.lock)}
 controller_container=ci-fleet-controller-1
 unit_names=(
   ci-fleet-health.service ci-fleet-health.timer
@@ -835,7 +835,7 @@ PY
       # identified as an OWNER/REPO (not a local checkout path)
       if [[ "$config_identity" == *"/"* && "$config_identity" != "/"* ]]; then
         systemctl enable --now "$opt_timer" >/dev/null
-      else
+      elif [[ -f "$systemd_dir/$opt_timer" ]]; then
         # Local checkout path — disable and stop any previously enabled timer
         systemctl disable --now "$opt_timer" >/dev/null
       fi
diff --git a/scripts/test-install-worker-controller.sh b/scripts/test-install-worker-controller.sh
index 539297f9..e9a87061 100755
--- a/scripts/test-install-worker-controller.sh
+++ b/scripts/test-install-worker-controller.sh
@@ -453,9 +453,10 @@ grep -Fq "CI_FLEET_CONFIG_REPOSITORY=$config_repo" "$root/etc/ci-fleet/ci-fleet.
 expect_success "$installer" --install "${remote_args[@]}" >/dev/null
 grep -Fq 'CI_FLEET_CONFIG_REPOSITORY=fixture-org/fleet-config' "$root/etc/ci-fleet/ci-fleet.env" || fail 'local checkout did not retain its durable repository identity'
 grep -Fq '"config_repository": "fixture-org/fleet-config"' "$install_state" || fail 'install state did not retain the durable repository identity'
-exec 9>"$root/run/ci-fleet-installer.lock"
+custom_lock=$root/run/custom-installer.lock
+exec 9>"$custom_lock"
 flock -n 9 || fail 'fixture could not acquire installer lock'
-expect_success env CI_FLEET_INSTALLER_LOCK_FD=9 "$installer" --check "${remote_args[@]}" >/dev/null
+expect_success env CI_FLEET_INSTALLER_LOCK="$custom_lock" CI_FLEET_INSTALLER_LOCK_FD=9 "$installer" --check "${remote_args[@]}" >/dev/null
 flock -u 9
 exec 9>&-
 expect_success "$installer" --install "${base_args[@]}" --ref "$ref_one" >/dev/null
diff --git a/scripts/test_remote_reconcile.py b/scripts/test_remote_reconcile.py
index 02e388f3..2afb82c7 100644
--- a/scripts/test_remote_reconcile.py
+++ b/scripts/test_remote_reconcile.py
@@ -254,6 +254,8 @@ def test_remote_installer_calls_keep_identity_and_lock(self):
         self.assertIn("--config-identity)", installer)
         self.assertIn("CI_FLEET_INSTALLER_LOCK_FD", installer)
         self.assertIn("/proc/self/fd/9", installer)
+        self.assertIn("lock_file=${CI_FLEET_INSTALLER_LOCK:-", installer)
+        self.assertIn('elif [[ -f "$systemd_dir/$opt_timer" ]]; then', installer)
 
     def test_same_commit_drift_is_reconciled(self):
         """Any failed same-commit check falls through to repair."""

From 5c6eca8f3e7c5f7e2d496ac0bed4fe6cfcbe7ad7 Mon Sep 17 00:00:00 2001
From: Nickfost <1572453+Nickfost@users.noreply.github.com>
Date: Tue, 28 Jul 2026 01:31:26 -0500
Subject: [PATCH 3/4] fix: report reconciliation failures as unhealthy

---
 scripts/health.py      | 32 ++++++++++++++++++++++++++++++++
 scripts/test_health.py | 15 +++++++++++++++
 2 files changed, 47 insertions(+)

diff --git a/scripts/health.py b/scripts/health.py
index 1bc4825c..70ee54c4 100644
--- a/scripts/health.py
+++ b/scripts/health.py
@@ -151,6 +151,16 @@ def add(check_id: str, severity: str, **details: Any) -> None:
     add("clock", "ok" if snapshot["clock_synchronized"] else "warning")
     backup = snapshot["backup"]
     add("backup", "warning" if backup == "failed" else "ok", state=backup)
+    reconciliation = snapshot.get("reconciliation")
+    if reconciliation:
+        add(
+            "reconciliation",
+            "ok" if reconciliation["status"] in {"converged", "bootstrap"} else "critical",
+            state=reconciliation["status"],
+            desired_commit=reconciliation["desired_commit"],
+            applied_commit=reconciliation["applied_commit"],
+            reported_health=reconciliation["health"],
+        )
 
     rank = max(({"ok": 0, "warning": 1, "critical": 2}[check["status"]] for check in checks), default=0)
     overall = ("healthy", "warning", "unhealthy")[rank]
@@ -294,6 +304,24 @@ def _backup_state(values: dict[str, str], run: Runner) -> str:
     return "ok" if run([str(path)]).returncode == 0 else "failed"
 
 
+def _reconcile_state(path: Path) -> dict[str, str]:
+    try:
+        value = json.loads(path.read_text())
+    except (OSError, json.JSONDecodeError):
+        return {"status": "missing", "desired_commit": "", "applied_commit": "", "health": ""}
+    if not isinstance(value, dict):
+        return {"status": "invalid", "desired_commit": "", "applied_commit": "", "health": ""}
+    status = value.get("status", "")
+    if status not in {"converged", "drift", "invalid", "pending", "rolled_back", "failed"}:
+        status = "invalid"
+    commits = [value.get(name, "") for name in ("desired_commit", "applied_commit")]
+    commits = [commit if isinstance(commit, str) and (not commit or re.fullmatch(r"[0-9a-f]{40}", commit)) else "invalid" for commit in commits]
+    reported_health = value.get("health", "")
+    if reported_health not in {"", "healthy", "warning", "unhealthy", "maintenance", "drift", "unknown"}:
+        reported_health = "invalid"
+    return {"status": status, "desired_commit": commits[0], "applied_commit": commits[1], "health": reported_health}
+
+
 def collect_snapshot(values: dict[str, str], *, root: Path = Path("/"), run: Runner = _run) -> dict[str, Any]:
     docker_root = values.get("CI_FLEET_DOCKER_ROOT", "/var/lib/docker")
     available, swap = _memory(root)
@@ -316,6 +344,9 @@ def collect_snapshot(values: dict[str, str], *, root: Path = Path("/"), run: Run
     configured = {"min": int(values.get("CI_FLEET_MIN_RUNNERS", 0)), "max": int(values.get("CI_FLEET_MAX_RUNNERS", 0))}
     timer_ages = {"health": 900, "cleanup": 172800, "drift": 3600}
     remote_config = bool(re.fullmatch(r"[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+", values.get("CI_FLEET_CONFIG_REPOSITORY", "")))
+    reconciliation = _reconcile_state(root / "var/lib/ci-fleet/reconcile/state.json") if remote_config else None
+    if reconciliation and values.get("CI_FLEET_HEALTH_BOOTSTRAP") == "1":
+        reconciliation["status"] = "bootstrap"
     if remote_config:
         timer_ages["reconcile"] = 900
     timers = {name: _unit_state(run, f"ci-fleet-{name}.timer", timer=True, max_age_seconds=age) for name, age in timer_ages.items()}
@@ -357,6 +388,7 @@ def collect_snapshot(values: dict[str, str], *, root: Path = Path("/"), run: Run
         "failed_packages": debian and bool(run(["dpkg", "--audit"]).stdout.strip()),
         "clock_synchronized": run(["timedatectl", "show", "--property=NTPSynchronized", "--value"]).stdout.strip() == "yes",
         "backup": _backup_state(values, run),
+        "reconciliation": reconciliation,
     }
 
 
diff --git a/scripts/test_health.py b/scripts/test_health.py
index 184f3c6a..c1f968cf 100644
--- a/scripts/test_health.py
+++ b/scripts/test_health.py
@@ -93,6 +93,18 @@ def test_drained_host_is_maintenance_not_unhealthy(self) -> None:
         report = health.evaluate(snapshot, health.Thresholds())
         self.assertEqual((report["status"], report["exit_code"]), ("maintenance", 0))
 
+    def test_remote_reconciliation_failure_is_unhealthy(self) -> None:
+        snapshot = healthy_snapshot()
+        snapshot["reconciliation"] = {
+            "status": "rolled_back",
+            "desired_commit": "1" * 40,
+            "applied_commit": "2" * 40,
+            "health": "healthy",
+        }
+        report = health.evaluate(snapshot, health.Thresholds())
+        self.assertEqual((report["status"], report["exit_code"]), ("unhealthy", 2))
+        self.assertEqual(next(check for check in report["checks"] if check["id"] == "reconciliation")["status"], "critical")
+
     def test_external_heartbeats_detect_missing_and_stale_active_hosts(self) -> None:
         controllers = {
             "fresh": {"state": "active", "lifecycle": "stable"},
@@ -146,6 +158,8 @@ def run(args):
                 self.assertEqual((snapshot["load_per_cpu"], snapshot["swap_used_percent"]), (3.0, 50))
                 self.assertEqual(set(snapshot["services"]), {"cleanup", "drift"})
                 self.assertEqual(set(snapshot["timers"]), {"health", "cleanup", "drift"})
+                (root / "var/lib/ci-fleet/reconcile").mkdir(parents=True)
+                (root / "var/lib/ci-fleet/reconcile/state.json").write_text('{"status":"rolled_back","desired_commit":"","applied_commit":"","health":"healthy"}\n')
                 remote = health.collect_snapshot(
                     {
                         "CI_FLEET_CONTROLLER_STATE": "disabled",
@@ -157,6 +171,7 @@ def run(args):
                 )
                 self.assertEqual(set(remote["services"]), {"cleanup", "drift", "reconcile"})
                 self.assertEqual(set(remote["timers"]), {"health", "cleanup", "drift", "reconcile"})
+                self.assertEqual(remote["reconciliation"]["status"], "bootstrap")
                 (root / "etc").mkdir()
                 (root / "etc/debian_version").write_text("13\n")
                 debian = health.collect_snapshot({"CI_FLEET_CONTROLLER_STATE": "disabled", "CI_FLEET_HEALTH_BOOTSTRAP": "1"}, root=root, run=run)

From 07f54139520968f98ddebbc27abafb7345f62096 Mon Sep 17 00:00:00 2001
From: Nickfost <1572453+Nickfost@users.noreply.github.com>
Date: Tue, 28 Jul 2026 01:49:26 -0500
Subject: [PATCH 4/4] fix: distinguish reconciliation in progress

---
 scripts/health.py           | 11 ++++++++---
 scripts/remote-reconcile.sh |  2 ++
 scripts/test_health.py      | 13 +++++++++++++
 3 files changed, 23 insertions(+), 3 deletions(-)

diff --git a/scripts/health.py b/scripts/health.py
index 70ee54c4..bbc39a8b 100644
--- a/scripts/health.py
+++ b/scripts/health.py
@@ -153,9 +153,14 @@ def add(check_id: str, severity: str, **details: Any) -> None:
     add("backup", "warning" if backup == "failed" else "ok", state=backup)
     reconciliation = snapshot.get("reconciliation")
     if reconciliation:
+        reconciliation_severity = (
+            "ok" if reconciliation["status"] in {"converged", "bootstrap"}
+            else "warning" if reconciliation["status"] in {"missing", "pending", "reconciling"}
+            else "critical"
+        )
         add(
             "reconciliation",
-            "ok" if reconciliation["status"] in {"converged", "bootstrap"} else "critical",
+            reconciliation_severity,
             state=reconciliation["status"],
             desired_commit=reconciliation["desired_commit"],
             applied_commit=reconciliation["applied_commit"],
@@ -312,12 +317,12 @@ def _reconcile_state(path: Path) -> dict[str, str]:
     if not isinstance(value, dict):
         return {"status": "invalid", "desired_commit": "", "applied_commit": "", "health": ""}
     status = value.get("status", "")
-    if status not in {"converged", "drift", "invalid", "pending", "rolled_back", "failed"}:
+    if not isinstance(status, str) or status not in {"converged", "drift", "invalid", "pending", "reconciling", "rolled_back", "failed"}:
         status = "invalid"
     commits = [value.get(name, "") for name in ("desired_commit", "applied_commit")]
     commits = [commit if isinstance(commit, str) and (not commit or re.fullmatch(r"[0-9a-f]{40}", commit)) else "invalid" for commit in commits]
     reported_health = value.get("health", "")
-    if reported_health not in {"", "healthy", "warning", "unhealthy", "maintenance", "drift", "unknown"}:
+    if not isinstance(reported_health, str) or reported_health not in {"", "healthy", "warning", "unhealthy", "maintenance", "drift", "unknown"}:
         reported_health = "invalid"
     return {"status": status, "desired_commit": commits[0], "applied_commit": commits[1], "health": reported_health}
 
diff --git a/scripts/remote-reconcile.sh b/scripts/remote-reconcile.sh
index 468d7e54..dc28a68f 100755
--- a/scripts/remote-reconcile.sh
+++ b/scripts/remote-reconcile.sh
@@ -427,6 +427,7 @@ git -C "$pinned_dir" checkout -q "$desired_commit"
 
 # Reconcile
 note "RECONCILING controller=${installed_controller} config_ref=${desired_commit}"
+save_reconcile_state 'reconciling' "$desired_commit" "$installed_config_ref" 'unknown' "reconciling to ${desired_commit}"
 if CI_FLEET_INSTALLER_LOCK_FD=9 "$installer" --upgrade \
   --config-repo "$pinned_dir" \
   --config-identity "$installed_config_repo" \
@@ -438,6 +439,7 @@ if CI_FLEET_INSTALLER_LOCK_FD=9 "$installer" --upgrade \
   save_lkg "$desired_commit"
 
   # Run health check
+  save_reconcile_state 'converged' "$desired_commit" "$desired_commit" 'unknown' "reconciled to ${desired_commit}; checking health"
   health_status=$(run_health_check "$temp_dir/health.json")
 
   save_reconcile_state 'converged' "$desired_commit" "$desired_commit" "$health_status" "reconciled to ${desired_commit}"
diff --git a/scripts/test_health.py b/scripts/test_health.py
index c1f968cf..4f385c1b 100644
--- a/scripts/test_health.py
+++ b/scripts/test_health.py
@@ -105,6 +105,19 @@ def test_remote_reconciliation_failure_is_unhealthy(self) -> None:
         self.assertEqual((report["status"], report["exit_code"]), ("unhealthy", 2))
         self.assertEqual(next(check for check in report["checks"] if check["id"] == "reconciliation")["status"], "critical")
 
+        snapshot["reconciliation"] = {"status": "missing", "desired_commit": "", "applied_commit": "", "health": ""}
+        report = health.evaluate(snapshot, health.Thresholds())
+        self.assertEqual((report["status"], report["exit_code"]), ("warning", 1))
+
+    def test_malformed_reconciliation_state_is_observable(self) -> None:
+        with tempfile.TemporaryDirectory() as directory:
+            path = Path(directory) / "state.json"
+            path.write_text('{"status":[],"health":{},"desired_commit":[],"applied_commit":null}\n')
+            self.assertEqual(
+                health._reconcile_state(path),
+                {"status": "invalid", "desired_commit": "invalid", "applied_commit": "invalid", "health": "invalid"},
+            )
+
     def test_external_heartbeats_detect_missing_and_stale_active_hosts(self) -> None:
         controllers = {
             "fresh": {"state": "active", "lifecycle": "stable"},