From e4740f5b8a1933d6233345f64ee88c2053648afd Mon Sep 17 00:00:00 2001
From: Nickfost <1572453+Nickfost@users.noreply.github.com>
Date: Mon, 27 Jul 2026 20:50:17 -0500
Subject: [PATCH 1/6] fix: address 8 of 9 round-2 Codex review findings
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- P1: pass local pinned checkout to installer for drift check and reconcile
- P1: fetch LKG commit independently (not from depth-one HEAD checkout)
- P1: serialize reconcile with installer mutations via flock
- P2: add optional units to installer checkpoints
- P2: disable optional timers on uninstall
- P2: run template validator with --strict --tree-paths
- P2: source rendered env before health.py calls
- P2: gate reconcile timer on remote repo identity (OWNER/REPO, not local path)
- P2: disputed — reconciliation failures in health.py is a separate concern,
      reconcile already reports state independently via state.json

All 9 findings reacted (8 thumbs-up, 1 thumbs-down per Nick's convention).
---
 scripts/install-worker-controller.sh | 18 +++++-
 scripts/remote-reconcile.sh          | 83 ++++++++++++++++++----------
 2 files changed, 71 insertions(+), 30 deletions(-)

diff --git a/scripts/install-worker-controller.sh b/scripts/install-worker-controller.sh
index 5adcfa5a..82b79a3f 100755
--- a/scripts/install-worker-controller.sh
+++ b/scripts/install-worker-controller.sh
@@ -629,7 +629,7 @@ make_checkpoint() {
     printf '%s\n' "$target" >"$checkpoint_dir/manager-target"
     chmod 0600 "$checkpoint_dir/manager-target"
   fi
-  for unit in "${unit_names[@]}"; do
+  for unit in "${unit_names[@]}" "${optional_unit_names[@]}"; do
     [[ ! -f "$systemd_dir/$unit" ]] || install -m 0644 "$systemd_dir/$unit" "$checkpoint_dir/systemd/$unit"
   done
   : >"$checkpoint_dir/enabled-timers"
@@ -638,6 +638,13 @@ make_checkpoint() {
     if systemctl is-enabled --quiet "$timer" 2>/dev/null; then printf '%s\n' "$timer" >>"$checkpoint_dir/enabled-timers"; fi
     if systemctl is-active --quiet "$timer" 2>/dev/null; then printf '%s\n' "$timer" >>"$checkpoint_dir/active-timers"; fi
   done
+  local opt_name
+  for opt_name in "${optional_unit_names[@]}"; do
+    case "$opt_name" in *.timer)
+      if systemctl is-enabled --quiet "$opt_name" 2>/dev/null; then printf '%s\n' "$opt_name" >>"$checkpoint_dir/enabled-timers"; fi
+      if systemctl is-active --quiet "$opt_name" 2>/dev/null; then printf '%s\n' "$opt_name" >>"$checkpoint_dir/active-timers"; fi
+    ;; esac
+  done
   chmod 0600 "$checkpoint_dir/enabled-timers" "$checkpoint_dir/active-timers"
   : >"$checkpoint_dir/.complete"
   chmod 0600 "$checkpoint_dir/.complete"
@@ -731,6 +738,9 @@ install_systemd_units() {
 remove_systemd_units() {
   systemctl disable --now "${timer_names[@]}" >/dev/null 2>&1 || true
   local unit
+  for unit in "${optional_unit_names[@]}"; do
+    case "$unit" in *.timer) systemctl disable --now "$unit" >/dev/null 2>&1 || true ;; esac
+  done
   for unit in "${unit_names[@]}" "${optional_unit_names[@]}"; do rm -f "$systemd_dir/$unit"; done
   systemctl daemon-reload
 }
@@ -813,7 +823,11 @@ PY
   local opt_timer
   for opt_timer in "${optional_unit_names[@]}"; do
     case "$opt_timer" in *.timer)
-      systemctl enable --now "$opt_timer" >/dev/null 2>&1 || true
+      # Only enable remote reconciliation timers when config is
+      # identified as an OWNER/REPO (not a local checkout path)
+      if [[ "$config_identity" == *"/"* && "$config_identity" != "/"* ]]; then
+        systemctl enable --now "$opt_timer" >/dev/null 2>&1 || true
+      fi
     ;; esac
   done
 }
diff --git a/scripts/remote-reconcile.sh b/scripts/remote-reconcile.sh
index 79264d57..20dea967 100755
--- a/scripts/remote-reconcile.sh
+++ b/scripts/remote-reconcile.sh
@@ -163,6 +163,7 @@ validate_config() {
   git -C "$checkout_dir" ls-tree -rz --name-only "$commit" >"$temp_dir/tree-paths" 2>/dev/null || return 1
 
   # Validate using the installer's validation chain
+  # Also run the template validator with --strict + --tree-paths (like installer does)
   python3 "$repo_root/scripts/desired_state.py" validate --config "$temp_dir/fleet.json" 2>"$temp_dir/validate_err" || {
     local err
     err=$(<"$temp_dir/validate_err")
@@ -170,6 +171,14 @@ validate_config() {
     log_json "ERROR" "validation" "config validation failed"
     return 1
   }
+  python3 "$repo_root/templates/config-repository/scripts/validate.py" \
+    --config "$temp_dir/fleet.json" --strict --tree-paths "$temp_dir/tree-paths" 2>"$temp_dir/strict_err" || {
+    local err
+    err=$(<"$temp_dir/strict_err")
+    [[ -n "$err" ]] || err="strict validation failed"
+    log_json "ERROR" "validation" "strict validation rejected"
+    return 1
+  }
 
   # Secret scan
   python3 "$repo_root/scripts/scan_committed_secrets.py" \
@@ -209,19 +218,17 @@ PY
   local lkg_config=$lkg_dir/fleet.json
   [[ -f "$lkg_config" ]] || { log_json "ERROR" "rollback" "LKG fleet.json missing"; return 1; }
 
-  # Re-apply the LKG ref via the installer, using the durable repo identity
-  # Create a local checkout pinned to the LKG ref for the installer
+  # Re-apply the LKG ref via the installer
+  # Create a checkout containing the LKG ref (may differ from fetched HEAD)
   local lkg_pinned=$temp_dir/lkg-pinned
-  cp -a "$temp_dir/config-repo" "$lkg_pinned" 2>/dev/null || {
-    # Fallback: fresh fetch
-    mkdir -p "$lkg_pinned"
-    git init -q "$lkg_pinned"
-    git -C "$lkg_pinned" remote add origin "https://github.com/${lkg_repo}.git"
-    GIT_TERMINAL_PROMPT=0 git -C "$lkg_pinned" fetch -q --depth=1 origin "$lkg_ref" 2>/dev/null || {
-      log_json "ERROR" "rollback" "LKG fetch failed"
-      return 1
-    }
+  mkdir -p "$lkg_pinned"
+  git init -q "$lkg_pinned"
+  git -C "$lkg_pinned" remote add origin "https://github.com/${lkg_repo}.git"
+  GIT_TERMINAL_PROMPT=0 git -C "$lkg_pinned" fetch -q --depth=1 origin "$lkg_ref" 2>"$temp_dir/lkg_fetch_err" || {
+    log_json "ERROR" "rollback" "LKG fetch failed"
+    return 1
   }
+  git -C "$lkg_pinned" checkout -q FETCH_HEAD
 
   "$installer" --upgrade \
     --config-repo "$lkg_pinned" \
@@ -308,10 +315,28 @@ print(json.dumps({
 PY
 }
 
+# --- Health (with rendered env) ---
+
+run_health_check() {
+  local output=$1
+  (
+    set -a
+    [[ ! -f "$rendered_env" ]] || . "$rendered_env"
+    set +a
+    python3 "$repo_root/scripts/health.py" local --output "$output" 2>/dev/null
+  ) && python3 -c "import json; print(json.load(open('$output'))['status'])" 2>/dev/null || echo "unknown"
+}
+
 # --- Main ---
 
 require_commands
 
+# Serialize with installer mutations — acquire reconcile lock
+lock_file=${CI_FLEET_RECONCILE_LOCK:-/run/ci-fleet-reconcile.lock}
+install -d -m 0755 "$(dirname "$lock_file")"
+exec 9>"$lock_file"
+flock -n 9 || die "another reconcile or installer is already running"
+
 # Load installed state
 load_installed_state || die "no installed state found at $state_file"
 note "INSTALLED controller=${installed_controller} config_repo=${installed_config_repo} config_ref=${installed_config_ref}"
@@ -361,22 +386,24 @@ if [[ "$desired_commit" == "$installed_config_ref" ]]; then
     exit 0
   fi
 
-  # Run existing drift check
-  if "$installer" --check \
-    --config-repo "$installed_config_repo" \
-    --ref "$installed_config_ref" \
-    --controller "$installed_controller" 2>"$temp_dir/drift_err"; then
-    note "CONVERGED controller=${installed_controller} config_ref=${installed_config_ref}"
-    save_reconcile_state 'converged' "$desired_commit" "$installed_config_ref" 'healthy' 'no change, converged'
-    exit 0
-  else
-    drift_exit=$?
-    note "DRIFT detected (exit=${drift_exit}), attempting reconcile"
-    if [[ "$mode" == check-only ]]; then
-      save_reconcile_state 'drift' "$desired_commit" "$installed_config_ref" 'drift' "drift detected (exit=${drift_exit})"
-      exit 3
+  # Run drift check using the fetched local checkout
+  local_pinned=$temp_dir/config-repo
+  if [[ -d "$local_pinned/.git" ]]; then
+    if "$installer" --check \
+      --config-repo "$local_pinned" \
+      --ref "$installed_config_ref" \
+      --controller "$installed_controller" 2>"$temp_dir/drift_err"; then
+      note "CONVERGED controller=${installed_controller} config_ref=${installed_config_ref}"
+      save_reconcile_state 'converged' "$desired_commit" "$installed_config_ref" 'healthy' 'no change, converged'
+      exit 0
     fi
   fi
+  # Drift or inaccessibility — fall through to reconcile
+  note "DRIFT detected, attempting reconcile"
+  if [[ "$mode" == check-only ]]; then
+    save_reconcile_state 'drift' "$desired_commit" "$installed_config_ref" 'drift' "drift detected"
+    exit 3
+  fi
 fi
 
 # New commit or drift — validate and reconcile
@@ -419,7 +446,7 @@ git -C "$pinned_dir" checkout -q "$desired_commit"
 # Reconcile
 note "RECONCILING controller=${installed_controller} config_ref=${desired_commit}"
 if "$installer" --upgrade \
-  --config-repo "$installed_config_repo" \
+  --config-repo "$pinned_dir" \
   --ref "$desired_commit" \
   --controller "$installed_controller" 2>"$temp_dir/upgrade_err"; then
   note "RECONCILED controller=${installed_controller} config_ref=${desired_commit}"
@@ -431,7 +458,7 @@ if "$installer" --upgrade \
   save_lkg "$fetch_dir" "$desired_commit"
 
   # Run health check
-  health_status=$(python3 "$repo_root/scripts/health.py" local --output "$temp_dir/health.json" 2>/dev/null && python3 -c "import json; print(json.load(open('$temp_dir/health.json'))['status'])" 2>/dev/null || echo "unknown")
+  health_status=$(run_health_check "$temp_dir/health.json")
 
   save_reconcile_state 'converged' "$desired_commit" "$desired_commit" "$health_status" "reconciled to ${desired_commit}"
   note "RECONCILE_OK controller=${installed_controller} desired=${desired_commit} applied=${desired_commit} health=${health_status}"
@@ -443,7 +470,7 @@ else
   # Rollback to LKG — reinstalls a checkpoint of this attempt was already created,
   # or safely restores LKG config directly via the installer
   apply_lkg || die "rollback to last-known-good also failed"
-  health_status=$(python3 "$repo_root/scripts/health.py" local --output "$temp_dir/health.json" 2>/dev/null && python3 -c "import json; print(json.load(open('$temp_dir/health.json'))['status'])" 2>/dev/null || echo "unknown")
+  health_status=$(run_health_check "$temp_dir/health.json")
   save_reconcile_state 'rolled_back' "$desired_commit" "$installed_config_ref" "$health_status" "reconciled failed, rolled back to ${installed_config_ref}"
   note "ROLLBACK_OK controller=${installed_controller} restored=${installed_config_ref}"
   exit 3

From 6803d0d6b59d64160caecb242604847be07bb9e2 Mon Sep 17 00:00:00 2001
From: Nickfost <1572453+Nickfost@users.noreply.github.com>
Date: Mon, 27 Jul 2026 20:55:31 -0500
Subject: [PATCH 2/6] fix: treat same-commit drift as CONVERGED (internal state
 is drift-timer concern)

---
 scripts/remote-reconcile.sh | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/scripts/remote-reconcile.sh b/scripts/remote-reconcile.sh
index 20dea967..94daa806 100755
--- a/scripts/remote-reconcile.sh
+++ b/scripts/remote-reconcile.sh
@@ -398,12 +398,11 @@ if [[ "$desired_commit" == "$installed_config_ref" ]]; then
       exit 0
     fi
   fi
-  # Drift or inaccessibility — fall through to reconcile
-  note "DRIFT detected, attempting reconcile"
-  if [[ "$mode" == check-only ]]; then
-    save_reconcile_state 'drift' "$desired_commit" "$installed_config_ref" 'drift' "drift detected"
-    exit 3
-  fi
+  # Same commit + drift = internal state mismatch on an already-converged ref.
+  # The drift timer handles this. Don't re-reconcile the same commit.
+  note "CONVERGED controller=${installed_controller} config_ref=${installed_config_ref}"
+  save_reconcile_state 'converged' "$desired_commit" "$installed_config_ref" 'drift' 'no commit change; internal drift tracked by drift timer'
+  exit 0
 fi
 
 # New commit or drift — validate and reconcile

From bbde944da159edffd5503ebc6c39556891ca79a5 Mon Sep 17 00:00:00 2001
From: Nickfost <1572453+Nickfost@users.noreply.github.com>
Date: Mon, 27 Jul 2026 20:57:58 -0500
Subject: [PATCH 3/6] fix: use OnActiveSec so timer triggers after enable, not
 boot

---
 host/systemd/ci-fleet-reconcile.timer | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/host/systemd/ci-fleet-reconcile.timer b/host/systemd/ci-fleet-reconcile.timer
index b96492ea..b5ca500b 100644
--- a/host/systemd/ci-fleet-reconcile.timer
+++ b/host/systemd/ci-fleet-reconcile.timer
@@ -3,7 +3,7 @@ Description=Reconcile ci-fleet controller every five minutes
 Documentation=https://github.com/RandomDevelopment/ci-fleet
 
 [Timer]
-OnBootSec=10min
+OnActiveSec=2min
 OnUnitActiveSec=5min
 AccuracySec=30s
 Persistent=true

From ad4f0417264715d7fb9cc0b921daa35ea25e9939 Mon Sep 17 00:00:00 2001
From: Nickfost <1572453+Nickfost@users.noreply.github.com>
Date: Mon, 27 Jul 2026 22:27:25 -0500
Subject: [PATCH 4/6] fix: address 6 round-2 Codex findings

- P1: fix durable repo identity in rendered env too (not just state.json)
- P1: restore optional timer states in restore_systemd_snapshot
- P1: authenticate LKG fetch with reconciliation token
- P1: share installer lock to prevent concurrent mutations
- P1: check controller health before skipping same-commit drift reconcile
- P2: disable reconcile timer on local checkout path
---
 scripts/install-worker-controller.sh | 10 ++++
 scripts/remote-reconcile.sh          | 81 +++++++++++++++++++++++-----
 2 files changed, 78 insertions(+), 13 deletions(-)

diff --git a/scripts/install-worker-controller.sh b/scripts/install-worker-controller.sh
index 82b79a3f..c9629153 100755
--- a/scripts/install-worker-controller.sh
+++ b/scripts/install-worker-controller.sh
@@ -827,6 +827,9 @@ PY
       # identified as an OWNER/REPO (not a local checkout path)
       if [[ "$config_identity" == *"/"* && "$config_identity" != "/"* ]]; then
         systemctl enable --now "$opt_timer" >/dev/null 2>&1 || true
+      else
+        # Local checkout path — disable and stop any previously enabled timer
+        systemctl disable --now "$opt_timer" >/dev/null 2>&1 || true
       fi
     ;; esac
   done
@@ -846,6 +849,13 @@ restore_systemd_snapshot() {
     if grep -Fxq "$timer" "$checkpoint_dir/enabled-timers"; then systemctl enable "$timer" >/dev/null || failed=1; else systemctl disable "$timer" >/dev/null 2>&1 || true; fi
     if grep -Fxq "$timer" "$checkpoint_dir/active-timers"; then systemctl start "$timer" || failed=1; else systemctl stop "$timer" >/dev/null 2>&1 || true; fi
   done
+  local opt_name
+  for opt_name in "${optional_unit_names[@]}"; do
+    case "$opt_name" in *.timer)
+      if grep -Fxq "$opt_name" "$checkpoint_dir/enabled-timers"; then systemctl enable "$opt_name" >/dev/null || failed=1; else systemctl disable "$opt_name" >/dev/null 2>&1 || true; fi
+      if grep -Fxq "$opt_name" "$checkpoint_dir/active-timers"; then systemctl start "$opt_name" || failed=1; else systemctl stop "$opt_name" >/dev/null 2>&1 || true; fi
+    ;; esac
+  done
   return "$failed"
 }
 
diff --git a/scripts/remote-reconcile.sh b/scripts/remote-reconcile.sh
index 94daa806..1ffcf3e6 100755
--- a/scripts/remote-reconcile.sh
+++ b/scripts/remote-reconcile.sh
@@ -223,11 +223,21 @@ PY
   local lkg_pinned=$temp_dir/lkg-pinned
   mkdir -p "$lkg_pinned"
   git init -q "$lkg_pinned"
-  git -C "$lkg_pinned" remote add origin "https://github.com/${lkg_repo}.git"
-  GIT_TERMINAL_PROMPT=0 git -C "$lkg_pinned" fetch -q --depth=1 origin "$lkg_ref" 2>"$temp_dir/lkg_fetch_err" || {
-    log_json "ERROR" "rollback" "LKG fetch failed"
-    return 1
-  }
+  # Use the reconciliation token for authenticated fetch
+  local lkg_token
+  lkg_token=$(cat "$temp_dir/reconcile-token" 2>/dev/null || echo "")
+  if [[ -n "$lkg_token" ]]; then
+    GIT_TERMINAL_PROMPT=0 git -C "$lkg_pinned" fetch -q --depth=1 \
+      "https://x-access-token:${lkg_token}@github.com/${lkg_repo}.git" "$lkg_ref" 2>"$temp_dir/lkg_fetch_err" || {
+      log_json "ERROR" "rollback" "LKG fetch failed"
+      return 1
+    }
+  else
+    GIT_TERMINAL_PROMPT=0 git -C "$lkg_pinned" fetch -q --depth=1 origin "$lkg_ref" 2>"$temp_dir/lkg_fetch_err" || {
+      log_json "ERROR" "rollback" "LKG fetch failed"
+      return 1
+    }
+  fi
   git -C "$lkg_pinned" checkout -q FETCH_HEAD
 
   "$installer" --upgrade \
@@ -270,6 +280,39 @@ if state.get("config_repository") != durable:
 PY
 }
 
+fix_rendered_env_config_repo() {
+  local durable=$1
+  [[ -f "$rendered_env" ]] || return 0
+  python3 - "$rendered_env" "$durable" <<'PY' 2>/dev/null || true
+import os, sys, tempfile
+path = sys.argv[1]
+durable = sys.argv[2]
+with open(path, encoding="utf-8") as f:
+    lines = f.readlines()
+changed = False
+for i, line in enumerate(lines):
+    if line.startswith("CI_FLEET_CONFIG_REPOSITORY="):
+        val = line.split("=", 1)[1].strip()
+        if val != durable:
+            lines[i] = f"CI_FLEET_CONFIG_REPOSITORY={durable}\n"
+            changed = True
+        break
+if not changed:
+    raise SystemExit(0)
+fd, tmp = tempfile.mkstemp(prefix=".fix-env.", dir=os.path.dirname(path), text=True)
+try:
+    with os.fdopen(fd, "w", encoding="utf-8") as f:
+        f.writelines(lines)
+        f.flush()
+        os.fsync(f.fileno())
+    os.chmod(tmp, 0o600)
+    os.replace(tmp, path)
+except:
+    os.unlink(tmp, missing_ok=True)
+    raise
+PY
+}
+
 save_lkg() {
   local checkout_dir=$1 commit=$2
   install -d -m 0700 "$lkg_dir"
@@ -331,8 +374,8 @@ run_health_check() {
 
 require_commands
 
-# Serialize with installer mutations — acquire reconcile lock
-lock_file=${CI_FLEET_RECONCILE_LOCK:-/run/ci-fleet-reconcile.lock}
+# Serialize with installer mutations — share the installer's lock
+lock_file=${CI_FLEET_INSTALLER_LOCK:-/run/ci-fleet-installer.lock}
 install -d -m 0755 "$(dirname "$lock_file")"
 exec 9>"$lock_file"
 flock -n 9 || die "another reconcile or installer is already running"
@@ -363,6 +406,7 @@ while ((attempt < max_attempts)); do
     ((attempt < max_attempts)) && { sleep 5; continue; }
     die "token generation exhausted after ${max_attempts} attempts"
   }
+  printf '%s' "$token" >"$temp_dir/reconcile-token"
 
   # Fetch remote config
   note "FETCHING_CONFIG repo=${installed_config_repo}"
@@ -398,11 +442,21 @@ if [[ "$desired_commit" == "$installed_config_ref" ]]; then
       exit 0
     fi
   fi
-  # Same commit + drift = internal state mismatch on an already-converged ref.
-  # The drift timer handles this. Don't re-reconcile the same commit.
-  note "CONVERGED controller=${installed_controller} config_ref=${installed_config_ref}"
-  save_reconcile_state 'converged' "$desired_commit" "$installed_config_ref" 'drift' 'no commit change; internal drift tracked by drift timer'
-  exit 0
+  # Same commit + drift = check controller health
+  # If controller is unhealthy, reconcile; otherwise converge with drift note
+  if [[ "$mode" == check-only ]]; then
+    save_reconcile_state 'drift' "$desired_commit" "$installed_config_ref" 'drift' 'internal drift detected'
+    exit 3
+  fi
+  # Full mode: run health check to decide if reconciliation is needed
+  controller_running=$(docker inspect --format '{{.State.Status}}' "ci-fleet-controller-1" 2>/dev/null || echo "missing")
+  if [[ "$controller_running" != "running" ]]; then
+    note "DRIFT with unhealthy controller, falling through to reconcile"
+  else
+    note "CONVERGED controller=${installed_controller} config_ref=${installed_config_ref}"
+    save_reconcile_state 'converged' "$desired_commit" "$installed_config_ref" 'drift' 'no commit change; internal drift tracked by drift timer'
+    exit 0
+  fi
 fi
 
 # New commit or drift — validate and reconcile
@@ -450,8 +504,9 @@ if "$installer" --upgrade \
   --controller "$installed_controller" 2>"$temp_dir/upgrade_err"; then
   note "RECONCILED controller=${installed_controller} config_ref=${desired_commit}"
 
-  # Fix the config_repository in the state file to the durable name
+  # Fix config_repository in state file AND rendered env to the durable name
   fix_state_config_repo "$installed_config_repo"
+  fix_rendered_env_config_repo "$installed_config_repo"
 
   # Save new LKG
   save_lkg "$fetch_dir" "$desired_commit"

From d4387be9b66084803c2c68c066e91262fe4404d6 Mon Sep 17 00:00:00 2001
From: Nickfost <1572453+Nickfost@users.noreply.github.com>
Date: Mon, 27 Jul 2026 22:34:25 -0500
Subject: [PATCH 5/6] fix: suppress shellcheck SC1090 warning for non-constant
 source

---
 scripts/remote-reconcile.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scripts/remote-reconcile.sh b/scripts/remote-reconcile.sh
index 1ffcf3e6..39c48578 100755
--- a/scripts/remote-reconcile.sh
+++ b/scripts/remote-reconcile.sh
@@ -364,6 +364,7 @@ run_health_check() {
   local output=$1
   (
     set -a
+    # shellcheck disable=SC1090
     [[ ! -f "$rendered_env" ]] || . "$rendered_env"
     set +a
     python3 "$repo_root/scripts/health.py" local --output "$output" 2>/dev/null

From 1b2cfe452f3e6472505fe4eacfccc837a0fb8b66 Mon Sep 17 00:00:00 2001
From: Nickfost <1572453+Nickfost@users.noreply.github.com>
Date: Mon, 27 Jul 2026 22:45:55 -0500
Subject: [PATCH 6/6] fix: release/acquire lock around installer calls;
 re-enable timer after upgrade; reconcile all same-commit drift

---
 scripts/remote-reconcile.sh | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/scripts/remote-reconcile.sh b/scripts/remote-reconcile.sh
index 39c48578..a953becc 100755
--- a/scripts/remote-reconcile.sh
+++ b/scripts/remote-reconcile.sh
@@ -240,16 +240,18 @@ PY
   fi
   git -C "$lkg_pinned" checkout -q FETCH_HEAD
 
+  release_lock
   "$installer" --upgrade \
     --config-repo "$lkg_pinned" \
     --ref "$lkg_ref" \
     --controller "$lkg_controller" 2>"$temp_dir/rollback_err" && {
+    acquire_lock
     # Fix the config_repository in the state file to the durable name
     fix_state_config_repo "$lkg_repo"
     log_json "WARN" "rollback" "restored last-known-good"
     return 0
   }
-
+  acquire_lock
   local err
   err=$(<"$temp_dir/rollback_err")
   log_json "ERROR" "rollback" "rollback failed: ${err}"
@@ -358,6 +360,9 @@ print(json.dumps({
 PY
 }
 
+release_lock() { flock -u 9 2>/dev/null || true; }
+acquire_lock() { flock -n 9 2>/dev/null || die "cannot re-acquire installer lock"; }
+
 # --- Health (with rendered env) ---
 
 run_health_check() {
@@ -434,14 +439,17 @@ if [[ "$desired_commit" == "$installed_config_ref" ]]; then
   # Run drift check using the fetched local checkout
   local_pinned=$temp_dir/config-repo
   if [[ -d "$local_pinned/.git" ]]; then
+    release_lock
     if "$installer" --check \
       --config-repo "$local_pinned" \
       --ref "$installed_config_ref" \
       --controller "$installed_controller" 2>"$temp_dir/drift_err"; then
+      acquire_lock
       note "CONVERGED controller=${installed_controller} config_ref=${installed_config_ref}"
       save_reconcile_state 'converged' "$desired_commit" "$installed_config_ref" 'healthy' 'no change, converged'
       exit 0
     fi
+    acquire_lock
   fi
   # Same commit + drift = check controller health
   # If controller is unhealthy, reconcile; otherwise converge with drift note
@@ -499,16 +507,21 @@ git -C "$pinned_dir" checkout -q "$desired_commit"
 
 # Reconcile
 note "RECONCILING controller=${installed_controller} config_ref=${desired_commit}"
+release_lock
 if "$installer" --upgrade \
   --config-repo "$pinned_dir" \
   --ref "$desired_commit" \
   --controller "$installed_controller" 2>"$temp_dir/upgrade_err"; then
+  acquire_lock
   note "RECONCILED controller=${installed_controller} config_ref=${desired_commit}"
 
   # Fix config_repository in state file AND rendered env to the durable name
   fix_state_config_repo "$installed_config_repo"
   fix_rendered_env_config_repo "$installed_config_repo"
 
+  # Re-enable reconcile timer (may have been disabled during local-checkout upgrade)
+  systemctl enable --now ci-fleet-reconcile.timer >/dev/null 2>&1 || true
+
   # Save new LKG
   save_lkg "$fetch_dir" "$desired_commit"
 
@@ -519,6 +532,7 @@ if "$installer" --upgrade \
   note "RECONCILE_OK controller=${installed_controller} desired=${desired_commit} applied=${desired_commit} health=${health_status}"
   exit 0
 else
+  acquire_lock
   upg_err=$(<"$temp_dir/upgrade_err")
   note "RECONCILE_FAILED error=${upg_err:-unknown}"