fix: address 8 round-2 Codex findings in remote reconciliation

host/systemd/ci-fleet-reconcile.timer

@@ -3,7 +3,7 @@ Description=Reconcile ci-fleet controller every five minutes
 Documentation=https://github.com/RandomDevelopment/ci-fleet
 
 [Timer]
-OnBootSec=10min
+OnActiveSec=2min
 OnUnitActiveSec=5min
 AccuracySec=30s
 Persistent=true

scripts/install-worker-controller.sh

@@ -629,7 +629,7 @@ make_checkpoint() {
     printf '%s\n' "$target" >"$checkpoint_dir/manager-target"
     chmod 0600 "$checkpoint_dir/manager-target"
   fi
-  for unit in "${unit_names[@]}"; do
+  for unit in "${unit_names[@]}" "${optional_unit_names[@]}"; do
     [[ ! -f "$systemd_dir/$unit" ]] || install -m 0644 "$systemd_dir/$unit" "$checkpoint_dir/systemd/$unit"
   done
   : >"$checkpoint_dir/enabled-timers"
@@ -638,6 +638,13 @@ make_checkpoint() {
     if systemctl is-enabled --quiet "$timer" 2>/dev/null; then printf '%s\n' "$timer" >>"$checkpoint_dir/enabled-timers"; fi
     if systemctl is-active --quiet "$timer" 2>/dev/null; then printf '%s\n' "$timer" >>"$checkpoint_dir/active-timers"; fi
   done
+  local opt_name
+  for opt_name in "${optional_unit_names[@]}"; do
+    case "$opt_name" in *.timer)
+      if systemctl is-enabled --quiet "$opt_name" 2>/dev/null; then printf '%s\n' "$opt_name" >>"$checkpoint_dir/enabled-timers"; fi
+      if systemctl is-active --quiet "$opt_name" 2>/dev/null; then printf '%s\n' "$opt_name" >>"$checkpoint_dir/active-timers"; fi
+    ;; esac
+  done
   chmod 0600 "$checkpoint_dir/enabled-timers" "$checkpoint_dir/active-timers"
   : >"$checkpoint_dir/.complete"
   chmod 0600 "$checkpoint_dir/.complete"
@@ -731,6 +738,9 @@ install_systemd_units() {
 remove_systemd_units() {
   systemctl disable --now "${timer_names[@]}" >/dev/null 2>&1 || true
   local unit
+  for unit in "${optional_unit_names[@]}"; do
+    case "$unit" in *.timer) systemctl disable --now "$unit" >/dev/null 2>&1 || true ;; esac
+  done
   for unit in "${unit_names[@]}" "${optional_unit_names[@]}"; do rm -f "$systemd_dir/$unit"; done
   systemctl daemon-reload
 }
@@ -813,7 +823,14 @@ PY
   local opt_timer
   for opt_timer in "${optional_unit_names[@]}"; do
     case "$opt_timer" in *.timer)
-      systemctl enable --now "$opt_timer" >/dev/null 2>&1 || true
+      # Only enable remote reconciliation timers when config is
+      # identified as an OWNER/REPO (not a local checkout path)
+      if [[ "$config_identity" == *"/"* && "$config_identity" != "/"* ]]; then
+        systemctl enable --now "$opt_timer" >/dev/null 2>&1 || true
+      else
+        # Local checkout path — disable and stop any previously enabled timer
+        systemctl disable --now "$opt_timer" >/dev/null 2>&1 || true
+      fi
     ;; esac
   done
 }
@@ -832,6 +849,13 @@ restore_systemd_snapshot() {
     if grep -Fxq "$timer" "$checkpoint_dir/enabled-timers"; then systemctl enable "$timer" >/dev/null || failed=1; else systemctl disable "$timer" >/dev/null 2>&1 || true; fi
     if grep -Fxq "$timer" "$checkpoint_dir/active-timers"; then systemctl start "$timer" || failed=1; else systemctl stop "$timer" >/dev/null 2>&1 || true; fi
   done
+  local opt_name
+  for opt_name in "${optional_unit_names[@]}"; do
+    case "$opt_name" in *.timer)
+      if grep -Fxq "$opt_name" "$checkpoint_dir/enabled-timers"; then systemctl enable "$opt_name" >/dev/null || failed=1; else systemctl disable "$opt_name" >/dev/null 2>&1 || true; fi
+      if grep -Fxq "$opt_name" "$checkpoint_dir/active-timers"; then systemctl start "$opt_name" || failed=1; else systemctl stop "$opt_name" >/dev/null 2>&1 || true; fi
+    ;; esac
+  done
   return "$failed"
 }
 

scripts/remote-reconcile.sh

@@ -163,13 +163,22 @@ validate_config() {
   git -C "$checkout_dir" ls-tree -rz --name-only "$commit" >"$temp_dir/tree-paths" 2>/dev/null || return 1
 
   # Validate using the installer's validation chain
+  # Also run the template validator with --strict + --tree-paths (like installer does)
   python3 "$repo_root/scripts/desired_state.py" validate --config "$temp_dir/fleet.json" 2>"$temp_dir/validate_err" || {
     local err
     err=$(<"$temp_dir/validate_err")
     [[ -n "$err" ]] || err="validation failed"
     log_json "ERROR" "validation" "config validation failed"
     return 1
   }
+  python3 "$repo_root/templates/config-repository/scripts/validate.py" \
+    --config "$temp_dir/fleet.json" --strict --tree-paths "$temp_dir/tree-paths" 2>"$temp_dir/strict_err" || {
+    local err
+    err=$(<"$temp_dir/strict_err")
+    [[ -n "$err" ]] || err="strict validation failed"
+    log_json "ERROR" "validation" "strict validation rejected"
+    return 1
+  }
 
   # Secret scan
   python3 "$repo_root/scripts/scan_committed_secrets.py" \
@@ -209,30 +218,40 @@ PY
   local lkg_config=$lkg_dir/fleet.json
   [[ -f "$lkg_config" ]] || { log_json "ERROR" "rollback" "LKG fleet.json missing"; return 1; }
 
-  # Re-apply the LKG ref via the installer, using the durable repo identity
-  # Create a local checkout pinned to the LKG ref for the installer
+  # Re-apply the LKG ref via the installer
+  # Create a checkout containing the LKG ref (may differ from fetched HEAD)
   local lkg_pinned=$temp_dir/lkg-pinned
-  cp -a "$temp_dir/config-repo" "$lkg_pinned" 2>/dev/null || {
-    # Fallback: fresh fetch
-    mkdir -p "$lkg_pinned"
-    git init -q "$lkg_pinned"
-    git -C "$lkg_pinned" remote add origin "https://github.com/${lkg_repo}.git"
-    GIT_TERMINAL_PROMPT=0 git -C "$lkg_pinned" fetch -q --depth=1 origin "$lkg_ref" 2>/dev/null || {
+  mkdir -p "$lkg_pinned"
+  git init -q "$lkg_pinned"
+  # Use the reconciliation token for authenticated fetch
+  local lkg_token
+  lkg_token=$(cat "$temp_dir/reconcile-token" 2>/dev/null || echo "")
+  if [[ -n "$lkg_token" ]]; then
+    GIT_TERMINAL_PROMPT=0 git -C "$lkg_pinned" fetch -q --depth=1 \
+      "https://x-access-token:${lkg_token}@github.com/${lkg_repo}.git" "$lkg_ref" 2>"$temp_dir/lkg_fetch_err" || {
       log_json "ERROR" "rollback" "LKG fetch failed"
       return 1
     }
-  }
+  else
+    GIT_TERMINAL_PROMPT=0 git -C "$lkg_pinned" fetch -q --depth=1 origin "$lkg_ref" 2>"$temp_dir/lkg_fetch_err" || {
+      log_json "ERROR" "rollback" "LKG fetch failed"
+      return 1
+    }
+  fi
+  git -C "$lkg_pinned" checkout -q FETCH_HEAD
 
+  release_lock
   "$installer" --upgrade \
     --config-repo "$lkg_pinned" \
     --ref "$lkg_ref" \
     --controller "$lkg_controller" 2>"$temp_dir/rollback_err" && {
+    acquire_lock
     # Fix the config_repository in the state file to the durable name
     fix_state_config_repo "$lkg_repo"
     log_json "WARN" "rollback" "restored last-known-good"
     return 0
   }
-
+  acquire_lock
   local err
   err=$(<"$temp_dir/rollback_err")
   log_json "ERROR" "rollback" "rollback failed: ${err}"
@@ -263,6 +282,39 @@ if state.get("config_repository") != durable:
 PY
 }
 
+fix_rendered_env_config_repo() {
+  local durable=$1
+  [[ -f "$rendered_env" ]] || return 0
+  python3 - "$rendered_env" "$durable" <<'PY' 2>/dev/null || true
+import os, sys, tempfile
+path = sys.argv[1]
+durable = sys.argv[2]
+with open(path, encoding="utf-8") as f:
+    lines = f.readlines()
+changed = False
+for i, line in enumerate(lines):
+    if line.startswith("CI_FLEET_CONFIG_REPOSITORY="):
+        val = line.split("=", 1)[1].strip()
+        if val != durable:
+            lines[i] = f"CI_FLEET_CONFIG_REPOSITORY={durable}\n"
+            changed = True
+        break
+if not changed:
+    raise SystemExit(0)
+fd, tmp = tempfile.mkstemp(prefix=".fix-env.", dir=os.path.dirname(path), text=True)
+try:
+    with os.fdopen(fd, "w", encoding="utf-8") as f:
+        f.writelines(lines)
+        f.flush()
+        os.fsync(f.fileno())
+    os.chmod(tmp, 0o600)
+    os.replace(tmp, path)
+except:
+    os.unlink(tmp, missing_ok=True)
+    raise
+PY
+}
+
 save_lkg() {
   local checkout_dir=$1 commit=$2
   install -d -m 0700 "$lkg_dir"
@@ -308,10 +360,32 @@ print(json.dumps({
 PY
 }
 
+release_lock() { flock -u 9 2>/dev/null || true; }
+acquire_lock() { flock -n 9 2>/dev/null || die "cannot re-acquire installer lock"; }
+
+# --- Health (with rendered env) ---
+
+run_health_check() {
+  local output=$1
+  (
+    set -a
+    # shellcheck disable=SC1090
+    [[ ! -f "$rendered_env" ]] || . "$rendered_env"
+    set +a
+    python3 "$repo_root/scripts/health.py" local --output "$output" 2>/dev/null
+  ) && python3 -c "import json; print(json.load(open('$output'))['status'])" 2>/dev/null || echo "unknown"
+}
+
 # --- Main ---
 
 require_commands
 
+# Serialize with installer mutations — share the installer's lock
+lock_file=${CI_FLEET_INSTALLER_LOCK:-/run/ci-fleet-installer.lock}
+install -d -m 0755 "$(dirname "$lock_file")"
+exec 9>"$lock_file"
+flock -n 9 || die "another reconcile or installer is already running"
+
 # Load installed state
 load_installed_state || die "no installed state found at $state_file"
 note "INSTALLED controller=${installed_controller} config_repo=${installed_config_repo} config_ref=${installed_config_ref}"
@@ -338,6 +412,7 @@ while ((attempt < max_attempts)); do
     ((attempt < max_attempts)) && { sleep 5; continue; }
     die "token generation exhausted after ${max_attempts} attempts"
   }
+  printf '%s' "$token" >"$temp_dir/reconcile-token"
 
   # Fetch remote config
   note "FETCHING_CONFIG repo=${installed_config_repo}"
@@ -361,21 +436,35 @@ if [[ "$desired_commit" == "$installed_config_ref" ]]; then
     exit 0
   fi
 
-  # Run existing drift check
-  if "$installer" --check \
-    --config-repo "$installed_config_repo" \
-    --ref "$installed_config_ref" \
-    --controller "$installed_controller" 2>"$temp_dir/drift_err"; then
+  # Run drift check using the fetched local checkout
+  local_pinned=$temp_dir/config-repo
+  if [[ -d "$local_pinned/.git" ]]; then
+    release_lock
+    if "$installer" --check \
+      --config-repo "$local_pinned" \
+      --ref "$installed_config_ref" \
+      --controller "$installed_controller" 2>"$temp_dir/drift_err"; then
+      acquire_lock
+      note "CONVERGED controller=${installed_controller} config_ref=${installed_config_ref}"
+      save_reconcile_state 'converged' "$desired_commit" "$installed_config_ref" 'healthy' 'no change, converged'
+      exit 0
+    fi
+    acquire_lock
+  fi
+  # Same commit + drift = check controller health
+  # If controller is unhealthy, reconcile; otherwise converge with drift note
+  if [[ "$mode" == check-only ]]; then
+    save_reconcile_state 'drift' "$desired_commit" "$installed_config_ref" 'drift' 'internal drift detected'
+    exit 3
+  fi
+  # Full mode: run health check to decide if reconciliation is needed
+  controller_running=$(docker inspect --format '{{.State.Status}}' "ci-fleet-controller-1" 2>/dev/null || echo "missing")
+  if [[ "$controller_running" != "running" ]]; then
+    note "DRIFT with unhealthy controller, falling through to reconcile"
+  else
     note "CONVERGED controller=${installed_controller} config_ref=${installed_config_ref}"
-    save_reconcile_state 'converged' "$desired_commit" "$installed_config_ref" 'healthy' 'no change, converged'
+    save_reconcile_state 'converged' "$desired_commit" "$installed_config_ref" 'drift' 'no commit change; internal drift tracked by drift timer'
     exit 0
-  else
-    drift_exit=$?
-    note "DRIFT detected (exit=${drift_exit}), attempting reconcile"
-    if [[ "$mode" == check-only ]]; then
-      save_reconcile_state 'drift' "$desired_commit" "$installed_config_ref" 'drift' "drift detected (exit=${drift_exit})"
-      exit 3
-    fi
   fi
 fi
 
@@ -418,32 +507,39 @@ git -C "$pinned_dir" checkout -q "$desired_commit"
 
 # Reconcile
 note "RECONCILING controller=${installed_controller} config_ref=${desired_commit}"
+release_lock
 if "$installer" --upgrade \
-  --config-repo "$installed_config_repo" \
+  --config-repo "$pinned_dir" \
   --ref "$desired_commit" \
   --controller "$installed_controller" 2>"$temp_dir/upgrade_err"; then
+  acquire_lock
   note "RECONCILED controller=${installed_controller} config_ref=${desired_commit}"
 
-  # Fix the config_repository in the state file to the durable name
+  # Fix config_repository in state file AND rendered env to the durable name
   fix_state_config_repo "$installed_config_repo"
+  fix_rendered_env_config_repo "$installed_config_repo"
+
+  # Re-enable reconcile timer (may have been disabled during local-checkout upgrade)
+  systemctl enable --now ci-fleet-reconcile.timer >/dev/null 2>&1 || true
 
   # Save new LKG
   save_lkg "$fetch_dir" "$desired_commit"
 
   # Run health check
-  health_status=$(python3 "$repo_root/scripts/health.py" local --output "$temp_dir/health.json" 2>/dev/null && python3 -c "import json; print(json.load(open('$temp_dir/health.json'))['status'])" 2>/dev/null || echo "unknown")
+  health_status=$(run_health_check "$temp_dir/health.json")
 
   save_reconcile_state 'converged' "$desired_commit" "$desired_commit" "$health_status" "reconciled to ${desired_commit}"
   note "RECONCILE_OK controller=${installed_controller} desired=${desired_commit} applied=${desired_commit} health=${health_status}"
   exit 0
 else
+  acquire_lock
   upg_err=$(<"$temp_dir/upgrade_err")
   note "RECONCILE_FAILED error=${upg_err:-unknown}"
 
   # Rollback to LKG — reinstalls a checkpoint of this attempt was already created,
   # or safely restores LKG config directly via the installer
   apply_lkg || die "rollback to last-known-good also failed"
-  health_status=$(python3 "$repo_root/scripts/health.py" local --output "$temp_dir/health.json" 2>/dev/null && python3 -c "import json; print(json.load(open('$temp_dir/health.json'))['status'])" 2>/dev/null || echo "unknown")
+  health_status=$(run_health_check "$temp_dir/health.json")
   save_reconcile_state 'rolled_back' "$desired_commit" "$installed_config_ref" "$health_status" "reconciled failed, rolled back to ${installed_config_ref}"
   note "ROLLBACK_OK controller=${installed_controller} restored=${installed_config_ref}"
   exit 3