From 8f972e988494e26f073f815038ede053e639cd1c Mon Sep 17 00:00:00 2001
From: Nickfost <1572453+Nickfost@users.noreply.github.com>
Date: Wed, 2 Sep 2026 23:13:06 -0500
Subject: [PATCH 01/14] fix: keep candidate builds outside upgrade transactions

---
 controller/main.go                        |  4 ++-
 controller/status.go                      |  6 ++--
 controller/status_test.go                 | 35 +++++++++--------------
 host/systemd/ci-fleet-reconcile.service   |  2 ++
 scripts/install-worker-controller.sh      |  2 +-
 scripts/preflight.sh                      |  2 +-
 scripts/test-install-worker-controller.sh | 14 ++++++++-
 scripts/test_remote_reconcile.py          |  7 +++++
 8 files changed, 43 insertions(+), 29 deletions(-)

diff --git a/controller/main.go b/controller/main.go
index 8919eb18..c369fb3f 100644
--- a/controller/main.go
+++ b/controller/main.go
@@ -76,7 +76,9 @@ func run(ctx context.Context) error {
 	scaler := &Scaler{runners: newRunnerState(), dockerClient: docker, scalesetClient: client, logger: logger, config: cfg, scaleSetID: set.ID}
 	if err := scaler.recoverStale(ctx); err != nil { return err }
 	scaler.writeStatus()
-	go scaler.publishStatus(ctx, time.Minute)
+	statusTicker := time.NewTicker(time.Minute)
+	defer statusTicker.Stop()
+	go scaler.publishStatus(ctx, statusTicker.C)
 	defer scaler.shutdown(context.WithoutCancel(ctx))
 	hostname, err := os.Hostname()
 	if err != nil { return fmt.Errorf("get hostname: %w", err) }
diff --git a/controller/status.go b/controller/status.go
index d63e7f16..bbc268b6 100644
--- a/controller/status.go
+++ b/controller/status.go
@@ -53,14 +53,12 @@ func (s *Scaler) writeStatus() {
 	if err != nil { s.logger.Warn("write controller status", "error", err) }
 }
 
-func (s *Scaler) publishStatus(ctx context.Context, interval time.Duration) {
-	ticker := time.NewTicker(interval)
-	defer ticker.Stop()
+func (s *Scaler) publishStatus(ctx context.Context, ticks <-chan time.Time) {
 	for {
 		select {
 		case <-ctx.Done():
 			return
-		case <-ticker.C:
+		case <-ticks:
 			s.writeStatus()
 		}
 	}
diff --git a/controller/status_test.go b/controller/status_test.go
index 8ed3bd59..2f304658 100644
--- a/controller/status_test.go
+++ b/controller/status_test.go
@@ -113,26 +113,19 @@ func TestStatusPublisherRefreshesIdleSnapshot(t *testing.T) {
 	}
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	done := make(chan struct{})
-	go func() {
-		scaler.publishStatus(ctx, time.Millisecond)
-		close(done)
-	}()
-	deadline := time.After(time.Second)
-	for {
-		if _, err := os.Stat(path); err == nil {
-			cancel()
-			select {
-			case <-done:
-			case <-time.After(time.Second):
-				t.Fatal("idle status publisher did not stop")
-			}
-			return
-		}
-		select {
-		case <-deadline:
-			t.Fatal("idle status publisher did not refresh snapshot")
-		case <-time.After(time.Millisecond):
-		}
+	original := encodeControllerStatus
+	encodeControllerStatus = func(file *os.File, value controllerStatus) error {
+		err := original(file, value)
+		cancel()
+		return err
 	}
+	defer func() { encodeControllerStatus = original }()
+	ticks := make(chan time.Time, 1)
+	ticks <- time.Now()
+	scaler.publishStatus(ctx, ticks)
+	body, err := os.ReadFile(path)
+	if err != nil { t.Fatal("idle status publisher did not refresh snapshot") }
+	var got controllerStatus
+	if err := json.Unmarshal(body, &got); err != nil { t.Fatal(err) }
+	if got.Current != 0 || got.Busy != 0 { t.Fatalf("published non-idle status: %+v", got) }
 }
diff --git a/host/systemd/ci-fleet-reconcile.service b/host/systemd/ci-fleet-reconcile.service
index d8f7d875..f351f2d4 100644
--- a/host/systemd/ci-fleet-reconcile.service
+++ b/host/systemd/ci-fleet-reconcile.service
@@ -9,6 +9,8 @@ Type=oneshot
 User=root
 WorkingDirectory=/opt/ci-fleet/manager/current
 ExecStart=/opt/ci-fleet/manager/current/scripts/remote-reconcile.sh
+# One hour for the installer lock, plus one hour for a cold build and rollback.
+TimeoutStartSec=2h
 Restart=no
 # 0=noop, 3=drift/invalid — timer retries
 SuccessExitStatus=0 3
diff --git a/scripts/install-worker-controller.sh b/scripts/install-worker-controller.sh
index 639f36d3..2cb84737 100755
--- a/scripts/install-worker-controller.sh
+++ b/scripts/install-worker-controller.sh
@@ -1065,6 +1065,7 @@ perform_converge() {
     return
   fi
   install_release
+  build_candidate
   make_checkpoint
   transaction_active=true
   if [[ -f "$state_file" || -f "$rendered_env" ]]; then
@@ -1075,7 +1076,6 @@ perform_converge() {
   drain_current
   controller_id=$desired_controller_id
   [[ "$target_state" == active ]] || remove_inactive_managed_runners
-  build_candidate
   activate_candidate
   transaction_active=false
   note "CONVERGED mode=$mode controller=$controller_id config_ref=$config_ref engine_ref=$engine_ref state=$target_state"
diff --git a/scripts/preflight.sh b/scripts/preflight.sh
index 7751d516..f2a4ee37 100755
--- a/scripts/preflight.sh
+++ b/scripts/preflight.sh
@@ -85,7 +85,7 @@ disk_used=$(df -P "$docker_root" 2>/dev/null | awk 'NR==2 {gsub(/%/, "", $5); pr
 if [[ -z "$disk_used" ]]; then disk_used=$(df -P / | awk 'NR==2 {gsub(/%/, "", $5); print $5}'); fi
 if ((disk_used >= 80)); then fail "Docker filesystem is ${disk_used}% full"; else ok "Docker filesystem is ${disk_used}% full"; fi
 
-if command -v docker >/dev/null; then
+if command -v docker >/dev/null && [[ "$managed" == false ]]; then
   active=$(docker ps -q \
     --filter label=io.randomdevelopment.ci-fleet.managed=true \
     --filter "label=io.randomdevelopment.ci-fleet.instance=${CI_FLEET_INSTANCE:-}" | wc -l | tr -d ' ')
diff --git a/scripts/test-install-worker-controller.sh b/scripts/test-install-worker-controller.sh
index cce7fd89..50228c91 100755
--- a/scripts/test-install-worker-controller.sh
+++ b/scripts/test-install-worker-controller.sh
@@ -139,6 +139,7 @@ case "${1:-}" in
         [[ -z "$paused_state" ]] || rm -f "$paused_state"
         ;;
       build)
+        [[ -z "${FAKE_FAIL_BUILD:-}" ]] || exit 46
         [[ -z "${FAKE_RUNNER_IMAGE_STATE:-}" ]] || printf '%s\n' "${FAKE_ENGINE_REF:?}" >"$FAKE_RUNNER_IMAGE_STATE"
         [[ -z "${FAKE_CONTROLLER_IMAGE_STATE:-}" ]] || printf '%s\n' "${FAKE_ENGINE_REF:?}" >"$FAKE_CONTROLLER_IMAGE_STATE"
         ;;
@@ -618,6 +619,17 @@ ln -sfn "$prior_manager" "$root/opt/ci-fleet/manager/current"
 expect_failure 'DRIFT maintenance_timers' "$installer" --check "${base_args[@]}" --ref "$ref_one"
 
 ref_two=$(write_config active 2 2)
+build_failure_output=$tmp/build-failure.out
+build_failure_root=$tmp/build-failure-root
+cp -a "$root" "$build_failure_root"
+export FAKE_FAIL_BUILD=1
+if "$installer" --upgrade "${base_args[@]}" --ref "$ref_two" >"$build_failure_output" 2>&1; then
+  fail 'candidate build failure unexpectedly succeeded'
+fi
+unset FAKE_FAIL_BUILD
+if grep -Eq 'CHECKPOINT_CREATED|DRAIN_READY|ROLLBACK_' "$build_failure_output"; then fail 'candidate build failure entered the transaction'; fi
+diff -r "$build_failure_root" "$root" >/dev/null || fail 'candidate build failure changed host state'
+[[ -f "$FAKE_DOCKER_STATE" ]] || fail 'candidate build failure stopped the installed controller'
 export FAKE_FAIL_KILL_ONCE=$tmp/fail-kill-once
 : >"$FAKE_FAIL_KILL_ONCE"
 expect_failure 'failed to signal the paused controller' "$installer" --upgrade "${base_args[@]}" --ref "$ref_two"
@@ -628,7 +640,7 @@ export FAKE_RUNNER_STATE=$tmp/managed-runner-active
 export FAKE_FAIL_UP_ONCE=$tmp/fail-up-once
 : >"$FAKE_FAIL_UP_ONCE"
 expect_failure 'ROLLBACK_RESTORED' "$installer" --upgrade "${base_args[@]}" --ref "$ref_two"
-[[ ! -f "$FAKE_RUNNER_STATE" ]] || fail 'upgrade preflight ran before the active runner was drained'
+[[ ! -f "$FAKE_RUNNER_STATE" ]] || fail 'upgrade did not drain the active runner before activation'
 unset FAKE_RUNNER_STATE
 unset FAKE_FAIL_UP_ONCE
 grep -Fq 'CI_FLEET_MAX_RUNNERS=1' "$root/etc/ci-fleet/ci-fleet.env" || fail 'failed activation did not restore capacity one'
diff --git a/scripts/test_remote_reconcile.py b/scripts/test_remote_reconcile.py
index 75a1b12c..9aad25ea 100644
--- a/scripts/test_remote_reconcile.py
+++ b/scripts/test_remote_reconcile.py
@@ -283,6 +283,13 @@ def test_service_file_exists(self):
         self.assertIn("ExecStart", content)
         self.assertIn("remote-reconcile.sh", content)
 
+    def test_service_timeout_covers_lock_build_and_rollback(self):
+        svc = REPO_ROOT / "host" / "systemd" / "ci-fleet-reconcile.service"
+        content = svc.read_text()
+        self.assertIn("lock_wait_seconds=3600", RECONCILE_SCRIPT.read_text())
+        self.assertIn("TimeoutStartSec=2h", content)
+        self.assertIn("One hour for the installer lock, plus one hour for a cold build and rollback.", content)
+
     def test_timer_file_exists(self):
         timer = REPO_ROOT / "host" / "systemd" / "ci-fleet-reconcile.timer"
         self.assertTrue(timer.exists())

From b980f677b71ed21d3c2b856321c0376bfc234465 Mon Sep 17 00:00:00 2001
From: Nickfost <1572453+Nickfost@users.noreply.github.com>
Date: Wed, 2 Sep 2026 23:57:39 -0500
Subject: [PATCH 02/14] fix: restore post-drain candidate preflight

---
 scripts/install-worker-controller.sh      |  2 +-
 scripts/preflight.sh                      |  2 +-
 scripts/test-install-worker-controller.sh | 15 +++++++++++++++
 3 files changed, 17 insertions(+), 2 deletions(-)

diff --git a/scripts/install-worker-controller.sh b/scripts/install-worker-controller.sh
index 2cb84737..eff3cf71 100755
--- a/scripts/install-worker-controller.sh
+++ b/scripts/install-worker-controller.sh
@@ -693,7 +693,6 @@ run_candidate_preflight() {
 }
 
 build_candidate() {
-  run_candidate_preflight
   compose "$release_dir" "$candidate_env" config --quiet
   compose "$release_dir" "$candidate_env" build runner-image controller
 }
@@ -1075,6 +1074,7 @@ perform_converge() {
   fi
   drain_current
   controller_id=$desired_controller_id
+  run_candidate_preflight
   [[ "$target_state" == active ]] || remove_inactive_managed_runners
   activate_candidate
   transaction_active=false
diff --git a/scripts/preflight.sh b/scripts/preflight.sh
index f2a4ee37..7751d516 100755
--- a/scripts/preflight.sh
+++ b/scripts/preflight.sh
@@ -85,7 +85,7 @@ disk_used=$(df -P "$docker_root" 2>/dev/null | awk 'NR==2 {gsub(/%/, "", $5); pr
 if [[ -z "$disk_used" ]]; then disk_used=$(df -P / | awk 'NR==2 {gsub(/%/, "", $5); print $5}'); fi
 if ((disk_used >= 80)); then fail "Docker filesystem is ${disk_used}% full"; else ok "Docker filesystem is ${disk_used}% full"; fi
 
-if command -v docker >/dev/null && [[ "$managed" == false ]]; then
+if command -v docker >/dev/null; then
   active=$(docker ps -q \
     --filter label=io.randomdevelopment.ci-fleet.managed=true \
     --filter "label=io.randomdevelopment.ci-fleet.instance=${CI_FLEET_INSTANCE:-}" | wc -l | tr -d ' ')
diff --git a/scripts/test-install-worker-controller.sh b/scripts/test-install-worker-controller.sh
index 50228c91..1b3646d4 100755
--- a/scripts/test-install-worker-controller.sh
+++ b/scripts/test-install-worker-controller.sh
@@ -64,6 +64,8 @@ case "${1:-}" in
       printf 'managed-runner\n'
     elif [[ -n "${FAKE_RUNNER_STATE:-}" && -f "$FAKE_RUNNER_STATE" ]]; then
       printf 'managed-runner\n'
+    elif [[ "$*" != *'io.randomdevelopment.ci-fleet.kind=runner'* && -n "${FAKE_ACTIVE_MANAGED_STATE:-}" && -f "$FAKE_ACTIVE_MANAGED_STATE" ]]; then
+      printf 'active-managed-container\n'
     fi
     exit 0
     ;;
@@ -124,6 +126,7 @@ case "${1:-}" in
       stop)
         if [[ -n "${FAKE_STOP_FAIL:-}" && -f "$FAKE_STOP_FAIL" ]]; then exit 42; fi
         rm -f "$state"; [[ -z "$status_file" ]] || rm -f "$status_file"; [[ -z "$paused_state" ]] || rm -f "$paused_state"; [[ -z "${FAKE_CONTROLLER_PROVENANCE_FILE:-}" ]] || rm -f "$FAKE_CONTROLLER_PROVENANCE_FILE"; [[ -z "${FAKE_CONTROLLER_IMAGE_ID_FILE:-}" ]] || rm -f "$FAKE_CONTROLLER_IMAGE_ID_FILE"; [[ -z "${FAKE_CONTROLLER_ENV_FILE:-}" ]] || rm -f "$FAKE_CONTROLLER_ENV_FILE"
+        [[ -z "${FAKE_ACTIVE_MANAGED_AFTER_STOP:-}" ]] || : >"$FAKE_ACTIVE_MANAGED_AFTER_STOP"
         ;;
       down|rm) rm -f "$state"; [[ -z "$status_file" ]] || rm -f "$status_file"; [[ -z "$paused_state" ]] || rm -f "$paused_state"; [[ -z "${FAKE_CONTROLLER_PROVENANCE_FILE:-}" ]] || rm -f "$FAKE_CONTROLLER_PROVENANCE_FILE"; [[ -z "${FAKE_CONTROLLER_IMAGE_ID_FILE:-}" ]] || rm -f "$FAKE_CONTROLLER_IMAGE_ID_FILE"; [[ -z "${FAKE_CONTROLLER_ENV_FILE:-}" ]] || rm -f "$FAKE_CONTROLLER_ENV_FILE" ;;
       pause)
@@ -630,6 +633,18 @@ unset FAKE_FAIL_BUILD
 if grep -Eq 'CHECKPOINT_CREATED|DRAIN_READY|ROLLBACK_' "$build_failure_output"; then fail 'candidate build failure entered the transaction'; fi
 diff -r "$build_failure_root" "$root" >/dev/null || fail 'candidate build failure changed host state'
 [[ -f "$FAKE_DOCKER_STATE" ]] || fail 'candidate build failure stopped the installed controller'
+managed_preflight_output=$tmp/managed-preflight.out
+export FAKE_ACTIVE_MANAGED_STATE=$tmp/active-managed-after-drain
+export FAKE_ACTIVE_MANAGED_AFTER_STOP=$FAKE_ACTIVE_MANAGED_STATE
+if "$installer" --upgrade "${base_args[@]}" --ref "$ref_two" >"$managed_preflight_output" 2>&1; then
+  fail 'managed candidate preflight skipped an active managed container after drain'
+fi
+unset FAKE_ACTIVE_MANAGED_AFTER_STOP FAKE_ACTIVE_MANAGED_STATE
+grep -Fq 'DRAIN_OK managed_runners=0' "$managed_preflight_output" || fail 'managed candidate preflight ran before drain completed'
+grep -Fq 'managed containers already active for this instance' "$managed_preflight_output" || fail 'managed candidate preflight did not check active managed containers'
+grep -Fq 'ROLLBACK_RESTORED' "$managed_preflight_output" || fail 'managed candidate preflight failure did not restore the checkpoint'
+rm -f "$tmp/active-managed-after-drain"
+[[ ${CI_FLEET_TEST_STOP_AFTER_MANAGED_PREFLIGHT:-0} != 1 ]] || { printf 'MANAGED_PREFLIGHT_REGRESSION_OK\n'; exit 0; }
 export FAKE_FAIL_KILL_ONCE=$tmp/fail-kill-once
 : >"$FAKE_FAIL_KILL_ONCE"
 expect_failure 'failed to signal the paused controller' "$installer" --upgrade "${base_args[@]}" --ref "$ref_two"

From 9e1488b861a247723c8e2bbf3cadf6ed3943a763 Mon Sep 17 00:00:00 2001
From: Nickfost <1572453+Nickfost@users.noreply.github.com>
Date: Thu, 3 Sep 2026 05:19:50 -0500
Subject: [PATCH 03/14] fix: harden reconcile transaction recovery

---
 docs/ADDING-A-HOST.md                     |  2 +-
 docs/DESIRED-STATE.md                     | 15 ++--
 host/systemd/ci-fleet-reconcile.service   |  4 +-
 scripts/install-worker-controller.sh      | 40 +++++++++--
 scripts/test-install-worker-controller.sh | 86 ++++++++++++++++++++++-
 scripts/test_remote_reconcile.py          |  6 +-
 6 files changed, 136 insertions(+), 17 deletions(-)

diff --git a/docs/ADDING-A-HOST.md b/docs/ADDING-A-HOST.md
index 03325b5c..f1ff9ab4 100644
--- a/docs/ADDING-A-HOST.md
+++ b/docs/ADDING-A-HOST.md
@@ -102,7 +102,7 @@ sudo ./scripts/install-worker-controller.sh \
   --controller example-ci-01
 ```
 
-Use `--adopt` when converting an existing manual controller. The installer validates the complete configuration, renders host-local runtime state, installs the pinned engine, creates a controller checkpoint, builds images, installs maintenance timers, and verifies health.
+Use `--adopt` when converting an existing manual controller. The installer validates the complete configuration, renders host-local runtime state, installs the pinned engine, and validates the candidate Compose configuration. It may build images with a distinct inert runner tag before creating a checkpoint and draining. A build that would retag the runner image used by the active controller waits until after checkpoint and drain. Managed preflight still runs after drain, before activation. A failed pre-transaction build may leave candidate image or layer artifacts, but it does not change installed state or stop the active controller.
 
 The configuration repository credential, when required, must be read-only and host-side. Credentials are never accepted in command arguments.
 
diff --git a/docs/DESIRED-STATE.md b/docs/DESIRED-STATE.md
index 5d262b7f..422f4ae4 100644
--- a/docs/DESIRED-STATE.md
+++ b/docs/DESIRED-STATE.md
@@ -136,12 +136,13 @@ The installer:
 3. selects exactly one logical controller;
 4. renders `/etc/ci-fleet/ci-fleet.env` without secret values;
 5. fetches and verifies the pinned public engine commit;
-6. creates a root-only controller checkpoint;
-7. drains the current controller and waits for every managed runner to finish, including orphaned runners left after a stopped or crashed controller;
-8. runs managed preflight and builds the pinned runner and controller images;
-9. installs health, cleanup, and pinned-state drift unit definitions;
-10. starts the controller only when its desired state is active and verifies runtime health;
-11. atomically records redacted installation state, then enables the maintenance timers.
+6. validates the candidate Compose configuration and builds images before the transaction when no active installed controller uses the candidate runner tag;
+7. creates a root-only controller checkpoint;
+8. drains the current controller and waits for every managed runner to finish, including orphaned runners left after a stopped or crashed controller;
+9. runs managed preflight and, when the candidate would retag the active controller's runner image, builds the images after drain;
+10. installs health, cleanup, and pinned-state drift unit definitions;
+11. starts the controller only when its desired state is active and verifies runtime health;
+12. atomically records redacted installation state, then enables the maintenance timers.
 
 A successful second `--install` run reports `NO_CHANGE` and performs no unnecessary replacement. A successful engine upgrade advances both the runtime release and the maintenance installer-manager to the same pinned commit; rollback restores both.
 
@@ -240,7 +241,7 @@ Legacy project-specific hosts remain until CI, promotion, and deployment no long
 
 ## Failure and recovery behavior
 
-Before mutation, the installer records the prior rendered environment, installation metadata, runtime release, installer-manager release, and maintenance unit/timer state under `/var/lib/ci-fleet/checkpoints`. Each checkpoint is staged and atomically renamed with a completion marker; rollback ignores partial staging directories. Build and validation happen before the active release changes. A failed activation or health check drains the candidate, restores those artifacts, restarts the prior controller only when no managed runner is active, and verifies prior-release health before reporting rollback success. A host-local installer lock serializes every check and mutation. Runtime and installer-manager releases are staged on their respective target filesystems and renamed atomically so a failed copy cannot masquerade as an installed immutable release.
+Before mutation, the installer records the prior rendered environment, installation metadata, runtime release, installer-manager release, and maintenance unit/timer state under `/var/lib/ci-fleet/checkpoints`. Each checkpoint is staged and atomically renamed with a completion marker; rollback ignores partial staging directories. Compose validation happens before the checkpoint. A build using a distinct inert runner tag may also happen before the checkpoint; failure may leave candidate image or layer artifacts, but installed state and the active controller remain unchanged. A build that would retag the active controller's runner image happens only after checkpoint and drain. A failed activation or health check drains the candidate, restores those artifacts, restarts the prior controller only when no managed runner is active, and verifies prior-release health before reporting rollback success. A host-local installer lock serializes every check and mutation. Runtime and installer-manager releases are staged on their respective target filesystems and renamed atomically so a failed copy cannot masquerade as an installed immutable release.
 
 Installer checkpoints and machine backups serve different failure classes. A checkpoint rolls back a single failed reconciliation. Recoverability of the machine itself is governed by each host's declared failure boundary (see [Adding a host](ADDING-A-HOST.md)): a disposable controller needs no machine backup at all — recovery is rebuilding from reviewed Git-authored desired state — while a non-disposable host follows its own documented local backup policy.
 
diff --git a/host/systemd/ci-fleet-reconcile.service b/host/systemd/ci-fleet-reconcile.service
index f351f2d4..639e2cf9 100644
--- a/host/systemd/ci-fleet-reconcile.service
+++ b/host/systemd/ci-fleet-reconcile.service
@@ -9,8 +9,10 @@ Type=oneshot
 User=root
 WorkingDirectory=/opt/ci-fleet/manager/current
 ExecStart=/opt/ci-fleet/manager/current/scripts/remote-reconcile.sh
-# One hour for the installer lock, plus one hour for a cold build and rollback.
+# One hour for the installer lock, plus one hour for a cold build and transaction.
 TimeoutStartSec=2h
+# Allow checkpoint restoration after TimeoutStartSec sends SIGTERM.
+TimeoutStopSec=15min
 Restart=no
 # 0=noop, 3=drift/invalid — timer retries
 SuccessExitStatus=0 3
diff --git a/scripts/install-worker-controller.sh b/scripts/install-worker-controller.sh
index eff3cf71..f6a5e3e5 100755
--- a/scripts/install-worker-controller.sh
+++ b/scripts/install-worker-controller.sh
@@ -36,10 +36,13 @@ EOF
 note() { printf '%s\n' "$*"; }
 die() {
   printf 'ERROR: %s\n' "$*" >&2
+  trap - ERR
+  trap '' TERM
   if [[ ${transaction_active:-false} == true ]] && declare -F restore_checkpoint >/dev/null; then
     restore_checkpoint || true
     transaction_active=false
   fi
+  trap - ERR
   exit 2
 }
 
@@ -693,7 +696,6 @@ run_candidate_preflight() {
 }
 
 build_candidate() {
-  compose "$release_dir" "$candidate_env" config --quiet
   compose "$release_dir" "$candidate_env" build runner-image controller
 }
 
@@ -1024,14 +1026,24 @@ restore_checkpoint() {
   note "ROLLBACK_RESTORED checkpoint=$checkpoint_dir"
 }
 
-on_error() {
-  local status=$?
+rollback_and_exit() {
+  local status=$1
+  trap - ERR
+  trap '' TERM
   if $transaction_active; then
     restore_checkpoint || true
+    transaction_active=false
   fi
+  trap - ERR
   exit "$status"
 }
+on_error() {
+  local status=$?
+  rollback_and_exit "$status"
+}
+on_term() { rollback_and_exit 143; }
 trap on_error ERR
+trap on_term TERM
 
 perform_check() {
   local count
@@ -1046,7 +1058,8 @@ perform_check() {
 }
 
 perform_converge() {
-  local count existing_status desired_controller_id=$controller_id
+  local count existing_status candidate_runner_image installed_runner_image expected_owner=0
+  local desired_controller_id=$controller_id build_before_drain=false
   if [[ "$mode" == upgrade && ! -f "$state_file" ]]; then
     die '--upgrade requires an existing managed installation; use --install or --adopt'
   fi
@@ -1064,7 +1077,19 @@ perform_converge() {
     return
   fi
   install_release
-  build_candidate
+  compose "$release_dir" "$candidate_env" config --quiet
+  candidate_runner_image=$(awk -F= '$1 == "CI_FLEET_RUNNER_IMAGE" {count++; value=substr($0, index($0, "=") + 1)} END {if (count != 1) exit 1; print value}' "$candidate_env") || die 'rendered candidate runner image is invalid'
+  [[ "$testing" != 1 ]] || expected_owner=$(id -u)
+  case "$existing_status" in
+    ''|exited|created|dead) build_before_drain=true ;;
+    running)
+      if [[ -f "$rendered_env" && $(stat -c %u "$rendered_env") == "$expected_owner" && $(stat -c %a "$rendered_env") == 600 ]] \
+        && installed_runner_image=$(awk -F= '$1 == "CI_FLEET_RUNNER_IMAGE" {count++; value=substr($0, index($0, "=") + 1)} END {if (count != 1) exit 1; print value}' "$rendered_env"); then
+        [[ "$candidate_runner_image" == "$installed_runner_image" ]] || build_before_drain=true
+      fi
+      ;;
+  esac
+  if $build_before_drain; then build_candidate; fi
   make_checkpoint
   transaction_active=true
   if [[ -f "$state_file" || -f "$rendered_env" ]]; then
@@ -1073,8 +1098,13 @@ perform_converge() {
     die '--adopt requires a trusted installed controller identity'
   fi
   drain_current
+  if [[ "$testing" == 1 && -n ${CI_FLEET_TEST_PAUSE_AFTER_DRAIN_FILE:-} ]]; then
+    printf '%s\n' "$BASHPID" >"$CI_FLEET_TEST_PAUSE_AFTER_DRAIN_FILE"
+    while [[ ! -f "$CI_FLEET_TEST_PAUSE_AFTER_DRAIN_FILE.continue" ]]; do sleep 0.05; done
+  fi
   controller_id=$desired_controller_id
   run_candidate_preflight
+  if ! $build_before_drain; then build_candidate; fi
   [[ "$target_state" == active ]] || remove_inactive_managed_runners
   activate_candidate
   transaction_active=false
diff --git a/scripts/test-install-worker-controller.sh b/scripts/test-install-worker-controller.sh
index 1b3646d4..35490c5a 100755
--- a/scripts/test-install-worker-controller.sh
+++ b/scripts/test-install-worker-controller.sh
@@ -108,6 +108,12 @@ case "${1:-}" in
     fi
     case "$command" in
       up)
+        if [[ -n "${FAKE_DELAY_UP_ONCE:-}" && -f "$FAKE_DELAY_UP_ONCE" ]]; then
+          delay_marker=$FAKE_DELAY_UP_ONCE
+          rm -f "$delay_marker"
+          : >"${delay_marker}.entered"
+          sleep 2
+        fi
         if [[ -n "${FAKE_FAIL_UP_ONCE:-}" && -f "$FAKE_FAIL_UP_ONCE" ]]; then
           rm -f "$FAKE_FAIL_UP_ONCE"
           exit 42
@@ -146,7 +152,8 @@ case "${1:-}" in
         [[ -z "${FAKE_RUNNER_IMAGE_STATE:-}" ]] || printf '%s\n' "${FAKE_ENGINE_REF:?}" >"$FAKE_RUNNER_IMAGE_STATE"
         [[ -z "${FAKE_CONTROLLER_IMAGE_STATE:-}" ]] || printf '%s\n' "${FAKE_ENGINE_REF:?}" >"$FAKE_CONTROLLER_IMAGE_STATE"
         ;;
-      config|logs) ;;
+      config) [[ -z "${FAKE_FAIL_CONFIG:-}" ]] || exit 47 ;;
+      logs) ;;
       *) exit 1 ;;
     esac
     ;;
@@ -622,6 +629,10 @@ ln -sfn "$prior_manager" "$root/opt/ci-fleet/manager/current"
 expect_failure 'DRIFT maintenance_timers' "$installer" --check "${base_args[@]}" --ref "$ref_one"
 
 ref_two=$(write_config active 2 2)
+prior_runner_image=ci-fleet-runner:prior
+for environment in "$rendered_env" "$FAKE_CONTROLLER_ENV_FILE"; do
+  python3 -c 'from pathlib import Path; import sys; path = Path(sys.argv[1]); path.write_text(path.read_text().replace(sys.argv[2], sys.argv[3]))' "$environment" "$FAKE_RUNNER_IMAGE" "$prior_runner_image"
+done
 build_failure_output=$tmp/build-failure.out
 build_failure_root=$tmp/build-failure-root
 cp -a "$root" "$build_failure_root"
@@ -633,6 +644,79 @@ unset FAKE_FAIL_BUILD
 if grep -Eq 'CHECKPOINT_CREATED|DRAIN_READY|ROLLBACK_' "$build_failure_output"; then fail 'candidate build failure entered the transaction'; fi
 diff -r "$build_failure_root" "$root" >/dev/null || fail 'candidate build failure changed host state'
 [[ -f "$FAKE_DOCKER_STATE" ]] || fail 'candidate build failure stopped the installed controller'
+for environment in "$rendered_env" "$FAKE_CONTROLLER_ENV_FILE"; do
+  python3 -c 'from pathlib import Path; import sys; path = Path(sys.argv[1]); path.write_text(path.read_text().replace(sys.argv[2], sys.argv[3]))' "$environment" "$prior_runner_image" "$FAKE_RUNNER_IMAGE"
+done
+config_failure_output=$tmp/config-failure.out
+export FAKE_FAIL_CONFIG=1
+if "$installer" --upgrade "${base_args[@]}" --ref "$ref_two" >"$config_failure_output" 2>&1; then
+  fail 'candidate Compose validation failure unexpectedly succeeded'
+fi
+unset FAKE_FAIL_CONFIG
+if grep -Eq 'CHECKPOINT_CREATED|DRAIN_READY|ROLLBACK_' "$config_failure_output"; then fail 'candidate Compose validation failure entered the transaction'; fi
+[[ -f "$FAKE_DOCKER_STATE" ]] || fail 'candidate Compose validation failure stopped the installed controller'
+live_tag_build_output=$tmp/live-tag-build.out
+export FAKE_COMPOSE_LOG=$tmp/live-tag-build-compose.log
+: >"$FAKE_COMPOSE_LOG"
+export FAKE_FAIL_BUILD=1
+if "$installer" --upgrade "${base_args[@]}" --ref "$ref_two" >"$live_tag_build_output" 2>&1; then
+  fail 'live-tag candidate build failure unexpectedly succeeded'
+fi
+unset FAKE_FAIL_BUILD
+stop_line=$(grep -n -m1 '^stop|' "$FAKE_COMPOSE_LOG" | cut -d: -f1 || true)
+build_line=$(grep -n -m1 '^build|' "$FAKE_COMPOSE_LOG" | cut -d: -f1 || true)
+[[ -n "$stop_line" && -n "$build_line" && "$stop_line" -lt "$build_line" ]] || fail 'live runner tag was built before drain'
+grep -Fq 'DRAIN_OK managed_runners=0' "$live_tag_build_output" || fail 'live runner tag build did not wait for drain'
+grep -Fq 'ROLLBACK_RESTORED' "$live_tag_build_output" || fail 'live runner tag build failure did not restore the checkpoint'
+grep -Fq 'CI_FLEET_MAX_RUNNERS=1' "$rendered_env" || fail 'live runner tag build failure changed installed state'
+unset FAKE_COMPOSE_LOG
+[[ ${CI_FLEET_TEST_STOP_AFTER_LIVE_TAG_BUILD:-0} != 1 ]] || { printf 'LIVE_TAG_BUILD_REGRESSION_OK\n'; exit 0; }
+terminate_upgrade() {
+  local output=$1 marker=$2 second_term_marker=${3:-} pid installer_pid status=0 attempt
+  export CI_FLEET_TEST_PAUSE_AFTER_DRAIN_FILE=$marker
+  if [[ -n "$second_term_marker" ]]; then
+    : >"$second_term_marker"
+    export FAKE_DELAY_UP_ONCE=$second_term_marker
+  fi
+  "$installer" --upgrade "${base_args[@]}" --ref "$ref_two" >"$output" 2>&1 &
+  pid=$!
+  for ((attempt = 0; attempt < 200; attempt++)); do
+    [[ ! -f "$marker" ]] || break
+    sleep 0.05
+  done
+  if [[ ! -f "$marker" ]]; then
+    kill -TERM "$pid" 2>/dev/null || true
+    wait "$pid" 2>/dev/null || true
+    fail 'TERM regression did not reach the drained transaction'
+  fi
+  installer_pid=$(<"$marker")
+  kill -TERM "$installer_pid"
+  if [[ -n "$second_term_marker" ]]; then
+    for ((attempt = 0; attempt < 200; attempt++)); do
+      [[ ! -f "${second_term_marker}.entered" ]] || break
+      sleep 0.05
+    done
+    [[ -f "${second_term_marker}.entered" ]] || fail 'TERM regression did not enter checkpoint restoration'
+    kill -TERM "$installer_pid"
+  fi
+  if wait "$pid"; then status=0; else status=$?; fi
+  unset CI_FLEET_TEST_PAUSE_AFTER_DRAIN_FILE FAKE_DELAY_UP_ONCE
+  [[ "$status" == 143 ]] || fail "TERM regression changed the signal exit status: $status"
+}
+term_output=$tmp/term-rollback.out
+terminate_upgrade "$term_output" "$tmp/term-pause" "$tmp/term-rollback-up"
+grep -Fq 'ROLLBACK_RESTORED' "$term_output" || fail "TERM did not report checkpoint restoration: $(<"$term_output")"
+[[ ! -f "$FAKE_PAUSED_STATE" && -f "$FAKE_DOCKER_STATE" ]] || fail 'TERM did not restore the active controller'
+grep -Fq 'CI_FLEET_MAX_RUNNERS=1' "$rendered_env" || fail 'TERM did not restore installed state'
+term_failure_output=$tmp/term-rollback-failure.out
+export FAKE_FAIL_UP_ONCE=$tmp/term-rollback-fail-up
+: >"$FAKE_FAIL_UP_ONCE"
+terminate_upgrade "$term_failure_output" "$tmp/term-failure-pause"
+unset FAKE_FAIL_UP_ONCE
+grep -Fq 'ROLLBACK_FAILED' "$term_failure_output" || fail 'TERM rollback failure was not reported'
+expect_success "$installer" --rollback >/dev/null
+[[ -f "$FAKE_DOCKER_STATE" ]] || fail 'explicit rollback did not recover after TERM rollback failure'
+[[ ${CI_FLEET_TEST_STOP_AFTER_TERM_ROLLBACK:-0} != 1 ]] || { printf 'TERM_ROLLBACK_REGRESSION_OK\n'; exit 0; }
 managed_preflight_output=$tmp/managed-preflight.out
 export FAKE_ACTIVE_MANAGED_STATE=$tmp/active-managed-after-drain
 export FAKE_ACTIVE_MANAGED_AFTER_STOP=$FAKE_ACTIVE_MANAGED_STATE
diff --git a/scripts/test_remote_reconcile.py b/scripts/test_remote_reconcile.py
index 9aad25ea..d92e206f 100644
--- a/scripts/test_remote_reconcile.py
+++ b/scripts/test_remote_reconcile.py
@@ -283,12 +283,14 @@ def test_service_file_exists(self):
         self.assertIn("ExecStart", content)
         self.assertIn("remote-reconcile.sh", content)
 
-    def test_service_timeout_covers_lock_build_and_rollback(self):
+    def test_service_timeouts_cover_lock_build_and_rollback(self):
         svc = REPO_ROOT / "host" / "systemd" / "ci-fleet-reconcile.service"
         content = svc.read_text()
         self.assertIn("lock_wait_seconds=3600", RECONCILE_SCRIPT.read_text())
         self.assertIn("TimeoutStartSec=2h", content)
-        self.assertIn("One hour for the installer lock, plus one hour for a cold build and rollback.", content)
+        self.assertIn("One hour for the installer lock, plus one hour for a cold build and transaction.", content)
+        self.assertIn("TimeoutStopSec=15min", content)
+        self.assertIn("Allow checkpoint restoration after TimeoutStartSec sends SIGTERM.", content)
 
     def test_timer_file_exists(self):
         timer = REPO_ROOT / "host" / "systemd" / "ci-fleet-reconcile.timer"

From d4f05168b3a4ffdafd56079934e50931cff76d90 Mon Sep 17 00:00:00 2001
From: Nickfost <1572453+Nickfost@users.noreply.github.com>
Date: Thu, 3 Sep 2026 12:23:59 -0500
Subject: [PATCH 04/14] fix: restore paused controller on reconcile termination

---
 scripts/install-worker-controller.sh      | 12 +++++--
 scripts/test-install-worker-controller.sh | 40 ++++++++++++++++++++---
 2 files changed, 44 insertions(+), 8 deletions(-)

diff --git a/scripts/install-worker-controller.sh b/scripts/install-worker-controller.sh
index f6a5e3e5..6e84616d 100755
--- a/scripts/install-worker-controller.sh
+++ b/scripts/install-worker-controller.sh
@@ -769,8 +769,14 @@ try_drain_current() {
     old_release=$(current_runtime_release)
     [[ -n "$old_release" ]] || old_release=$fallback_release
     if [[ -z "$old_release" || ! -f "$old_release/deploy/compose.yaml" ]]; then drain_error='cannot locate the running controller Compose release for safe adoption'; return 1; fi
-    if ! compose "$old_release" "$drain_env" pause controller >/dev/null; then drain_error='could not pause the controller for drain'; return 1; fi
-    paused=true
+    if [[ $(docker inspect --format '{{.State.Paused}}' "$controller_container" 2>/dev/null || true) == true ]]; then
+      paused=true
+    elif compose "$old_release" "$drain_env" pause controller >/dev/null; then
+      paused=true
+    else
+      drain_error='could not pause the controller for drain'
+      return 1
+    fi
   fi
   deadline=$((SECONDS + ${CI_FLEET_DRAIN_TIMEOUT_SECONDS:-300}))
   while :; do
@@ -1099,7 +1105,7 @@ perform_converge() {
   fi
   drain_current
   if [[ "$testing" == 1 && -n ${CI_FLEET_TEST_PAUSE_AFTER_DRAIN_FILE:-} ]]; then
-    printf '%s\n' "$BASHPID" >"$CI_FLEET_TEST_PAUSE_AFTER_DRAIN_FILE"
+    : >"$CI_FLEET_TEST_PAUSE_AFTER_DRAIN_FILE"
     while [[ ! -f "$CI_FLEET_TEST_PAUSE_AFTER_DRAIN_FILE.continue" ]]; do sleep 0.05; done
   fi
   controller_id=$desired_controller_id
diff --git a/scripts/test-install-worker-controller.sh b/scripts/test-install-worker-controller.sh
index 35490c5a..0e47cebe 100755
--- a/scripts/test-install-worker-controller.sh
+++ b/scripts/test-install-worker-controller.sh
@@ -136,8 +136,12 @@ case "${1:-}" in
         ;;
       down|rm) rm -f "$state"; [[ -z "$status_file" ]] || rm -f "$status_file"; [[ -z "$paused_state" ]] || rm -f "$paused_state"; [[ -z "${FAKE_CONTROLLER_PROVENANCE_FILE:-}" ]] || rm -f "$FAKE_CONTROLLER_PROVENANCE_FILE"; [[ -z "${FAKE_CONTROLLER_IMAGE_ID_FILE:-}" ]] || rm -f "$FAKE_CONTROLLER_IMAGE_ID_FILE"; [[ -z "${FAKE_CONTROLLER_ENV_FILE:-}" ]] || rm -f "$FAKE_CONTROLLER_ENV_FILE" ;;
       pause)
+        if [[ -n "$paused_state" && -f "$paused_state" ]]; then
+          printf 'Error response from daemon: container is already paused\n' >&2
+          exit 42
+        fi
         [[ -z "$paused_state" ]] || : >"$paused_state"
-        [[ -z "${FAKE_RUNNER_STATE:-}" ]] || rm -f "$FAKE_RUNNER_STATE"
+        [[ -n "${FAKE_KEEP_RUNNER_ON_PAUSE:-}" || -z "${FAKE_RUNNER_STATE:-}" ]] || rm -f "$FAKE_RUNNER_STATE"
         ;;
       unpause) [[ -z "$paused_state" ]] || rm -f "$paused_state" ;;
       kill)
@@ -672,7 +676,7 @@ grep -Fq 'CI_FLEET_MAX_RUNNERS=1' "$rendered_env" || fail 'live runner tag build
 unset FAKE_COMPOSE_LOG
 [[ ${CI_FLEET_TEST_STOP_AFTER_LIVE_TAG_BUILD:-0} != 1 ]] || { printf 'LIVE_TAG_BUILD_REGRESSION_OK\n'; exit 0; }
 terminate_upgrade() {
-  local output=$1 marker=$2 second_term_marker=${3:-} pid installer_pid status=0 attempt
+  local output=$1 marker=$2 second_term_marker=${3:-} pid status=0 attempt
   export CI_FLEET_TEST_PAUSE_AFTER_DRAIN_FILE=$marker
   if [[ -n "$second_term_marker" ]]; then
     : >"$second_term_marker"
@@ -689,20 +693,46 @@ terminate_upgrade() {
     wait "$pid" 2>/dev/null || true
     fail 'TERM regression did not reach the drained transaction'
   fi
-  installer_pid=$(<"$marker")
-  kill -TERM "$installer_pid"
+  kill -TERM "$pid"
   if [[ -n "$second_term_marker" ]]; then
     for ((attempt = 0; attempt < 200; attempt++)); do
       [[ ! -f "${second_term_marker}.entered" ]] || break
       sleep 0.05
     done
     [[ -f "${second_term_marker}.entered" ]] || fail 'TERM regression did not enter checkpoint restoration'
-    kill -TERM "$installer_pid"
+    kill -TERM "$pid"
   fi
   if wait "$pid"; then status=0; else status=$?; fi
   unset CI_FLEET_TEST_PAUSE_AFTER_DRAIN_FILE FAKE_DELAY_UP_ONCE
   [[ "$status" == 143 ]] || fail "TERM regression changed the signal exit status: $status"
 }
+paused_term_output=$tmp/paused-term-rollback.out
+export FAKE_RUNNER_STATE=$tmp/paused-term-managed-runner
+export FAKE_KEEP_RUNNER_ON_PAUSE=1
+: >"$FAKE_RUNNER_STATE"
+rm -f "$FAKE_PAUSED_STATE"
+"$installer" --upgrade "${base_args[@]}" --ref "$ref_two" >"$paused_term_output" 2>&1 &
+paused_term_pid=$!
+for ((attempt = 0; attempt < 200; attempt++)); do
+  [[ ! -f "$FAKE_PAUSED_STATE" ]] || break
+  sleep 0.05
+done
+if [[ ! -f "$FAKE_PAUSED_STATE" ]]; then
+  kill -TERM "$paused_term_pid" 2>/dev/null || true
+  wait "$paused_term_pid" 2>/dev/null || true
+  fail 'TERM regression did not reach the paused drain phase'
+fi
+if grep -Fq 'DRAIN_OK managed_runners=0' "$paused_term_output"; then fail 'TERM regression passed the drain phase before signaling'; fi
+kill -TERM "$paused_term_pid"
+rm -f "$FAKE_RUNNER_STATE"
+if wait "$paused_term_pid"; then paused_term_status=0; else paused_term_status=$?; fi
+unset FAKE_KEEP_RUNNER_ON_PAUSE FAKE_RUNNER_STATE
+[[ "$paused_term_status" == 143 ]] || fail "paused TERM regression changed the signal exit status: $paused_term_status"
+grep -Fq 'ROLLBACK_RESTORED' "$paused_term_output" || fail "paused TERM did not report checkpoint restoration: $(<"$paused_term_output")"
+[[ ! -f "$FAKE_PAUSED_STATE" && -f "$FAKE_DOCKER_STATE" ]] || fail 'TERM before DRAIN_OK did not restore the active controller'
+grep -Fq 'CI_FLEET_MAX_RUNNERS=1' "$rendered_env" || fail 'TERM before DRAIN_OK did not restore installed state'
+[[ $(readlink -f "$root/opt/ci-fleet/manager/current") == "$prior_manager" ]] || fail 'TERM before DRAIN_OK did not restore the prior manager release'
+[[ ${CI_FLEET_TEST_STOP_AFTER_PAUSED_TERM:-0} != 1 ]] || { printf 'PAUSED_TERM_REGRESSION_OK\n'; exit 0; }
 term_output=$tmp/term-rollback.out
 terminate_upgrade "$term_output" "$tmp/term-pause" "$tmp/term-rollback-up"
 grep -Fq 'ROLLBACK_RESTORED' "$term_output" || fail "TERM did not report checkpoint restoration: $(<"$term_output")"

From a84fe237d4043887cf9aa17cfd66e29e1085f985 Mon Sep 17 00:00:00 2001
From: Nickfost <1572453+Nickfost@users.noreply.github.com>
Date: Thu, 3 Sep 2026 14:26:16 -0500
Subject: [PATCH 05/14] fix: validate live state before candidate transaction

---
 scripts/install-worker-controller.sh      |  9 ++---
 scripts/test-install-worker-controller.sh | 42 +++++++++++++++++++++--
 2 files changed, 44 insertions(+), 7 deletions(-)

diff --git a/scripts/install-worker-controller.sh b/scripts/install-worker-controller.sh
index 6e84616d..4d1ab99e 100755
--- a/scripts/install-worker-controller.sh
+++ b/scripts/install-worker-controller.sh
@@ -1064,7 +1064,7 @@ perform_check() {
 }
 
 perform_converge() {
-  local count existing_status candidate_runner_image installed_runner_image expected_owner=0
+  local count existing_status candidate_runner_image installed_runner_image live_runner_image expected_owner=0
   local desired_controller_id=$controller_id build_before_drain=false
   if [[ "$mode" == upgrade && ! -f "$state_file" ]]; then
     die '--upgrade requires an existing managed installation; use --install or --adopt'
@@ -1090,12 +1090,13 @@ perform_converge() {
     ''|exited|created|dead) build_before_drain=true ;;
     running)
       if [[ -f "$rendered_env" && $(stat -c %u "$rendered_env") == "$expected_owner" && $(stat -c %a "$rendered_env") == 600 ]] \
-        && installed_runner_image=$(awk -F= '$1 == "CI_FLEET_RUNNER_IMAGE" {count++; value=substr($0, index($0, "=") + 1)} END {if (count != 1) exit 1; print value}' "$rendered_env"); then
-        [[ "$candidate_runner_image" == "$installed_runner_image" ]] || build_before_drain=true
+        && installed_runner_image=$(awk -F= '$1 == "CI_FLEET_RUNNER_IMAGE" {count++; value=substr($0, index($0, "=") + 1)} END {if (count != 1) exit 1; print value}' "$rendered_env") \
+        && live_runner_image=$(docker inspect --format '{{range .Config.Env}}{{println .}}{{end}}' "$controller_container" 2>/dev/null | awk -F= '$1 == "CI_FLEET_RUNNER_IMAGE" {count++; value=substr($0, index($0, "=") + 1)} END {if (count != 1) exit 1; print value}'); then
+        [[ "$candidate_runner_image" == "$installed_runner_image" || "$candidate_runner_image" == "$live_runner_image" ]] || build_before_drain=true
       fi
       ;;
   esac
-  if $build_before_drain; then build_candidate; fi
+  if $build_before_drain; then build_candidate; require_commands; fi
   make_checkpoint
   transaction_active=true
   if [[ -f "$state_file" || -f "$rendered_env" ]]; then
diff --git a/scripts/test-install-worker-controller.sh b/scripts/test-install-worker-controller.sh
index 0e47cebe..bac56075 100755
--- a/scripts/test-install-worker-controller.sh
+++ b/scripts/test-install-worker-controller.sh
@@ -155,6 +155,7 @@ case "${1:-}" in
         [[ -z "${FAKE_FAIL_BUILD:-}" ]] || exit 46
         [[ -z "${FAKE_RUNNER_IMAGE_STATE:-}" ]] || printf '%s\n' "${FAKE_ENGINE_REF:?}" >"$FAKE_RUNNER_IMAGE_STATE"
         [[ -z "${FAKE_CONTROLLER_IMAGE_STATE:-}" ]] || printf '%s\n' "${FAKE_ENGINE_REF:?}" >"$FAKE_CONTROLLER_IMAGE_STATE"
+        [[ -z "${FAKE_DISK_USED_PERCENT_AFTER_BUILD:-}" || -z "${FAKE_DISK_USED_PERCENT_FILE:-}" ]] || printf '%s\n' "$FAKE_DISK_USED_PERCENT_AFTER_BUILD" >"$FAKE_DISK_USED_PERCENT_FILE"
         ;;
       config) [[ -z "${FAKE_FAIL_CONFIG:-}" ]] || exit 47 ;;
       logs) ;;
@@ -214,12 +215,18 @@ chmod 700 "$fake_bin/git"
 
 cat >"$fake_bin/df" <<'EOF'
 #!/usr/bin/env bash
-if [[ -n ${FAKE_DISK_USED_PERCENT:-} ]]; then
+if [[ -n ${FAKE_DISK_USED_PERCENT_FILE:-} && -f $FAKE_DISK_USED_PERCENT_FILE ]]; then
+  used=$(<"$FAKE_DISK_USED_PERCENT_FILE")
+elif [[ -n ${FAKE_DISK_USED_PERCENT:-} ]]; then
+  used=$FAKE_DISK_USED_PERCENT
+else
+  exec "$REAL_DF" "$@"
+fi
+if [[ -n ${used:-} ]]; then
   printf 'Filesystem 1024-blocks Used Available Capacity Mounted on\n'
-  printf 'fixture 100 90 10 %s%% /fixture\n' "$FAKE_DISK_USED_PERCENT"
+  printf 'fixture 100 90 10 %s%% /fixture\n' "$used"
   exit 0
 fi
-exec "$REAL_DF" "$@"
 EOF
 chmod 700 "$fake_bin/df"
 
@@ -648,6 +655,35 @@ unset FAKE_FAIL_BUILD
 if grep -Eq 'CHECKPOINT_CREATED|DRAIN_READY|ROLLBACK_' "$build_failure_output"; then fail 'candidate build failure entered the transaction'; fi
 diff -r "$build_failure_root" "$root" >/dev/null || fail 'candidate build failure changed host state'
 [[ -f "$FAKE_DOCKER_STATE" ]] || fail 'candidate build failure stopped the installed controller'
+python3 -c 'from pathlib import Path; import sys; path = Path(sys.argv[1]); path.write_text(path.read_text().replace(sys.argv[2], sys.argv[3]))' "$FAKE_CONTROLLER_ENV_FILE" "$prior_runner_image" "$FAKE_RUNNER_IMAGE"
+live_drift_build_output=$tmp/live-drift-build.out
+export FAKE_COMPOSE_LOG=$tmp/live-drift-build-compose.log
+: >"$FAKE_COMPOSE_LOG"
+export FAKE_FAIL_BUILD=1
+if "$installer" --upgrade "${base_args[@]}" --ref "$ref_two" >"$live_drift_build_output" 2>&1; then
+  fail 'live-drift candidate build failure unexpectedly succeeded'
+fi
+unset FAKE_FAIL_BUILD
+stop_line=$(grep -n -m1 '^stop|' "$FAKE_COMPOSE_LOG" | cut -d: -f1 || true)
+build_line=$(grep -n -m1 '^build|' "$FAKE_COMPOSE_LOG" | cut -d: -f1 || true)
+[[ -n "$stop_line" && -n "$build_line" && "$stop_line" -lt "$build_line" ]] || fail 'live drift runner tag was built before drain'
+grep -Fq 'DRAIN_OK managed_runners=0' "$live_drift_build_output" || fail 'live drift runner tag build did not wait for drain'
+grep -Fq 'ROLLBACK_RESTORED' "$live_drift_build_output" || fail 'live drift runner tag build failure did not restore the checkpoint'
+unset FAKE_COMPOSE_LOG
+[[ ${CI_FLEET_TEST_STOP_AFTER_LIVE_DRIFT_BUILD:-0} != 1 ]] || { printf 'LIVE_DRIFT_BUILD_REGRESSION_OK\n'; exit 0; }
+postbuild_capacity_output=$tmp/postbuild-capacity.out
+export FAKE_DISK_USED_PERCENT_FILE=$tmp/docker-disk-used-percent
+export FAKE_DISK_USED_PERCENT_AFTER_BUILD=80
+printf '79\n' >"$FAKE_DISK_USED_PERCENT_FILE"
+if "$installer" --upgrade "${base_args[@]}" --ref "$ref_two" >"$postbuild_capacity_output" 2>&1; then
+  fail 'post-build Docker capacity failure unexpectedly succeeded'
+fi
+unset FAKE_DISK_USED_PERCENT_AFTER_BUILD FAKE_DISK_USED_PERCENT_FILE
+grep -Fq 'Docker filesystem' "$postbuild_capacity_output" || fail 'post-build Docker capacity failure was not reported'
+if grep -Eq 'CHECKPOINT_CREATED|DRAIN_READY|ROLLBACK_' "$postbuild_capacity_output"; then fail 'post-build Docker capacity failure entered the transaction'; fi
+[[ -f "$FAKE_DOCKER_STATE" ]] || fail 'post-build Docker capacity failure stopped the installed controller'
+grep -Fq 'CI_FLEET_MAX_RUNNERS=1' "$rendered_env" || fail 'post-build Docker capacity failure changed installed state'
+[[ ${CI_FLEET_TEST_STOP_AFTER_POSTBUILD_CAPACITY:-0} != 1 ]] || { printf 'POSTBUILD_CAPACITY_REGRESSION_OK\n'; exit 0; }
 for environment in "$rendered_env" "$FAKE_CONTROLLER_ENV_FILE"; do
   python3 -c 'from pathlib import Path; import sys; path = Path(sys.argv[1]); path.write_text(path.read_text().replace(sys.argv[2], sys.argv[3]))' "$environment" "$prior_runner_image" "$FAKE_RUNNER_IMAGE"
 done

From c0208303763ea914e6752ab8752f2c526d149b20 Mon Sep 17 00:00:00 2001
From: Nickfost <1572453+Nickfost@users.noreply.github.com>
Date: Thu, 3 Sep 2026 15:08:06 -0500
Subject: [PATCH 06/14] fix: reject empty live runner image state

---
 scripts/install-worker-controller.sh      |  3 ++-
 scripts/test-install-worker-controller.sh | 21 +++++++++++++++++++++
 2 files changed, 23 insertions(+), 1 deletion(-)

diff --git a/scripts/install-worker-controller.sh b/scripts/install-worker-controller.sh
index 4d1ab99e..a2de03b2 100755
--- a/scripts/install-worker-controller.sh
+++ b/scripts/install-worker-controller.sh
@@ -1091,7 +1091,8 @@ perform_converge() {
     running)
       if [[ -f "$rendered_env" && $(stat -c %u "$rendered_env") == "$expected_owner" && $(stat -c %a "$rendered_env") == 600 ]] \
         && installed_runner_image=$(awk -F= '$1 == "CI_FLEET_RUNNER_IMAGE" {count++; value=substr($0, index($0, "=") + 1)} END {if (count != 1) exit 1; print value}' "$rendered_env") \
-        && live_runner_image=$(docker inspect --format '{{range .Config.Env}}{{println .}}{{end}}' "$controller_container" 2>/dev/null | awk -F= '$1 == "CI_FLEET_RUNNER_IMAGE" {count++; value=substr($0, index($0, "=") + 1)} END {if (count != 1) exit 1; print value}'); then
+        && live_runner_image=$(docker inspect --format '{{range .Config.Env}}{{println .}}{{end}}' "$controller_container" 2>/dev/null | awk -F= '$1 == "CI_FLEET_RUNNER_IMAGE" {count++; value=substr($0, index($0, "=") + 1)} END {if (count != 1) exit 1; print value}') \
+        && [[ -n "$installed_runner_image" && -n "$live_runner_image" ]]; then
         [[ "$candidate_runner_image" == "$installed_runner_image" || "$candidate_runner_image" == "$live_runner_image" ]] || build_before_drain=true
       fi
       ;;
diff --git a/scripts/test-install-worker-controller.sh b/scripts/test-install-worker-controller.sh
index bac56075..7b6ec898 100755
--- a/scripts/test-install-worker-controller.sh
+++ b/scripts/test-install-worker-controller.sh
@@ -655,6 +655,27 @@ unset FAKE_FAIL_BUILD
 if grep -Eq 'CHECKPOINT_CREATED|DRAIN_READY|ROLLBACK_' "$build_failure_output"; then fail 'candidate build failure entered the transaction'; fi
 diff -r "$build_failure_root" "$root" >/dev/null || fail 'candidate build failure changed host state'
 [[ -f "$FAKE_DOCKER_STATE" ]] || fail 'candidate build failure stopped the installed controller'
+python3 -c 'from pathlib import Path; import sys; path = Path(sys.argv[1]); path.write_text(path.read_text().replace(sys.argv[2], "CI_FLEET_RUNNER_IMAGE="))' "$FAKE_CONTROLLER_ENV_FILE" "CI_FLEET_RUNNER_IMAGE=$prior_runner_image"
+empty_live_image_output=$tmp/empty-live-image-build.out
+export FAKE_COMPOSE_LOG=$tmp/empty-live-image-build-compose.log
+: >"$FAKE_COMPOSE_LOG"
+export FAKE_FAIL_BUILD=1
+if "$installer" --upgrade "${base_args[@]}" --ref "$ref_two" >"$empty_live_image_output" 2>&1; then
+  fail 'empty-live-image candidate build failure unexpectedly succeeded'
+fi
+unset FAKE_FAIL_BUILD
+stop_line=$(grep -n -m1 '^stop|' "$FAKE_COMPOSE_LOG" | cut -d: -f1 || true)
+build_line=$(grep -n -m1 '^build|' "$FAKE_COMPOSE_LOG" | cut -d: -f1 || true)
+[[ -n "$stop_line" && -n "$build_line" && "$stop_line" -lt "$build_line" ]] || fail 'empty live runner image was accepted before drain'
+grep -Fq 'DRAIN_OK managed_runners=0' "$empty_live_image_output" || fail 'empty live runner image build did not wait for drain'
+grep -Fq 'ROLLBACK_RESTORED' "$empty_live_image_output" || fail 'empty live runner image build failure did not restore the checkpoint'
+[[ -f "$FAKE_DOCKER_STATE" ]] || fail 'empty live runner image build failure stopped the installed controller'
+grep -Fxq "CI_FLEET_RUNNER_IMAGE=$prior_runner_image" "$rendered_env" || fail 'empty live runner image build failure changed installed environment'
+grep -Fxq "CI_FLEET_RUNNER_IMAGE=$prior_runner_image" "$FAKE_CONTROLLER_ENV_FILE" || fail 'empty live runner image build failure did not restore the installed controller'
+grep -Fq 'CI_FLEET_MAX_RUNNERS=1' "$rendered_env" || fail 'empty live runner image build failure changed installed state'
+[[ $(readlink -f "$root/opt/ci-fleet/manager/current") == "$prior_manager" ]] || fail 'empty live runner image build failure changed the installed manager'
+unset FAKE_COMPOSE_LOG
+[[ ${CI_FLEET_TEST_STOP_AFTER_EMPTY_LIVE_IMAGE_BUILD:-0} != 1 ]] || { printf 'EMPTY_LIVE_IMAGE_BUILD_REGRESSION_OK\n'; exit 0; }
 python3 -c 'from pathlib import Path; import sys; path = Path(sys.argv[1]); path.write_text(path.read_text().replace(sys.argv[2], sys.argv[3]))' "$FAKE_CONTROLLER_ENV_FILE" "$prior_runner_image" "$FAKE_RUNNER_IMAGE"
 live_drift_build_output=$tmp/live-drift-build.out
 export FAKE_COMPOSE_LOG=$tmp/live-drift-build-compose.log

From 545a1313d2135dc85a35e6d11a373aeb2e8bcc2a Mon Sep 17 00:00:00 2001
From: Nickfost <1572453+Nickfost@users.noreply.github.com>
Date: Thu, 3 Sep 2026 16:29:21 -0500
Subject: [PATCH 07/14] fix: defer restartable controller image builds

---
 scripts/install-worker-controller.sh      | 16 ++++++++---
 scripts/test-install-worker-controller.sh | 33 ++++++++++++++---------
 2 files changed, 34 insertions(+), 15 deletions(-)

diff --git a/scripts/install-worker-controller.sh b/scripts/install-worker-controller.sh
index a2de03b2..e69dfc65 100755
--- a/scripts/install-worker-controller.sh
+++ b/scripts/install-worker-controller.sh
@@ -750,7 +750,15 @@ try_drain_current() {
   drain_error=
   status=$(controller_status)
   case "$status" in
-    running|''|exited|created|dead) ;;
+    running|'') ;;
+    exited|created|dead)
+      [[ -f "$drain_env" ]] || { drain_error="cannot stop restartable controller state without its rendered environment: $status"; return 1; }
+      old_release=$(current_runtime_release)
+      [[ -n "$old_release" ]] || old_release=$fallback_release
+      [[ -n "$old_release" ]] || { drain_error="cannot stop restartable controller state without its runtime release: $status"; return 1; }
+      compose "$old_release" "$drain_env" stop --timeout "$shutdown_timeout" controller >/dev/null 2>&1 || { drain_error="failed to stop restartable controller state: $status"; return 1; }
+      status=
+      ;;
     *)
       if [[ "$force_nonterminal" != true ]]; then
         drain_error="cannot safely drain controller in non-terminal state: $status"
@@ -1087,8 +1095,10 @@ perform_converge() {
   candidate_runner_image=$(awk -F= '$1 == "CI_FLEET_RUNNER_IMAGE" {count++; value=substr($0, index($0, "=") + 1)} END {if (count != 1) exit 1; print value}' "$candidate_env") || die 'rendered candidate runner image is invalid'
   [[ "$testing" != 1 ]] || expected_owner=$(id -u)
   case "$existing_status" in
-    ''|exited|created|dead) build_before_drain=true ;;
-    running)
+    '')
+      [[ "$mode" == install && ! -f "$rendered_env" && ! -f "$state_file" ]] && build_before_drain=true
+      ;;
+    running|exited|created|dead)
       if [[ -f "$rendered_env" && $(stat -c %u "$rendered_env") == "$expected_owner" && $(stat -c %a "$rendered_env") == 600 ]] \
         && installed_runner_image=$(awk -F= '$1 == "CI_FLEET_RUNNER_IMAGE" {count++; value=substr($0, index($0, "=") + 1)} END {if (count != 1) exit 1; print value}' "$rendered_env") \
         && live_runner_image=$(docker inspect --format '{{range .Config.Env}}{{println .}}{{end}}' "$controller_container" 2>/dev/null | awk -F= '$1 == "CI_FLEET_RUNNER_IMAGE" {count++; value=substr($0, index($0, "=") + 1)} END {if (count != 1) exit 1; print value}') \
diff --git a/scripts/test-install-worker-controller.sh b/scripts/test-install-worker-controller.sh
index 7b6ec898..da99f480 100755
--- a/scripts/test-install-worker-controller.sh
+++ b/scripts/test-install-worker-controller.sh
@@ -647,14 +647,21 @@ done
 build_failure_output=$tmp/build-failure.out
 build_failure_root=$tmp/build-failure-root
 cp -a "$root" "$build_failure_root"
+export FAKE_COMPOSE_LOG=$tmp/stopped-distinct-tag-build-compose.log
+: >"$FAKE_COMPOSE_LOG"
+printf 'exited\n' >"$FAKE_CONTROLLER_STATUS_FILE"
 export FAKE_FAIL_BUILD=1
 if "$installer" --upgrade "${base_args[@]}" --ref "$ref_two" >"$build_failure_output" 2>&1; then
   fail 'candidate build failure unexpectedly succeeded'
 fi
 unset FAKE_FAIL_BUILD
 if grep -Eq 'CHECKPOINT_CREATED|DRAIN_READY|ROLLBACK_' "$build_failure_output"; then fail 'candidate build failure entered the transaction'; fi
+grep -q '^build|' "$FAKE_COMPOSE_LOG" || fail 'stopped controller with distinct trusted tags did not prebuild'
+if grep -q '^stop|' "$FAKE_COMPOSE_LOG"; then fail 'stopped controller with distinct trusted tags was stopped before prebuild'; fi
 diff -r "$build_failure_root" "$root" >/dev/null || fail 'candidate build failure changed host state'
 [[ -f "$FAKE_DOCKER_STATE" ]] || fail 'candidate build failure stopped the installed controller'
+rm -f "$FAKE_CONTROLLER_STATUS_FILE"
+unset FAKE_COMPOSE_LOG
 python3 -c 'from pathlib import Path; import sys; path = Path(sys.argv[1]); path.write_text(path.read_text().replace(sys.argv[2], "CI_FLEET_RUNNER_IMAGE="))' "$FAKE_CONTROLLER_ENV_FILE" "CI_FLEET_RUNNER_IMAGE=$prior_runner_image"
 empty_live_image_output=$tmp/empty-live-image-build.out
 export FAKE_COMPOSE_LOG=$tmp/empty-live-image-build-compose.log
@@ -716,22 +723,27 @@ fi
 unset FAKE_FAIL_CONFIG
 if grep -Eq 'CHECKPOINT_CREATED|DRAIN_READY|ROLLBACK_' "$config_failure_output"; then fail 'candidate Compose validation failure entered the transaction'; fi
 [[ -f "$FAKE_DOCKER_STATE" ]] || fail 'candidate Compose validation failure stopped the installed controller'
-live_tag_build_output=$tmp/live-tag-build.out
-export FAKE_COMPOSE_LOG=$tmp/live-tag-build-compose.log
+restartable_tag_build_output=$tmp/restartable-tag-build.out
+export FAKE_COMPOSE_LOG=$tmp/restartable-tag-build-compose.log
 : >"$FAKE_COMPOSE_LOG"
+printf 'exited\n' >"$FAKE_CONTROLLER_STATUS_FILE"
 export FAKE_FAIL_BUILD=1
-if "$installer" --upgrade "${base_args[@]}" --ref "$ref_two" >"$live_tag_build_output" 2>&1; then
-  fail 'live-tag candidate build failure unexpectedly succeeded'
+if "$installer" --upgrade "${base_args[@]}" --ref "$ref_two" >"$restartable_tag_build_output" 2>&1; then
+  fail 'restartable-tag candidate build failure unexpectedly succeeded'
 fi
 unset FAKE_FAIL_BUILD
 stop_line=$(grep -n -m1 '^stop|' "$FAKE_COMPOSE_LOG" | cut -d: -f1 || true)
 build_line=$(grep -n -m1 '^build|' "$FAKE_COMPOSE_LOG" | cut -d: -f1 || true)
-[[ -n "$stop_line" && -n "$build_line" && "$stop_line" -lt "$build_line" ]] || fail 'live runner tag was built before drain'
-grep -Fq 'DRAIN_OK managed_runners=0' "$live_tag_build_output" || fail 'live runner tag build did not wait for drain'
-grep -Fq 'ROLLBACK_RESTORED' "$live_tag_build_output" || fail 'live runner tag build failure did not restore the checkpoint'
-grep -Fq 'CI_FLEET_MAX_RUNNERS=1' "$rendered_env" || fail 'live runner tag build failure changed installed state'
+[[ -n "$stop_line" && -n "$build_line" && "$stop_line" -lt "$build_line" ]] || fail 'restartable controller runner tag was built before stop'
+grep -Fq 'DRAIN_OK managed_runners=0' "$restartable_tag_build_output" || fail 'restartable controller runner tag build did not wait for drain'
+grep -Fq 'ROLLBACK_RESTORED' "$restartable_tag_build_output" || fail 'restartable controller runner tag build failure did not restore the checkpoint'
+[[ -f "$FAKE_DOCKER_STATE" && ! -f "$FAKE_CONTROLLER_STATUS_FILE" ]] || fail 'restartable controller runner tag build failure did not restore the installed controller'
+grep -Fxq "CI_FLEET_RUNNER_IMAGE=$FAKE_RUNNER_IMAGE" "$rendered_env" || fail 'restartable controller runner tag build failure changed installed environment'
+grep -Fxq "CI_FLEET_RUNNER_IMAGE=$FAKE_RUNNER_IMAGE" "$FAKE_CONTROLLER_ENV_FILE" || fail 'restartable controller runner tag build failure did not restore the installed controller environment'
+grep -Fq 'CI_FLEET_MAX_RUNNERS=1' "$rendered_env" || fail 'restartable controller runner tag build failure changed installed state'
+[[ $(readlink -f "$root/opt/ci-fleet/manager/current") == "$prior_manager" ]] || fail 'restartable controller runner tag build failure changed the installed manager'
 unset FAKE_COMPOSE_LOG
-[[ ${CI_FLEET_TEST_STOP_AFTER_LIVE_TAG_BUILD:-0} != 1 ]] || { printf 'LIVE_TAG_BUILD_REGRESSION_OK\n'; exit 0; }
+[[ ${CI_FLEET_TEST_STOP_AFTER_RESTARTABLE_TAG_BUILD:-0} != 1 ]] || { printf 'RESTARTABLE_TAG_BUILD_REGRESSION_OK\n'; exit 0; }
 terminate_upgrade() {
   local output=$1 marker=$2 second_term_marker=${3:-} pid status=0 attempt
   export CI_FLEET_TEST_PAUSE_AFTER_DRAIN_FILE=$marker
@@ -854,10 +866,7 @@ grep -Fq 'CI_FLEET_MAX_RUNNERS=1' "$root/etc/ci-fleet/ci-fleet.env" || fail 'rol
 
 ref_three=$(write_config drained 2 2)
 printf 'dead\n' >"$FAKE_CONTROLLER_STATUS_FILE"
-export FAKE_STOP_FAIL=$tmp/stop-dead-fails
-: >"$FAKE_STOP_FAIL"
 expect_success "$installer" --upgrade "${base_args[@]}" --ref "$ref_three" >/dev/null
-unset FAKE_STOP_FAIL
 [[ ! -f "$FAKE_DOCKER_STATE" && ! -f "$FAKE_CONTROLLER_STATUS_FILE" ]] || fail 'non-active convergence retained a dead controller'
 grep -Fq 'CI_FLEET_CONTROLLER_STATE=drained' "$root/etc/ci-fleet/ci-fleet.env" || fail 'drained state was not rendered'
 grep -Fq 'CI_FLEET_MAX_RUNNERS=0' "$root/etc/ci-fleet/ci-fleet.env" || fail 'drained controller retained effective capacity'

From 3621b5ae9f203d0f7a8d9f5a5ecee53c6a589527 Mon Sep 17 00:00:00 2001
From: Nickfost <1572453+Nickfost@users.noreply.github.com>
Date: Thu, 3 Sep 2026 20:10:45 -0500
Subject: [PATCH 08/14] fix: unpause controller before rollback stop

---
 scripts/install-worker-controller.sh      | 3 +++
 scripts/test-install-worker-controller.sh | 3 ++-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/scripts/install-worker-controller.sh b/scripts/install-worker-controller.sh
index e69dfc65..c602dc49 100755
--- a/scripts/install-worker-controller.sh
+++ b/scripts/install-worker-controller.sh
@@ -768,6 +768,9 @@ try_drain_current() {
       old_release=$(current_runtime_release)
       [[ -n "$old_release" ]] || old_release=$fallback_release
       [[ -n "$old_release" ]] || { drain_error='cannot stop a non-terminal candidate without its runtime release'; return 1; }
+      if [[ "$status" == paused ]]; then
+        compose "$old_release" "$drain_env" unpause controller >/dev/null 2>&1 || { drain_error="failed to unpause non-terminal candidate state: $status"; return 1; }
+      fi
       compose "$old_release" "$drain_env" stop --timeout "$shutdown_timeout" controller >/dev/null 2>&1 || { drain_error="failed to stop non-terminal candidate state: $status"; return 1; }
       status=
       ;;
diff --git a/scripts/test-install-worker-controller.sh b/scripts/test-install-worker-controller.sh
index da99f480..19256a08 100755
--- a/scripts/test-install-worker-controller.sh
+++ b/scripts/test-install-worker-controller.sh
@@ -48,7 +48,7 @@ case "${1:-}" in
       [[ -n "${FAKE_CONTROLLER_IMAGE_ID_FILE:-}" && -f "$FAKE_CONTROLLER_IMAGE_ID_FILE" ]] || exit 1
       cat "$FAKE_CONTROLLER_IMAGE_ID_FILE"
     elif [[ "$*" == *'.State.Status'* ]]; then
-      if [[ -n "$status_file" && -f "$status_file" ]]; then cat "$status_file"; else printf '%s\n' "${FAKE_CONTROLLER_STATUS:-running}"; fi
+      if [[ -n "$paused_state" && -f "$paused_state" ]]; then printf 'paused\n'; elif [[ -n "$status_file" && -f "$status_file" ]]; then cat "$status_file"; else printf '%s\n' "${FAKE_CONTROLLER_STATUS:-running}"; fi
     elif [[ "$*" == *'.State.Paused'* ]]; then
       if [[ -n "$paused_state" && -f "$paused_state" ]]; then printf 'true\n'; else printf 'false\n'; fi
     else
@@ -130,6 +130,7 @@ case "${1:-}" in
         fi
         ;;
       stop)
+        if [[ -n "$paused_state" && -f "$paused_state" ]]; then exit 42; fi
         if [[ -n "${FAKE_STOP_FAIL:-}" && -f "$FAKE_STOP_FAIL" ]]; then exit 42; fi
         rm -f "$state"; [[ -z "$status_file" ]] || rm -f "$status_file"; [[ -z "$paused_state" ]] || rm -f "$paused_state"; [[ -z "${FAKE_CONTROLLER_PROVENANCE_FILE:-}" ]] || rm -f "$FAKE_CONTROLLER_PROVENANCE_FILE"; [[ -z "${FAKE_CONTROLLER_IMAGE_ID_FILE:-}" ]] || rm -f "$FAKE_CONTROLLER_IMAGE_ID_FILE"; [[ -z "${FAKE_CONTROLLER_ENV_FILE:-}" ]] || rm -f "$FAKE_CONTROLLER_ENV_FILE"
         [[ -z "${FAKE_ACTIVE_MANAGED_AFTER_STOP:-}" ]] || : >"$FAKE_ACTIVE_MANAGED_AFTER_STOP"

From 65d000600b035f3208722daf8c6539398411b09c Mon Sep 17 00:00:00 2001
From: Nickfost <1572453+Nickfost@users.noreply.github.com>
Date: Thu, 3 Sep 2026 23:59:38 -0500
Subject: [PATCH 09/14] fix: restore checkpointed image tags on rollback

---
 scripts/install-worker-controller.sh      |  76 +++++++++++-
 scripts/test-install-worker-controller.sh | 141 +++++++++++++++++++---
 2 files changed, 191 insertions(+), 26 deletions(-)

diff --git a/scripts/install-worker-controller.sh b/scripts/install-worker-controller.sh
index c602dc49..c05ac2cb 100755
--- a/scripts/install-worker-controller.sh
+++ b/scripts/install-worker-controller.sh
@@ -699,8 +699,39 @@ build_candidate() {
   compose "$release_dir" "$candidate_env" build runner-image controller
 }
 
+load_checkpoint_images() {
+  local environment=$1 image_ids=${2:-} output expected=2
+  output=$(python3 - "$environment" "$image_ids" "$repo_root/scripts" <<'PY'
+import re
+import sys
+from pathlib import Path
+
+sys.path.insert(0, sys.argv[3])
+from desired_state import parse_env
+
+image_keys = ("CI_FLEET_RUNNER_IMAGE", "CI_FLEET_CONTROLLER_IMAGE")
+values = parse_env(Path(sys.argv[1]), allow_unknown=True)
+if any(not values.get(key) for key in image_keys):
+    raise SystemExit(1)
+for key in image_keys:
+    print(values[key])
+
+if sys.argv[2]:
+    id_keys = tuple(f"{key}_ID" for key in image_keys)
+    values = parse_env(Path(sys.argv[2]), allow_unknown=True)
+    if set(values) != set(id_keys) or any(not re.fullmatch(r"sha256:[0-9a-f]{64}", values[key]) for key in id_keys):
+        raise SystemExit(1)
+    for key in id_keys:
+        print(values[key])
+PY
+  ) || return 1
+  [[ -z "$image_ids" ]] || expected=4
+  mapfile -t checkpoint_images <<<"$output"
+  [[ ${#checkpoint_images[@]} == "$expected" ]]
+}
+
 make_checkpoint() {
-  local timestamp target unit timer final_checkpoint staged_checkpoint
+  local timestamp target unit timer final_checkpoint staged_checkpoint expected_owner=0 runner_id controller_id
   timestamp=$(date -u +%Y%m%dT%H%M%SZ)
   final_checkpoint=$checkpoints_dir/${timestamp}-$$
   install -d -m 0700 "$checkpoints_dir"
@@ -708,7 +739,18 @@ make_checkpoint() {
   staging_paths+=("$staged_checkpoint")
   checkpoint_dir=$staged_checkpoint
   install -d -m 0700 "$checkpoint_dir/systemd"
-  [[ ! -f "$rendered_env" ]] || install -m 0600 "$rendered_env" "$checkpoint_dir/ci-fleet.env"
+  if [[ -f "$rendered_env" ]]; then
+    [[ "$testing" != 1 ]] || expected_owner=$(id -u)
+    [[ $(stat -c %u "$rendered_env") == "$expected_owner" && $(stat -c %a "$rendered_env") == 600 ]] || die "rendered environment must be owned by root with mode 0600: $rendered_env"
+    install -m 0600 "$rendered_env" "$checkpoint_dir/ci-fleet.env"
+    load_checkpoint_images "$checkpoint_dir/ci-fleet.env" || die 'installed image tags are invalid'
+    runner_id=$(docker image inspect --format '{{.Id}}' "${checkpoint_images[0]}" 2>/dev/null) || die 'installed runner image mapping is unavailable'
+    controller_id=$(docker image inspect --format '{{.Id}}' "${checkpoint_images[1]}" 2>/dev/null) || die 'installed controller image mapping is unavailable'
+    [[ "$runner_id" =~ ^sha256:[0-9a-f]{64}$ ]] || die 'installed runner image ID is invalid'
+    [[ "$controller_id" =~ ^sha256:[0-9a-f]{64}$ ]] || die 'installed controller image ID is invalid'
+    printf 'CI_FLEET_RUNNER_IMAGE_ID=%s\nCI_FLEET_CONTROLLER_IMAGE_ID=%s\n' "$runner_id" "$controller_id" >"$checkpoint_dir/image-ids.env"
+    chmod 0600 "$checkpoint_dir/image-ids.env"
+  fi
   [[ ! -f "$state_file" ]] || install -m 0600 "$state_file" "$checkpoint_dir/install-state.json"
   target=$(current_runtime_release)
   if [[ -n "$target" ]]; then
@@ -969,9 +1011,20 @@ restore_systemd_snapshot() {
 }
 
 restore_checkpoint() {
-  local target restored_state failed=0 checkpoint_release='' drain_env=$rendered_env drain_release=''
+  local target restored_state actual failed=0 checkpoint_release='' drain_env=$rendered_env drain_release='' restore_images=false
   [[ -n "$checkpoint_dir" && -d "$checkpoint_dir" ]] || return 1
   [[ ! -f "$checkpoint_dir/release-target" ]] || checkpoint_release=$(<"$checkpoint_dir/release-target")
+  if [[ -f "$checkpoint_dir/ci-fleet.env" ]]; then
+    if [[ ! -f "$checkpoint_dir/image-ids.env" || -L "$checkpoint_dir/image-ids.env" || $(stat -c %a "$checkpoint_dir/image-ids.env") != 600 ]] \
+      || ! load_checkpoint_images "$checkpoint_dir/ci-fleet.env" "$checkpoint_dir/image-ids.env"; then
+      note 'ROLLBACK_FAILED reason=checkpoint image mappings are invalid'
+      return 1
+    fi
+    restore_images=true
+  elif [[ -e "$checkpoint_dir/image-ids.env" || -L "$checkpoint_dir/image-ids.env" ]]; then
+    note 'ROLLBACK_FAILED reason=checkpoint image mappings are invalid'
+    return 1
+  fi
   drain_release=$(current_runtime_release)
   if [[ -f "$rendered_env" ]]; then
     load_installed_controller_identity "$temporary/no-install-state" "$rendered_env"
@@ -1021,6 +1074,14 @@ restore_checkpoint() {
   else
     rm -f "$manager_current" || failed=1
   fi
+  if $restore_images; then
+    docker image tag "${checkpoint_images[2]}" "${checkpoint_images[0]}" || failed=1
+    docker image tag "${checkpoint_images[3]}" "${checkpoint_images[1]}" || failed=1
+    actual=$(docker image inspect --format '{{.Id}}' "${checkpoint_images[0]}" 2>/dev/null) || failed=1
+    [[ "$actual" == "${checkpoint_images[2]}" ]] || failed=1
+    actual=$(docker image inspect --format '{{.Id}}' "${checkpoint_images[1]}" 2>/dev/null) || failed=1
+    [[ "$actual" == "${checkpoint_images[3]}" ]] || failed=1
+  fi
   restore_systemd_snapshot || failed=1
   if [[ -n "$release_dir" && -f "$rendered_env" ]]; then
     restored_state=$(awk -F= '$1 == "CI_FLEET_CONTROLLER_STATE" {print $2}' "$rendered_env") || failed=1
@@ -1075,7 +1136,7 @@ perform_check() {
 }
 
 perform_converge() {
-  local count existing_status candidate_runner_image installed_runner_image live_runner_image expected_owner=0
+  local count existing_status candidate_runner_image candidate_controller_image installed_runner_image installed_controller_image live_runner_image expected_owner=0
   local desired_controller_id=$controller_id build_before_drain=false
   if [[ "$mode" == upgrade && ! -f "$state_file" ]]; then
     die '--upgrade requires an existing managed installation; use --install or --adopt'
@@ -1096,6 +1157,7 @@ perform_converge() {
   install_release
   compose "$release_dir" "$candidate_env" config --quiet
   candidate_runner_image=$(awk -F= '$1 == "CI_FLEET_RUNNER_IMAGE" {count++; value=substr($0, index($0, "=") + 1)} END {if (count != 1) exit 1; print value}' "$candidate_env") || die 'rendered candidate runner image is invalid'
+  candidate_controller_image=$(awk -F= '$1 == "CI_FLEET_CONTROLLER_IMAGE" {count++; value=substr($0, index($0, "=") + 1)} END {if (count != 1) exit 1; print value}' "$candidate_env") || die 'rendered candidate controller image is invalid'
   [[ "$testing" != 1 ]] || expected_owner=$(id -u)
   case "$existing_status" in
     '')
@@ -1104,9 +1166,11 @@ perform_converge() {
     running|exited|created|dead)
       if [[ -f "$rendered_env" && $(stat -c %u "$rendered_env") == "$expected_owner" && $(stat -c %a "$rendered_env") == 600 ]] \
         && installed_runner_image=$(awk -F= '$1 == "CI_FLEET_RUNNER_IMAGE" {count++; value=substr($0, index($0, "=") + 1)} END {if (count != 1) exit 1; print value}' "$rendered_env") \
+        && installed_controller_image=$(awk -F= '$1 == "CI_FLEET_CONTROLLER_IMAGE" {count++; value=substr($0, index($0, "=") + 1)} END {if (count != 1) exit 1; print value}' "$rendered_env") \
         && live_runner_image=$(docker inspect --format '{{range .Config.Env}}{{println .}}{{end}}' "$controller_container" 2>/dev/null | awk -F= '$1 == "CI_FLEET_RUNNER_IMAGE" {count++; value=substr($0, index($0, "=") + 1)} END {if (count != 1) exit 1; print value}') \
-        && [[ -n "$installed_runner_image" && -n "$live_runner_image" ]]; then
-        [[ "$candidate_runner_image" == "$installed_runner_image" || "$candidate_runner_image" == "$live_runner_image" ]] || build_before_drain=true
+        && [[ -n "$installed_runner_image" && -n "$installed_controller_image" && -n "$live_runner_image" ]]; then
+        if [[ "$candidate_runner_image" != "$installed_runner_image" && "$candidate_runner_image" != "$live_runner_image" \
+          && "$candidate_controller_image" != "$installed_controller_image" ]]; then build_before_drain=true; fi
       fi
       ;;
   esac
diff --git a/scripts/test-install-worker-controller.sh b/scripts/test-install-worker-controller.sh
index 19256a08..ca2f2900 100755
--- a/scripts/test-install-worker-controller.sh
+++ b/scripts/test-install-worker-controller.sh
@@ -70,18 +70,38 @@ case "${1:-}" in
     exit 0
     ;;
   image)
-    [[ "${2:-}" == inspect ]] || exit 1
-    image=${!#}
-    [[ -z "${FAKE_IMAGE_INSPECT_LOG:-}" ]] || printf '%s\n' "$image" >>"$FAKE_IMAGE_INSPECT_LOG"
-    if [[ "$image" == "${FAKE_RUNNER_IMAGE:-}" ]]; then
-      image_state=${FAKE_RUNNER_IMAGE_STATE:-}
-    elif [[ "$image" == "${FAKE_CONTROLLER_IMAGE:-}" ]]; then
-      image_state=${FAKE_CONTROLLER_IMAGE_STATE:-}
-    else
-      exit 1
-    fi
-    [[ -f "$image_state" ]] || exit 1
-    if [[ "$*" == *'{{.Id}}'* ]]; then printf 'sha256:%s\n' "$(<"$image_state")"; else cat "$image_state"; fi
+    case "${2:-}" in
+      inspect)
+        image=${!#}
+        [[ -z "${FAKE_IMAGE_INSPECT_LOG:-}" ]] || printf '%s\n' "$image" >>"$FAKE_IMAGE_INSPECT_LOG"
+        if [[ "$image" == "${FAKE_RUNNER_IMAGE:-}" || "$image" == "${FAKE_PRIOR_RUNNER_IMAGE:-}" || "$image" == "${FAKE_PREVIOUS_RUNNER_IMAGE:-}" ]]; then
+          image_state=${FAKE_RUNNER_IMAGE_STATE:-}
+          image_id_state=${FAKE_RUNNER_IMAGE_ID_STATE:-}
+        elif [[ "$image" == "${FAKE_CONTROLLER_IMAGE:-}" || "$image" == "${FAKE_PRIOR_CONTROLLER_IMAGE:-}" || "$image" == "${FAKE_PREVIOUS_CONTROLLER_IMAGE:-}" ]]; then
+          image_state=${FAKE_CONTROLLER_IMAGE_STATE:-}
+          image_id_state=${FAKE_CONTROLLER_IMAGE_ID_STATE:-}
+        else
+          exit 1
+        fi
+        if [[ "$*" == *'{{.Id}}'* ]]; then [[ -f "$image_id_state" ]] && cat "$image_id_state"; else [[ -f "$image_state" ]] && cat "$image_state"; fi
+        ;;
+      tag)
+        image_id=${3:-}
+        image=${4:-}
+        [[ -n "${FAKE_AVAILABLE_IMAGE_IDS:-}" && -f "$FAKE_AVAILABLE_IMAGE_IDS" ]] || exit 1
+        grep -Fxq "$image_id" "$FAKE_AVAILABLE_IMAGE_IDS" || exit 1
+        if [[ "$image" == "${FAKE_RUNNER_IMAGE:-}" || "$image" == "${FAKE_PRIOR_RUNNER_IMAGE:-}" || "$image" == "${FAKE_PREVIOUS_RUNNER_IMAGE:-}" ]]; then
+          image_id_state=${FAKE_RUNNER_IMAGE_ID_STATE:-}
+        elif [[ "$image" == "${FAKE_CONTROLLER_IMAGE:-}" || "$image" == "${FAKE_PRIOR_CONTROLLER_IMAGE:-}" || "$image" == "${FAKE_PREVIOUS_CONTROLLER_IMAGE:-}" ]]; then
+          image_id_state=${FAKE_CONTROLLER_IMAGE_ID_STATE:-}
+        else
+          exit 1
+        fi
+        printf '%s\n' "$image_id" >"$image_id_state"
+        [[ -z "${FAKE_COMPOSE_LOG:-}" ]] || printf 'image-tag|%s|%s\n' "$image_id" "$image" >>"$FAKE_COMPOSE_LOG"
+        ;;
+      *) exit 1 ;;
+    esac
     ;;
   rm)
     (($# >= 2)) || exit 1
@@ -120,7 +140,7 @@ case "${1:-}" in
         fi
         : >"$state"
         [[ -z "${FAKE_CONTROLLER_PROVENANCE_FILE:-}" ]] || printf '%s\n' "${FAKE_ENGINE_REF:?}" >"$FAKE_CONTROLLER_PROVENANCE_FILE"
-        [[ -z "${FAKE_CONTROLLER_IMAGE_ID_FILE:-}" ]] || printf 'sha256:%s\n' "${FAKE_ENGINE_REF:?}" >"$FAKE_CONTROLLER_IMAGE_ID_FILE"
+        [[ -z "${FAKE_CONTROLLER_IMAGE_ID_FILE:-}" || -z "${FAKE_CONTROLLER_IMAGE_ID_STATE:-}" ]] || cp "$FAKE_CONTROLLER_IMAGE_ID_STATE" "$FAKE_CONTROLLER_IMAGE_ID_FILE"
         [[ -z "${FAKE_CONTROLLER_ENV_FILE:-}" ]] || cp "$env_file" "$FAKE_CONTROLLER_ENV_FILE"
         if [[ -n "${FAKE_RESTART_AFTER_UP:-}" && -f "$FAKE_RESTART_AFTER_UP" ]]; then
           rm -f "$FAKE_RESTART_AFTER_UP"
@@ -154,8 +174,16 @@ case "${1:-}" in
         ;;
       build)
         [[ -z "${FAKE_FAIL_BUILD:-}" ]] || exit 46
+        if [[ -n "${FAKE_PARTIAL_BUILD_FAIL:-}" ]]; then
+          printf '%s\n' "${FAKE_PARTIAL_RUNNER_IMAGE_ID:?}" >>"${FAKE_AVAILABLE_IMAGE_IDS:?}"
+          printf '%s\n' "$FAKE_PARTIAL_RUNNER_IMAGE_ID" >"${FAKE_RUNNER_IMAGE_ID_STATE:?}"
+          exit 46
+        fi
         [[ -z "${FAKE_RUNNER_IMAGE_STATE:-}" ]] || printf '%s\n' "${FAKE_ENGINE_REF:?}" >"$FAKE_RUNNER_IMAGE_STATE"
         [[ -z "${FAKE_CONTROLLER_IMAGE_STATE:-}" ]] || printf '%s\n' "${FAKE_ENGINE_REF:?}" >"$FAKE_CONTROLLER_IMAGE_STATE"
+        [[ -z "${FAKE_RUNNER_IMAGE_ID_STATE:-}" ]] || printf '%s\n' "${FAKE_CANDIDATE_RUNNER_IMAGE_ID:?}" >"$FAKE_RUNNER_IMAGE_ID_STATE"
+        [[ -z "${FAKE_CONTROLLER_IMAGE_ID_STATE:-}" ]] || printf '%s\n' "${FAKE_CANDIDATE_CONTROLLER_IMAGE_ID:?}" >"$FAKE_CONTROLLER_IMAGE_ID_STATE"
+        [[ -z "${FAKE_AVAILABLE_IMAGE_IDS:-}" ]] || printf '%s\n%s\n' "$FAKE_CANDIDATE_RUNNER_IMAGE_ID" "$FAKE_CANDIDATE_CONTROLLER_IMAGE_ID" >>"$FAKE_AVAILABLE_IMAGE_IDS"
         [[ -z "${FAKE_DISK_USED_PERCENT_AFTER_BUILD:-}" || -z "${FAKE_DISK_USED_PERCENT_FILE:-}" ]] || printf '%s\n' "$FAKE_DISK_USED_PERCENT_AFTER_BUILD" >"$FAKE_DISK_USED_PERCENT_FILE"
         ;;
       config) [[ -z "${FAKE_FAIL_CONFIG:-}" ]] || exit 47 ;;
@@ -287,6 +315,12 @@ export FAKE_RUNNER_IMAGE=$runner_image
 export FAKE_CONTROLLER_IMAGE=ci-fleet-controller:${engine_ref:0:12}
 export FAKE_RUNNER_IMAGE_STATE=$tmp/runner-image-present
 export FAKE_CONTROLLER_IMAGE_STATE=$tmp/controller-image-present
+export FAKE_RUNNER_IMAGE_ID_STATE=$tmp/runner-image-id
+export FAKE_CONTROLLER_IMAGE_ID_STATE=$tmp/controller-image-id
+export FAKE_AVAILABLE_IMAGE_IDS=$tmp/available-image-ids
+export FAKE_CANDIDATE_RUNNER_IMAGE_ID=sha256:1111111111111111111111111111111111111111111111111111111111111111
+export FAKE_CANDIDATE_CONTROLLER_IMAGE_ID=sha256:2222222222222222222222222222222222222222222222222222222222222222
+export FAKE_PARTIAL_RUNNER_IMAGE_ID=sha256:3333333333333333333333333333333333333333333333333333333333333333
 export FAKE_IMAGE_INSPECT_LOG=$tmp/image-inspects
 for dockerfile in "$repo_root/controller/Dockerfile" "$repo_root/runner/Dockerfile"; do
   grep -Fq "LABEL org.opencontainers.image.revision=\"\${CI_FLEET_COMMIT}\"" "$dockerfile" || fail "managed image lacks engine provenance label: $dockerfile"
@@ -421,6 +455,7 @@ grep -Fq 'CONVERGED mode=install' <<<"$first" || fail 'fresh install did not con
 [[ -L "$root/opt/ci-fleet/current" && -f "$root/var/lib/ci-fleet/install-state.json" ]] || fail 'fresh install state is incomplete'
 [[ $(readlink -f "$root/opt/ci-fleet/manager/current") == "$root/opt/ci-fleet/manager/releases/$engine_ref" ]] || fail 'installer manager did not activate the desired engine release'
 [[ -f "$FAKE_DOCKER_STATE" ]] || fail 'active controller was not started'
+if find "$root/var/lib/ci-fleet/checkpoints" -name image-ids.env -print -quit | grep -q .; then fail 'fresh install checkpoint stored a prior image map'; fi
 install_state=$root/var/lib/ci-fleet/install-state.json
 chmod 644 "$install_state"
 expect_failure 'install state must be owned by root with mode 0600' env CI_FLEET_INSTALL_STATE_FILE="$install_state" CI_FLEET_INSTALLER="$installer" "$repo_root/scripts/check-installed-state.sh"
@@ -430,8 +465,9 @@ expect_success "$installer" --install "${base_args[@]}" --ref "$ref_one" >/dev/n
 rendered_env=$root/etc/ci-fleet/ci-fleet.env
 chmod 644 "$rendered_env"
 expect_failure 'DRIFT rendered_environment' "$installer" --check "${base_args[@]}" --ref "$ref_one"
-expect_success "$installer" --install "${base_args[@]}" --ref "$ref_one" >/dev/null
-[[ $(stat -c %a "$rendered_env") == 600 ]] || fail 'convergence did not repair rendered-environment mode'
+expect_failure 'rendered environment must be owned by root with mode 0600' "$installer" --install "${base_args[@]}" --ref "$ref_one"
+[[ $(stat -c %a "$rendered_env") == 644 ]] || fail 'untrusted rendered environment was changed before checkpoint validation'
+chmod 600 "$rendered_env"
 export FAKE_REQUIRE_LOCAL_DOCKER_ENDPOINT=1
 manual_health_result=0
 "$repo_root/scripts/healthcheck.sh" >/dev/null || manual_health_result=$?
@@ -506,7 +542,7 @@ expect_success "$installer" --install "${base_args[@]}" --ref "$ref_one" >/dev/n
 printf 'sha256:%040d\n' 0 >"$FAKE_CONTROLLER_IMAGE_ID_FILE"
 expect_failure 'DRIFT controller_runtime' "$installer" --check "${base_args[@]}" --ref "$ref_one"
 expect_success "$installer" --install "${base_args[@]}" --ref "$ref_one" >/dev/null
-[[ $(<"$FAKE_CONTROLLER_IMAGE_ID_FILE") == "sha256:$engine_ref" ]] || fail 'controller convergence did not restore live image identity'
+[[ $(<"$FAKE_CONTROLLER_IMAGE_ID_FILE") == "$FAKE_CANDIDATE_CONTROLLER_IMAGE_ID" ]] || fail 'controller convergence did not restore live image identity'
 python3 -c 'from pathlib import Path; import sys; path = Path(sys.argv[1]); path.write_text(path.read_text().replace("CI_FLEET_MAX_RUNNERS=1", "CI_FLEET_MAX_RUNNERS=9"))' "$FAKE_CONTROLLER_ENV_FILE"
 grep -Fxq 'CI_FLEET_MAX_RUNNERS=9' "$FAKE_CONTROLLER_ENV_FILE" || fail 'live-environment fixture did not mutate'
 expect_failure 'DRIFT controller_runtime' "$installer" --check "${base_args[@]}" --ref "$ref_one"
@@ -642,9 +678,13 @@ expect_failure 'DRIFT maintenance_timers' "$installer" --check "${base_args[@]}"
 
 ref_two=$(write_config active 2 2)
 prior_runner_image=ci-fleet-runner:prior
+prior_controller_image=ci-fleet-controller:prior
+export FAKE_PRIOR_RUNNER_IMAGE=$prior_runner_image
+export FAKE_PRIOR_CONTROLLER_IMAGE=$prior_controller_image
 for environment in "$rendered_env" "$FAKE_CONTROLLER_ENV_FILE"; do
   python3 -c 'from pathlib import Path; import sys; path = Path(sys.argv[1]); path.write_text(path.read_text().replace(sys.argv[2], sys.argv[3]))' "$environment" "$FAKE_RUNNER_IMAGE" "$prior_runner_image"
 done
+python3 -c 'from pathlib import Path; import sys; path = Path(sys.argv[1]); path.write_text(path.read_text().replace(sys.argv[2], sys.argv[3]))' "$rendered_env" "$FAKE_CONTROLLER_IMAGE" "$prior_controller_image"
 build_failure_output=$tmp/build-failure.out
 build_failure_root=$tmp/build-failure-root
 cp -a "$root" "$build_failure_root"
@@ -661,7 +701,24 @@ grep -q '^build|' "$FAKE_COMPOSE_LOG" || fail 'stopped controller with distinct
 if grep -q '^stop|' "$FAKE_COMPOSE_LOG"; then fail 'stopped controller with distinct trusted tags was stopped before prebuild'; fi
 diff -r "$build_failure_root" "$root" >/dev/null || fail 'candidate build failure changed host state'
 [[ -f "$FAKE_DOCKER_STATE" ]] || fail 'candidate build failure stopped the installed controller'
-rm -f "$FAKE_CONTROLLER_STATUS_FILE"
+unset FAKE_COMPOSE_LOG
+python3 -c 'from pathlib import Path; import sys; path = Path(sys.argv[1]); path.write_text(path.read_text().replace(sys.argv[2], sys.argv[3]))' "$rendered_env" "$prior_controller_image" "$FAKE_CONTROLLER_IMAGE"
+same_controller_output=$tmp/same-controller-build.out
+export FAKE_COMPOSE_LOG=$tmp/same-controller-build-compose.log
+: >"$FAKE_COMPOSE_LOG"
+export FAKE_FAIL_BUILD=1
+if "$installer" --upgrade "${base_args[@]}" --ref "$ref_two" >"$same_controller_output" 2>&1; then
+  fail 'same-controller-tag build failure unexpectedly succeeded'
+fi
+unset FAKE_FAIL_BUILD
+stop_line=$(grep -n -m1 '^stop|' "$FAKE_COMPOSE_LOG" | cut -d: -f1 || true)
+build_line=$(grep -n -m1 '^build|' "$FAKE_COMPOSE_LOG" | cut -d: -f1 || true)
+[[ -n "$stop_line" && -n "$build_line" && "$stop_line" -lt "$build_line" ]] || fail 'installed controller tag was rebuilt before checkpoint and stop'
+grep -Fq 'CHECKPOINT_CREATED' "$same_controller_output" || fail 'same controller tag was rebuilt without a checkpoint'
+grep -Fq 'ROLLBACK_RESTORED' "$same_controller_output" || fail 'same-controller-tag build failure did not restore the checkpoint'
+grep -Fxq "CI_FLEET_RUNNER_IMAGE=$prior_runner_image" "$rendered_env" || fail 'same-controller-tag build failure changed installed runner state'
+grep -Fxq "CI_FLEET_CONTROLLER_IMAGE=$FAKE_CONTROLLER_IMAGE" "$rendered_env" || fail 'same-controller-tag build failure changed installed controller state'
+[[ $(readlink -f "$root/opt/ci-fleet/manager/current") == "$prior_manager" ]] || fail 'same-controller-tag build failure changed the installed manager'
 unset FAKE_COMPOSE_LOG
 python3 -c 'from pathlib import Path; import sys; path = Path(sys.argv[1]); path.write_text(path.read_text().replace(sys.argv[2], "CI_FLEET_RUNNER_IMAGE="))' "$FAKE_CONTROLLER_ENV_FILE" "CI_FLEET_RUNNER_IMAGE=$prior_runner_image"
 empty_live_image_output=$tmp/empty-live-image-build.out
@@ -700,6 +757,7 @@ grep -Fq 'DRAIN_OK managed_runners=0' "$live_drift_build_output" || fail 'live d
 grep -Fq 'ROLLBACK_RESTORED' "$live_drift_build_output" || fail 'live drift runner tag build failure did not restore the checkpoint'
 unset FAKE_COMPOSE_LOG
 [[ ${CI_FLEET_TEST_STOP_AFTER_LIVE_DRIFT_BUILD:-0} != 1 ]] || { printf 'LIVE_DRIFT_BUILD_REGRESSION_OK\n'; exit 0; }
+python3 -c 'from pathlib import Path; import sys; path = Path(sys.argv[1]); path.write_text(path.read_text().replace(sys.argv[2], sys.argv[3]))' "$rendered_env" "$FAKE_CONTROLLER_IMAGE" "$prior_controller_image"
 postbuild_capacity_output=$tmp/postbuild-capacity.out
 export FAKE_DISK_USED_PERCENT_FILE=$tmp/docker-disk-used-percent
 export FAKE_DISK_USED_PERCENT_AFTER_BUILD=80
@@ -713,6 +771,7 @@ if grep -Eq 'CHECKPOINT_CREATED|DRAIN_READY|ROLLBACK_' "$postbuild_capacity_outp
 [[ -f "$FAKE_DOCKER_STATE" ]] || fail 'post-build Docker capacity failure stopped the installed controller'
 grep -Fq 'CI_FLEET_MAX_RUNNERS=1' "$rendered_env" || fail 'post-build Docker capacity failure changed installed state'
 [[ ${CI_FLEET_TEST_STOP_AFTER_POSTBUILD_CAPACITY:-0} != 1 ]] || { printf 'POSTBUILD_CAPACITY_REGRESSION_OK\n'; exit 0; }
+python3 -c 'from pathlib import Path; import sys; path = Path(sys.argv[1]); path.write_text(path.read_text().replace(sys.argv[2], sys.argv[3]))' "$rendered_env" "$prior_controller_image" "$FAKE_CONTROLLER_IMAGE"
 for environment in "$rendered_env" "$FAKE_CONTROLLER_ENV_FILE"; do
   python3 -c 'from pathlib import Path; import sys; path = Path(sys.argv[1]); path.write_text(path.read_text().replace(sys.argv[2], sys.argv[3]))' "$environment" "$prior_runner_image" "$FAKE_RUNNER_IMAGE"
 done
@@ -724,15 +783,31 @@ fi
 unset FAKE_FAIL_CONFIG
 if grep -Eq 'CHECKPOINT_CREATED|DRAIN_READY|ROLLBACK_' "$config_failure_output"; then fail 'candidate Compose validation failure entered the transaction'; fi
 [[ -f "$FAKE_DOCKER_STATE" ]] || fail 'candidate Compose validation failure stopped the installed controller'
+invalid_image_output=$tmp/invalid-image-id.out
+export FAKE_COMPOSE_LOG=$tmp/invalid-image-id-compose.log
+: >"$FAKE_COMPOSE_LOG"
+prior_runner_image_id=$(<"$FAKE_RUNNER_IMAGE_ID_STATE")
+printf 'not-a-sha256-image-id\n' >"$FAKE_RUNNER_IMAGE_ID_STATE"
+if "$installer" --upgrade "${base_args[@]}" --ref "$ref_two" >"$invalid_image_output" 2>&1; then
+  fail 'invalid installed image ID unexpectedly reached the transaction'
+fi
+printf '%s\n' "$prior_runner_image_id" >"$FAKE_RUNNER_IMAGE_ID_STATE"
+grep -Fq 'installed runner image ID is invalid' "$invalid_image_output" || fail 'invalid installed image ID was not rejected'
+if grep -Eq 'CHECKPOINT_CREATED|DRAIN_READY|ROLLBACK_' "$invalid_image_output" || grep -Eq '^(stop|build)\|' "$FAKE_COMPOSE_LOG"; then fail 'invalid installed image ID caused a transaction side effect'; fi
+[[ -f "$FAKE_DOCKER_STATE" ]] || fail 'invalid installed image ID stopped the installed controller'
+grep -Fq 'CI_FLEET_MAX_RUNNERS=1' "$rendered_env" || fail 'invalid installed image ID changed installed state'
+[[ $(readlink -f "$root/opt/ci-fleet/manager/current") == "$prior_manager" ]] || fail 'invalid installed image ID changed the installed manager'
 restartable_tag_build_output=$tmp/restartable-tag-build.out
 export FAKE_COMPOSE_LOG=$tmp/restartable-tag-build-compose.log
 : >"$FAKE_COMPOSE_LOG"
 printf 'exited\n' >"$FAKE_CONTROLLER_STATUS_FILE"
-export FAKE_FAIL_BUILD=1
+prior_runner_image_id=$(<"$FAKE_RUNNER_IMAGE_ID_STATE")
+prior_controller_image_id=$(<"$FAKE_CONTROLLER_IMAGE_ID_STATE")
+export FAKE_PARTIAL_BUILD_FAIL=1
 if "$installer" --upgrade "${base_args[@]}" --ref "$ref_two" >"$restartable_tag_build_output" 2>&1; then
   fail 'restartable-tag candidate build failure unexpectedly succeeded'
 fi
-unset FAKE_FAIL_BUILD
+unset FAKE_PARTIAL_BUILD_FAIL
 stop_line=$(grep -n -m1 '^stop|' "$FAKE_COMPOSE_LOG" | cut -d: -f1 || true)
 build_line=$(grep -n -m1 '^build|' "$FAKE_COMPOSE_LOG" | cut -d: -f1 || true)
 [[ -n "$stop_line" && -n "$build_line" && "$stop_line" -lt "$build_line" ]] || fail 'restartable controller runner tag was built before stop'
@@ -743,8 +818,30 @@ grep -Fxq "CI_FLEET_RUNNER_IMAGE=$FAKE_RUNNER_IMAGE" "$rendered_env" || fail 're
 grep -Fxq "CI_FLEET_RUNNER_IMAGE=$FAKE_RUNNER_IMAGE" "$FAKE_CONTROLLER_ENV_FILE" || fail 'restartable controller runner tag build failure did not restore the installed controller environment'
 grep -Fq 'CI_FLEET_MAX_RUNNERS=1' "$rendered_env" || fail 'restartable controller runner tag build failure changed installed state'
 [[ $(readlink -f "$root/opt/ci-fleet/manager/current") == "$prior_manager" ]] || fail 'restartable controller runner tag build failure changed the installed manager'
-unset FAKE_COMPOSE_LOG
+[[ $(<"$FAKE_RUNNER_IMAGE_ID_STATE") == "$prior_runner_image_id" ]] || fail 'rollback reported restored but left the runner tag on the partial build image'
+[[ $(<"$FAKE_CONTROLLER_IMAGE_ID_STATE") == "$prior_controller_image_id" ]] || fail 'rollback did not restore the prior controller tag image ID'
+checkpoint_path=$(awk '$1 == "CHECKPOINT_CREATED" {sub(/^path=/, "", $2); value=$2} END {print value}' "$restartable_tag_build_output")
+image_ids_file=$checkpoint_path/image-ids.env
+[[ -d "$checkpoint_path" && $(stat -c %a "$checkpoint_path") == 700 ]] || fail 'image rollback checkpoint is not a mode-0700 directory'
+[[ -f "$image_ids_file" && $(stat -c %a "$image_ids_file") == 600 && $(wc -l <"$image_ids_file") == 2 ]] || fail 'checkpoint image map is not exactly one mode-0600 two-ID file'
+grep -Fxq "CI_FLEET_RUNNER_IMAGE_ID=$prior_runner_image_id" "$image_ids_file" || fail 'checkpoint omitted the prior runner image ID'
+grep -Fxq "CI_FLEET_CONTROLLER_IMAGE_ID=$prior_controller_image_id" "$image_ids_file" || fail 'checkpoint omitted the prior controller image ID'
+runner_tag_line=$(grep -n -m1 "^image-tag|$prior_runner_image_id|$FAKE_RUNNER_IMAGE$" "$FAKE_COMPOSE_LOG" | cut -d: -f1 || true)
+controller_tag_line=$(grep -n -m1 "^image-tag|$prior_controller_image_id|$FAKE_CONTROLLER_IMAGE$" "$FAKE_COMPOSE_LOG" | cut -d: -f1 || true)
+rollback_up_line=$(grep -n '^up|' "$FAKE_COMPOSE_LOG" | tail -n1 | cut -d: -f1 || true)
+[[ -n "$runner_tag_line" && -n "$controller_tag_line" && -n "$rollback_up_line" && "$runner_tag_line" -lt "$rollback_up_line" && "$controller_tag_line" -lt "$rollback_up_line" ]] || fail 'rollback started the prior controller before restoring both image tags'
 [[ ${CI_FLEET_TEST_STOP_AFTER_RESTARTABLE_TAG_BUILD:-0} != 1 ]] || { printf 'RESTARTABLE_TAG_BUILD_REGRESSION_OK\n'; exit 0; }
+printf 'CI_FLEET_RUNNER_IMAGE_ID=invalid\nCI_FLEET_CONTROLLER_IMAGE_ID=%s\n' "$prior_controller_image_id" >"$image_ids_file"
+: >"$FAKE_COMPOSE_LOG"
+rm -f "$FAKE_DOCKER_STATE" "$FAKE_CONTROLLER_STATUS_FILE" "$FAKE_CONTROLLER_PROVENANCE_FILE" "$FAKE_CONTROLLER_IMAGE_ID_FILE" "$FAKE_CONTROLLER_ENV_FILE"
+expect_failure 'checkpoint image mappings are invalid' "$installer" --rollback
+if grep -q '^up|' "$FAKE_COMPOSE_LOG"; then fail 'malformed checkpoint image ID restarted the prior controller'; fi
+[[ ! -f "$FAKE_DOCKER_STATE" ]] || fail 'malformed checkpoint image ID restored the prior controller'
+printf 'CI_FLEET_RUNNER_IMAGE_ID=%s\nCI_FLEET_CONTROLLER_IMAGE_ID=%s\n' "$prior_runner_image_id" "$prior_controller_image_id" >"$image_ids_file"
+expect_success "$installer" --rollback >/dev/null
+[[ -f "$FAKE_DOCKER_STATE" ]] || fail 'repaired checkpoint did not restore the prior controller'
+unset FAKE_COMPOSE_LOG
+[[ ${CI_FLEET_TEST_STOP_AFTER_IMAGE_ROLLBACK:-0} != 1 ]] || { printf 'IMAGE_ROLLBACK_REGRESSION_OK\n'; exit 0; }
 terminate_upgrade() {
   local output=$1 marker=$2 second_term_marker=${3:-} pid status=0 attempt
   export CI_FLEET_TEST_PAUSE_AFTER_DRAIN_FILE=$marker
@@ -936,6 +1033,8 @@ printf '%s\n' \
   "CI_FLEET_GITHUB_APP_PRIVATE_KEY_FILE=$adopt_pem" \
   'CI_FLEET_RUNNER_TTL=6h' \
   'CI_FLEET_CONTROLLER_STATE=active' \
+  "CI_FLEET_RUNNER_IMAGE=$FAKE_RUNNER_IMAGE" \
+  "CI_FLEET_CONTROLLER_IMAGE=$FAKE_CONTROLLER_IMAGE" \
   'CI_FLEET_INSTANCE=legacy-ci-01' >"$adopt_root/etc/ci-fleet/ci-fleet.env"
 chmod 600 "$adopt_root/etc/ci-fleet/ci-fleet.env"
 printf 'CI_FLEET_HEALTH_DISK_WARN_PERCENT=75\n' >"$adopt_root/etc/ci-fleet/monitoring.env"
@@ -971,6 +1070,8 @@ expect_failure 'selected engine does not support status reporting configuration'
 legacy_required_ref=$(write_config active 1 1 "$legacy_engine_ref" true omit)
 expect_failure 'selected engine does not advertise required status reporting' "$installer" --upgrade "${base_args[@]}" --ref "$legacy_required_ref"
 legacy_ref=$(write_config active 1 1 "$legacy_engine_ref" omit omit)
+export FAKE_PREVIOUS_RUNNER_IMAGE=$FAKE_RUNNER_IMAGE
+export FAKE_PREVIOUS_CONTROLLER_IMAGE=$FAKE_CONTROLLER_IMAGE
 export FAKE_ENGINE_REF=$legacy_engine_ref
 export FAKE_RUNNER_IMAGE=ci-fleet-runner:${legacy_engine_ref:0:12}
 export FAKE_CONTROLLER_IMAGE=ci-fleet-controller:${legacy_engine_ref:0:12}

From 524b5f44a40112a19bcbc30194f65f4b639b605c Mon Sep 17 00:00:00 2001
From: Nickfost <1572453+Nickfost@users.noreply.github.com>
Date: Fri, 4 Sep 2026 01:59:45 -0500
Subject: [PATCH 10/14] fix: preserve absent image tags and legacy checkpoints

---
 scripts/install-worker-controller.sh      | 67 ++++++++++++----
 scripts/test-install-worker-controller.sh | 94 ++++++++++++++++++++++-
 2 files changed, 144 insertions(+), 17 deletions(-)

diff --git a/scripts/install-worker-controller.sh b/scripts/install-worker-controller.sh
index c05ac2cb..ecd5e578 100755
--- a/scripts/install-worker-controller.sh
+++ b/scripts/install-worker-controller.sh
@@ -719,7 +719,7 @@ for key in image_keys:
 if sys.argv[2]:
     id_keys = tuple(f"{key}_ID" for key in image_keys)
     values = parse_env(Path(sys.argv[2]), allow_unknown=True)
-    if set(values) != set(id_keys) or any(not re.fullmatch(r"sha256:[0-9a-f]{64}", values[key]) for key in id_keys):
+    if set(values) != set(id_keys) or any(values[key] != "absent" and not re.fullmatch(r"sha256:[0-9a-f]{64}", values[key]) for key in id_keys):
         raise SystemExit(1)
     for key in id_keys:
         print(values[key])
@@ -739,15 +739,23 @@ make_checkpoint() {
   staging_paths+=("$staged_checkpoint")
   checkpoint_dir=$staged_checkpoint
   install -d -m 0700 "$checkpoint_dir/systemd"
+  printf '2\n' >"$checkpoint_dir/format-version"
+  chmod 0600 "$checkpoint_dir/format-version"
   if [[ -f "$rendered_env" ]]; then
     [[ "$testing" != 1 ]] || expected_owner=$(id -u)
     [[ $(stat -c %u "$rendered_env") == "$expected_owner" && $(stat -c %a "$rendered_env") == 600 ]] || die "rendered environment must be owned by root with mode 0600: $rendered_env"
     install -m 0600 "$rendered_env" "$checkpoint_dir/ci-fleet.env"
     load_checkpoint_images "$checkpoint_dir/ci-fleet.env" || die 'installed image tags are invalid'
-    runner_id=$(docker image inspect --format '{{.Id}}' "${checkpoint_images[0]}" 2>/dev/null) || die 'installed runner image mapping is unavailable'
-    controller_id=$(docker image inspect --format '{{.Id}}' "${checkpoint_images[1]}" 2>/dev/null) || die 'installed controller image mapping is unavailable'
-    [[ "$runner_id" =~ ^sha256:[0-9a-f]{64}$ ]] || die 'installed runner image ID is invalid'
-    [[ "$controller_id" =~ ^sha256:[0-9a-f]{64}$ ]] || die 'installed controller image ID is invalid'
+    if ! runner_id=$(docker image inspect --format '{{.Id}}' "${checkpoint_images[0]}" 2>/dev/null); then
+      docker info >/dev/null 2>&1 || die 'Docker daemon is unavailable'
+      runner_id=absent
+    fi
+    if ! controller_id=$(docker image inspect --format '{{.Id}}' "${checkpoint_images[1]}" 2>/dev/null); then
+      docker info >/dev/null 2>&1 || die 'Docker daemon is unavailable'
+      controller_id=absent
+    fi
+    [[ "$runner_id" == absent || "$runner_id" =~ ^sha256:[0-9a-f]{64}$ ]] || die 'installed runner image ID is invalid'
+    [[ "$controller_id" == absent || "$controller_id" =~ ^sha256:[0-9a-f]{64}$ ]] || die 'installed controller image ID is invalid'
     printf 'CI_FLEET_RUNNER_IMAGE_ID=%s\nCI_FLEET_CONTROLLER_IMAGE_ID=%s\n' "$runner_id" "$controller_id" >"$checkpoint_dir/image-ids.env"
     chmod 0600 "$checkpoint_dir/image-ids.env"
   fi
@@ -1011,17 +1019,32 @@ restore_systemd_snapshot() {
 }
 
 restore_checkpoint() {
-  local target restored_state actual failed=0 checkpoint_release='' drain_env=$rendered_env drain_release='' restore_images=false
+  local target restored_state actual index expected_owner=0 failed=0 checkpoint_release='' drain_env=$rendered_env drain_release='' restore_images=false new_format=false
+  local format_marker=$checkpoint_dir/format-version image_ids=$checkpoint_dir/image-ids.env
   [[ -n "$checkpoint_dir" && -d "$checkpoint_dir" ]] || return 1
+  [[ "$testing" != 1 ]] || expected_owner=$(id -u)
+  if [[ -e "$format_marker" || -L "$format_marker" ]]; then
+    if [[ -L "$format_marker" || ! -f "$format_marker" || $(stat -c %u "$format_marker") != "$expected_owner" \
+      || $(stat -c %a "$format_marker") != 600 || $(stat -c %s "$format_marker") != 2 || $(<"$format_marker") != 2 ]]; then
+      note 'ROLLBACK_FAILED reason=checkpoint format marker is invalid'
+      return 1
+    fi
+    new_format=true
+  elif [[ -e "$image_ids" || -L "$image_ids" ]]; then
+    note 'ROLLBACK_FAILED reason=checkpoint image mappings are invalid'
+    return 1
+  elif [[ -f "$checkpoint_dir/ci-fleet.env" ]]; then
+    note 'ROLLBACK_LEGACY_IMAGE_STATE_UNVERIFIED'
+  fi
   [[ ! -f "$checkpoint_dir/release-target" ]] || checkpoint_release=$(<"$checkpoint_dir/release-target")
-  if [[ -f "$checkpoint_dir/ci-fleet.env" ]]; then
-    if [[ ! -f "$checkpoint_dir/image-ids.env" || -L "$checkpoint_dir/image-ids.env" || $(stat -c %a "$checkpoint_dir/image-ids.env") != 600 ]] \
-      || ! load_checkpoint_images "$checkpoint_dir/ci-fleet.env" "$checkpoint_dir/image-ids.env"; then
+  if $new_format && [[ -f "$checkpoint_dir/ci-fleet.env" ]]; then
+    if [[ ! -f "$image_ids" || -L "$image_ids" || $(stat -c %u "$image_ids") != "$expected_owner" || $(stat -c %a "$image_ids") != 600 ]] \
+      || ! load_checkpoint_images "$checkpoint_dir/ci-fleet.env" "$image_ids"; then
       note 'ROLLBACK_FAILED reason=checkpoint image mappings are invalid'
       return 1
     fi
     restore_images=true
-  elif [[ -e "$checkpoint_dir/image-ids.env" || -L "$checkpoint_dir/image-ids.env" ]]; then
+  elif $new_format && [[ -e "$image_ids" || -L "$image_ids" ]]; then
     note 'ROLLBACK_FAILED reason=checkpoint image mappings are invalid'
     return 1
   fi
@@ -1075,12 +1098,24 @@ restore_checkpoint() {
     rm -f "$manager_current" || failed=1
   fi
   if $restore_images; then
-    docker image tag "${checkpoint_images[2]}" "${checkpoint_images[0]}" || failed=1
-    docker image tag "${checkpoint_images[3]}" "${checkpoint_images[1]}" || failed=1
-    actual=$(docker image inspect --format '{{.Id}}' "${checkpoint_images[0]}" 2>/dev/null) || failed=1
-    [[ "$actual" == "${checkpoint_images[2]}" ]] || failed=1
-    actual=$(docker image inspect --format '{{.Id}}' "${checkpoint_images[1]}" 2>/dev/null) || failed=1
-    [[ "$actual" == "${checkpoint_images[3]}" ]] || failed=1
+    for index in 0 1; do
+      if [[ ${checkpoint_images[index + 2]} == absent ]]; then
+        if docker image inspect --format '{{.Id}}' "${checkpoint_images[index]}" >/dev/null 2>&1; then
+          docker image rm "${checkpoint_images[index]}" >/dev/null || failed=1
+        else
+          docker info >/dev/null 2>&1 || failed=1
+        fi
+        if docker image inspect --format '{{.Id}}' "${checkpoint_images[index]}" >/dev/null 2>&1; then
+          failed=1
+        else
+          docker info >/dev/null 2>&1 || failed=1
+        fi
+      else
+        docker image tag "${checkpoint_images[index + 2]}" "${checkpoint_images[index]}" || failed=1
+        actual=$(docker image inspect --format '{{.Id}}' "${checkpoint_images[index]}" 2>/dev/null) || failed=1
+        [[ "$actual" == "${checkpoint_images[index + 2]}" ]] || failed=1
+      fi
+    done
   fi
   restore_systemd_snapshot || failed=1
   if [[ -n "$release_dir" && -f "$rendered_env" ]]; then
diff --git a/scripts/test-install-worker-controller.sh b/scripts/test-install-worker-controller.sh
index ca2f2900..d2e7c946 100755
--- a/scripts/test-install-worker-controller.sh
+++ b/scripts/test-install-worker-controller.sh
@@ -34,6 +34,7 @@ if [[ -n ${FAKE_REQUIRE_LOCAL_DOCKER_ENDPOINT:-} ]]; then
 fi
 case "${1:-}" in
   info)
+    if [[ -n ${FAKE_DOCKER_INFO_FAIL_AFTER_IMAGE_INSPECT:-} && -f $FAKE_DOCKER_INFO_FAIL_AFTER_IMAGE_INSPECT ]]; then exit 1; fi
     [[ "$*" != *DockerRootDir* ]] || printf '%s\n' "${CI_FLEET_DOCKER_ROOT:?}"
     exit 0
     ;;
@@ -83,7 +84,15 @@ case "${1:-}" in
         else
           exit 1
         fi
-        if [[ "$*" == *'{{.Id}}'* ]]; then [[ -f "$image_id_state" ]] && cat "$image_id_state"; else [[ -f "$image_state" ]] && cat "$image_state"; fi
+        if [[ "$*" == *'{{.Id}}'* ]]; then
+          if [[ ! -f "$image_id_state" ]]; then
+            [[ -z ${FAKE_DOCKER_INFO_FAIL_AFTER_IMAGE_INSPECT:-} ]] || : >"$FAKE_DOCKER_INFO_FAIL_AFTER_IMAGE_INSPECT"
+            exit 1
+          fi
+          cat "$image_id_state"
+        else
+          [[ -f "$image_state" ]] && cat "$image_state"
+        fi
         ;;
       tag)
         image_id=${3:-}
@@ -100,6 +109,20 @@ case "${1:-}" in
         printf '%s\n' "$image_id" >"$image_id_state"
         [[ -z "${FAKE_COMPOSE_LOG:-}" ]] || printf 'image-tag|%s|%s\n' "$image_id" "$image" >>"$FAKE_COMPOSE_LOG"
         ;;
+      rm)
+        image=${3:-}
+        if [[ "$image" == "${FAKE_RUNNER_IMAGE:-}" || "$image" == "${FAKE_PRIOR_RUNNER_IMAGE:-}" || "$image" == "${FAKE_PREVIOUS_RUNNER_IMAGE:-}" ]]; then
+          image_state=${FAKE_RUNNER_IMAGE_STATE:-}
+          image_id_state=${FAKE_RUNNER_IMAGE_ID_STATE:-}
+        elif [[ "$image" == "${FAKE_CONTROLLER_IMAGE:-}" || "$image" == "${FAKE_PRIOR_CONTROLLER_IMAGE:-}" || "$image" == "${FAKE_PREVIOUS_CONTROLLER_IMAGE:-}" ]]; then
+          image_state=${FAKE_CONTROLLER_IMAGE_STATE:-}
+          image_id_state=${FAKE_CONTROLLER_IMAGE_ID_STATE:-}
+        else
+          exit 1
+        fi
+        rm -f "$image_state" "$image_id_state"
+        [[ -z "${FAKE_COMPOSE_LOG:-}" ]] || printf 'image-rm|%s\n' "$image" >>"$FAKE_COMPOSE_LOG"
+        ;;
       *) exit 1 ;;
     esac
     ;;
@@ -455,6 +478,10 @@ grep -Fq 'CONVERGED mode=install' <<<"$first" || fail 'fresh install did not con
 [[ -L "$root/opt/ci-fleet/current" && -f "$root/var/lib/ci-fleet/install-state.json" ]] || fail 'fresh install state is incomplete'
 [[ $(readlink -f "$root/opt/ci-fleet/manager/current") == "$root/opt/ci-fleet/manager/releases/$engine_ref" ]] || fail 'installer manager did not activate the desired engine release'
 [[ -f "$FAKE_DOCKER_STATE" ]] || fail 'active controller was not started'
+fresh_checkpoint=$(find "$root/var/lib/ci-fleet/checkpoints" -mindepth 1 -maxdepth 1 -type d ! -name '.checkpoint.staging.*' -print -quit)
+fresh_format_marker=$fresh_checkpoint/format-version
+[[ -f "$fresh_format_marker" && ! -L "$fresh_format_marker" && $(stat -c '%u:%a:%s' "$fresh_format_marker") == "$(id -u):600:2" && $(<"$fresh_format_marker") == 2 ]] || fail 'fresh checkpoint lacks the exact root-owned mode-0600 format marker'
+[[ ! -e "$fresh_checkpoint/ci-fleet.env" && ! -e "$fresh_checkpoint/image-ids.env" ]] || fail 'fresh checkpoint stored prior managed image state'
 if find "$root/var/lib/ci-fleet/checkpoints" -name image-ids.env -print -quit | grep -q .; then fail 'fresh install checkpoint stored a prior image map'; fi
 install_state=$root/var/lib/ci-fleet/install-state.json
 chmod 644 "$install_state"
@@ -797,6 +824,47 @@ if grep -Eq 'CHECKPOINT_CREATED|DRAIN_READY|ROLLBACK_' "$invalid_image_output" |
 [[ -f "$FAKE_DOCKER_STATE" ]] || fail 'invalid installed image ID stopped the installed controller'
 grep -Fq 'CI_FLEET_MAX_RUNNERS=1' "$rendered_env" || fail 'invalid installed image ID changed installed state'
 [[ $(readlink -f "$root/opt/ci-fleet/manager/current") == "$prior_manager" ]] || fail 'invalid installed image ID changed the installed manager'
+missing_tag_output=$tmp/missing-tag-build.out
+export FAKE_COMPOSE_LOG=$tmp/missing-tag-build-compose.log
+: >"$FAKE_COMPOSE_LOG"
+prior_runner_image_id=$(<"$FAKE_RUNNER_IMAGE_ID_STATE")
+prior_controller_image_id=$(<"$FAKE_CONTROLLER_IMAGE_ID_STATE")
+rm -f "$FAKE_RUNNER_IMAGE_STATE" "$FAKE_RUNNER_IMAGE_ID_STATE"
+export FAKE_PARTIAL_BUILD_FAIL=1
+if "$installer" --upgrade "${base_args[@]}" --ref "$ref_two" >"$missing_tag_output" 2>&1; then
+  fail 'missing-tag candidate build failure unexpectedly succeeded'
+fi
+unset FAKE_PARTIAL_BUILD_FAIL
+grep -Fq 'DRAIN_OK managed_runners=0' "$missing_tag_output" || fail "missing managed tag did not reach the deferred build failure: $(<"$missing_tag_output")"
+grep -Fq 'ROLLBACK_RESTORED' "$missing_tag_output" || fail 'missing-tag build failure did not restore the checkpoint'
+checkpoint_path=$(awk '$1 == "CHECKPOINT_CREATED" {sub(/^path=/, "", $2); value=$2} END {print value}' "$missing_tag_output")
+grep -Fxq 'CI_FLEET_RUNNER_IMAGE_ID=absent' "$checkpoint_path/image-ids.env" || fail 'checkpoint did not record the missing runner tag'
+[[ ! -e "$FAKE_RUNNER_IMAGE_STATE" && ! -e "$FAKE_RUNNER_IMAGE_ID_STATE" ]] || fail 'rollback retained a runner tag that was absent before repair'
+[[ $(<"$FAKE_CONTROLLER_IMAGE_ID_STATE") == "$prior_controller_image_id" ]] || fail 'missing-tag rollback did not restore the prior controller image ID'
+grep -Fxq "CI_FLEET_RUNNER_IMAGE=$FAKE_RUNNER_IMAGE" "$rendered_env" || fail 'missing-tag rollback changed the installed environment'
+grep -Fq '"configured_max_runners": 1' "$install_state" || fail 'missing-tag rollback changed installed state'
+[[ $(readlink -f "$root/opt/ci-fleet/manager/current") == "$prior_manager" ]] || fail 'missing-tag rollback changed the installed manager'
+runner_rm_line=$(grep -n -m1 "^image-rm|$FAKE_RUNNER_IMAGE$" "$FAKE_COMPOSE_LOG" | cut -d: -f1 || true)
+controller_tag_line=$(grep -n -m1 "^image-tag|$prior_controller_image_id|$FAKE_CONTROLLER_IMAGE$" "$FAKE_COMPOSE_LOG" | cut -d: -f1 || true)
+rollback_up_line=$(grep -n '^up|' "$FAKE_COMPOSE_LOG" | tail -n1 | cut -d: -f1 || true)
+[[ -n "$runner_rm_line" && -n "$controller_tag_line" && -n "$rollback_up_line" && "$runner_rm_line" -lt "$rollback_up_line" && "$controller_tag_line" -lt "$rollback_up_line" ]] || fail 'missing-tag rollback restarted the controller before restoring both image states'
+daemon_failure_output=$tmp/missing-tag-daemon-failure.out
+export FAKE_DOCKER_INFO_FAIL_AFTER_IMAGE_INSPECT=$tmp/docker-info-fail-after-image-inspect
+: >"$FAKE_COMPOSE_LOG"
+if "$installer" --upgrade "${base_args[@]}" --ref "$ref_two" >"$daemon_failure_output" 2>&1; then
+  fail 'missing-tag daemon failure unexpectedly succeeded'
+fi
+rm -f "$FAKE_DOCKER_INFO_FAIL_AFTER_IMAGE_INSPECT"
+unset FAKE_DOCKER_INFO_FAIL_AFTER_IMAGE_INSPECT
+grep -Fq 'Docker daemon is unavailable' "$daemon_failure_output" || fail 'image absence was accepted without confirming Docker availability'
+if grep -Eq 'CHECKPOINT_CREATED|DRAIN_READY|ROLLBACK_' "$daemon_failure_output" || grep -Eq '^(stop|build)\|' "$FAKE_COMPOSE_LOG"; then fail 'Docker daemon failure entered the transaction'; fi
+uninstall_output=$(expect_success "$installer" --uninstall)
+grep -Fq 'UNINSTALL_OK' <<<"$uninstall_output" || fail 'uninstall rejected a missing managed image tag'
+expect_success "$installer" --rollback >/dev/null
+[[ ! -e "$FAKE_RUNNER_IMAGE_STATE" && ! -e "$FAKE_RUNNER_IMAGE_ID_STATE" && -f "$FAKE_DOCKER_STATE" ]] || fail 'uninstall rollback did not restore the missing-tag installation'
+printf '%s\n' "$engine_ref" >"$FAKE_RUNNER_IMAGE_STATE"
+printf '%s\n' "$prior_runner_image_id" >"$FAKE_RUNNER_IMAGE_ID_STATE"
+unset FAKE_COMPOSE_LOG
 restartable_tag_build_output=$tmp/restartable-tag-build.out
 export FAKE_COMPOSE_LOG=$tmp/restartable-tag-build-compose.log
 : >"$FAKE_COMPOSE_LOG"
@@ -822,7 +890,9 @@ grep -Fq 'CI_FLEET_MAX_RUNNERS=1' "$rendered_env" || fail 'restartable controlle
 [[ $(<"$FAKE_CONTROLLER_IMAGE_ID_STATE") == "$prior_controller_image_id" ]] || fail 'rollback did not restore the prior controller tag image ID'
 checkpoint_path=$(awk '$1 == "CHECKPOINT_CREATED" {sub(/^path=/, "", $2); value=$2} END {print value}' "$restartable_tag_build_output")
 image_ids_file=$checkpoint_path/image-ids.env
+format_marker=$checkpoint_path/format-version
 [[ -d "$checkpoint_path" && $(stat -c %a "$checkpoint_path") == 700 ]] || fail 'image rollback checkpoint is not a mode-0700 directory'
+[[ -f "$format_marker" && ! -L "$format_marker" && $(stat -c '%u:%a:%s' "$format_marker") == "$(id -u):600:2" && $(<"$format_marker") == 2 ]] || fail 'managed checkpoint lacks the exact root-owned mode-0600 format marker'
 [[ -f "$image_ids_file" && $(stat -c %a "$image_ids_file") == 600 && $(wc -l <"$image_ids_file") == 2 ]] || fail 'checkpoint image map is not exactly one mode-0600 two-ID file'
 grep -Fxq "CI_FLEET_RUNNER_IMAGE_ID=$prior_runner_image_id" "$image_ids_file" || fail 'checkpoint omitted the prior runner image ID'
 grep -Fxq "CI_FLEET_CONTROLLER_IMAGE_ID=$prior_controller_image_id" "$image_ids_file" || fail 'checkpoint omitted the prior controller image ID'
@@ -840,7 +910,29 @@ if grep -q '^up|' "$FAKE_COMPOSE_LOG"; then fail 'malformed checkpoint image ID
 printf 'CI_FLEET_RUNNER_IMAGE_ID=%s\nCI_FLEET_CONTROLLER_IMAGE_ID=%s\n' "$prior_runner_image_id" "$prior_controller_image_id" >"$image_ids_file"
 expect_success "$installer" --rollback >/dev/null
 [[ -f "$FAKE_DOCKER_STATE" ]] || fail 'repaired checkpoint did not restore the prior controller'
+legacy_checkpoint=$root/var/lib/ci-fleet/checkpoints/legacy-checkpoint
+cp -a "$checkpoint_path" "$legacy_checkpoint"
+rm -f "$legacy_checkpoint/format-version" "$legacy_checkpoint/image-ids.env"
+touch "$legacy_checkpoint/.complete"
+: >"$FAKE_COMPOSE_LOG"
+legacy_output=$tmp/legacy-checkpoint.out
+if ! "$installer" --rollback >"$legacy_output" 2>&1; then
+  fail "legacy checkpoint was rejected: $(<"$legacy_output")"
+fi
+grep -Fq 'ROLLBACK_LEGACY_IMAGE_STATE_UNVERIFIED' "$legacy_output" || fail 'legacy rollback omitted its image-state warning'
+if grep -Eq '^image-(tag|rm)\|' "$FAKE_COMPOSE_LOG"; then fail 'legacy rollback claimed or changed image identity'; fi
+printf '3\n' >"$legacy_checkpoint/format-version"
+chmod 0600 "$legacy_checkpoint/format-version"
+: >"$FAKE_COMPOSE_LOG"
+expect_failure 'checkpoint format marker is invalid' "$installer" --rollback
+if [[ ! -f "$FAKE_DOCKER_STATE" ]] || grep -Eq '^(stop|up|image-(tag|rm))\|' "$FAKE_COMPOSE_LOG"; then fail 'unknown checkpoint version caused rollback side effects'; fi
+printf '2\n' >"$legacy_checkpoint/format-version"
+: >"$FAKE_COMPOSE_LOG"
+expect_failure 'checkpoint image mappings are invalid' "$installer" --rollback
+if [[ ! -f "$FAKE_DOCKER_STATE" ]] || grep -Eq '^(stop|up|image-(tag|rm))\|' "$FAKE_COMPOSE_LOG"; then fail 'version-2 checkpoint without image state caused rollback side effects'; fi
+rm -rf "$legacy_checkpoint"
 unset FAKE_COMPOSE_LOG
+[[ ${CI_FLEET_TEST_STOP_AFTER_CHECKPOINT_COMPAT:-0} != 1 ]] || { printf 'CHECKPOINT_COMPAT_REGRESSIONS_OK\n'; exit 0; }
 [[ ${CI_FLEET_TEST_STOP_AFTER_IMAGE_ROLLBACK:-0} != 1 ]] || { printf 'IMAGE_ROLLBACK_REGRESSION_OK\n'; exit 0; }
 terminate_upgrade() {
   local output=$1 marker=$2 second_term_marker=${3:-} pid status=0 attempt

From c4e8e05f3021bd3b015f017360ae2351cda0507c Mon Sep 17 00:00:00 2001
From: Nickfost <1572453+Nickfost@users.noreply.github.com>
Date: Fri, 4 Sep 2026 03:31:40 -0500
Subject: [PATCH 11/14] fix: remove candidate container before image rollback

---
 scripts/install-worker-controller.sh      | 36 +++++++++--------
 scripts/test-install-worker-controller.sh | 48 +++++++++++++++++++++--
 2 files changed, 64 insertions(+), 20 deletions(-)

diff --git a/scripts/install-worker-controller.sh b/scripts/install-worker-controller.sh
index ecd5e578..61d92d0a 100755
--- a/scripts/install-worker-controller.sh
+++ b/scripts/install-worker-controller.sh
@@ -795,7 +795,7 @@ make_checkpoint() {
 }
 
 try_drain_current() {
-  local deadline count old_release status paused=false force_nonterminal=${1:-false}
+  local deadline count old_release='' status paused=false force_nonterminal=${1:-false}
   local drain_env=${2:-$rendered_env} fallback_release=${3:-} shutdown_timeout=${CI_FLEET_DRAIN_TIMEOUT_SECONDS:-300}
   drain_error=
   status=$(controller_status)
@@ -851,25 +851,29 @@ try_drain_current() {
     sleep 2
   done
   note 'DRAIN_READY managed_runners=0'
-  if [[ "$status" != running ]]; then
-    note 'DRAIN_OK managed_runners=0'
-    return 0
+  if [[ "$status" == running ]]; then
+    compose "$old_release" "$drain_env" kill --signal SIGTERM controller >/dev/null || {
+      compose "$old_release" "$drain_env" unpause controller >/dev/null 2>&1 || true
+      drain_error='failed to signal the paused controller for graceful scale-set cleanup'
+      return 1
+    }
+    if [[ $(docker inspect --format '{{.State.Paused}}' "$controller_container" 2>/dev/null || true) == true ]]; then
+      compose "$old_release" "$drain_env" unpause controller >/dev/null || {
+        drain_error='failed to unpause the signaled controller for graceful shutdown'
+        return 1
+      }
+    fi
+    compose "$old_release" "$drain_env" stop --timeout "$shutdown_timeout" controller >/dev/null || {
+      drain_error='could not stop the drained controller'
+      return 1
+    }
   fi
-  compose "$old_release" "$drain_env" kill --signal SIGTERM controller >/dev/null || {
-    compose "$old_release" "$drain_env" unpause controller >/dev/null 2>&1 || true
-    drain_error='failed to signal the paused controller for graceful scale-set cleanup'
-    return 1
-  }
-  if [[ $(docker inspect --format '{{.State.Paused}}' "$controller_container" 2>/dev/null || true) == true ]]; then
-    compose "$old_release" "$drain_env" unpause controller >/dev/null || {
-      drain_error='failed to unpause the signaled controller for graceful shutdown'
+  if [[ "$force_nonterminal" == true && -n "$old_release" ]]; then
+    compose "$old_release" "$drain_env" rm -f controller >/dev/null || {
+      drain_error='could not remove the stopped candidate controller'
       return 1
     }
   fi
-  compose "$old_release" "$drain_env" stop --timeout "$shutdown_timeout" controller >/dev/null || {
-    drain_error='could not stop the drained controller'
-    return 1
-  }
   note 'DRAIN_OK managed_runners=0'
 }
 
diff --git a/scripts/test-install-worker-controller.sh b/scripts/test-install-worker-controller.sh
index d2e7c946..da9a0a53 100755
--- a/scripts/test-install-worker-controller.sh
+++ b/scripts/test-install-worker-controller.sh
@@ -25,6 +25,7 @@ set -u
 state=${FAKE_DOCKER_STATE:?}
 status_file=${FAKE_CONTROLLER_STATUS_FILE:-}
 paused_state=${FAKE_PAUSED_STATE:-}
+stopped_state=${FAKE_STOPPED_CONTROLLER_STATE:-}
 if [[ -n ${FAKE_REQUIRE_LOCAL_DOCKER_ENDPOINT:-} ]]; then
   expected_socket=${CI_FLEET_ROOT_PREFIX:-}/var/run/docker.sock
   [[ ${DOCKER_HOST:-} == "unix://$expected_socket" ]] || {
@@ -39,7 +40,7 @@ case "${1:-}" in
     exit 0
     ;;
   inspect)
-    [[ -f "$state" ]] || exit 1
+    [[ -f "$state" || -n "$stopped_state" && -f "$stopped_state" ]] || exit 1
     if [[ "$*" == *'.Config.Env'* ]]; then
       [[ -n "${FAKE_CONTROLLER_ENV_FILE:-}" && -f "$FAKE_CONTROLLER_ENV_FILE" ]] || exit 1
       cat "$FAKE_CONTROLLER_ENV_FILE"
@@ -49,7 +50,7 @@ case "${1:-}" in
       [[ -n "${FAKE_CONTROLLER_IMAGE_ID_FILE:-}" && -f "$FAKE_CONTROLLER_IMAGE_ID_FILE" ]] || exit 1
       cat "$FAKE_CONTROLLER_IMAGE_ID_FILE"
     elif [[ "$*" == *'.State.Status'* ]]; then
-      if [[ -n "$paused_state" && -f "$paused_state" ]]; then printf 'paused\n'; elif [[ -n "$status_file" && -f "$status_file" ]]; then cat "$status_file"; else printf '%s\n' "${FAKE_CONTROLLER_STATUS:-running}"; fi
+      if [[ -n "$paused_state" && -f "$paused_state" ]]; then printf 'paused\n'; elif [[ -n "$status_file" && -f "$status_file" ]]; then cat "$status_file"; elif [[ -n "$stopped_state" && -f "$stopped_state" ]]; then printf 'exited\n'; else printf '%s\n' "${FAKE_CONTROLLER_STATUS:-running}"; fi
     elif [[ "$*" == *'.State.Paused'* ]]; then
       if [[ -n "$paused_state" && -f "$paused_state" ]]; then printf 'true\n'; else printf 'false\n'; fi
     else
@@ -120,6 +121,11 @@ case "${1:-}" in
         else
           exit 1
         fi
+        if [[ -n "$stopped_state" && -f "$stopped_state" && -n "${FAKE_CONTROLLER_IMAGE_ID_FILE:-}" && -f "$FAKE_CONTROLLER_IMAGE_ID_FILE" && -f "$image_id_state" ]] \
+          && cmp -s "$FAKE_CONTROLLER_IMAGE_ID_FILE" "$image_id_state"; then
+          [[ -z "${FAKE_COMPOSE_LOG:-}" ]] || printf 'image-rm-blocked|%s\n' "$image" >>"$FAKE_COMPOSE_LOG"
+          exit 48
+        fi
         rm -f "$image_state" "$image_id_state"
         [[ -z "${FAKE_COMPOSE_LOG:-}" ]] || printf 'image-rm|%s\n' "$image" >>"$FAKE_COMPOSE_LOG"
         ;;
@@ -162,6 +168,7 @@ case "${1:-}" in
           exit 42
         fi
         : >"$state"
+        [[ -z "$stopped_state" ]] || rm -f "$stopped_state"
         [[ -z "${FAKE_CONTROLLER_PROVENANCE_FILE:-}" ]] || printf '%s\n' "${FAKE_ENGINE_REF:?}" >"$FAKE_CONTROLLER_PROVENANCE_FILE"
         [[ -z "${FAKE_CONTROLLER_IMAGE_ID_FILE:-}" || -z "${FAKE_CONTROLLER_IMAGE_ID_STATE:-}" ]] || cp "$FAKE_CONTROLLER_IMAGE_ID_STATE" "$FAKE_CONTROLLER_IMAGE_ID_FILE"
         [[ -z "${FAKE_CONTROLLER_ENV_FILE:-}" ]] || cp "$env_file" "$FAKE_CONTROLLER_ENV_FILE"
@@ -175,10 +182,15 @@ case "${1:-}" in
       stop)
         if [[ -n "$paused_state" && -f "$paused_state" ]]; then exit 42; fi
         if [[ -n "${FAKE_STOP_FAIL:-}" && -f "$FAKE_STOP_FAIL" ]]; then exit 42; fi
-        rm -f "$state"; [[ -z "$status_file" ]] || rm -f "$status_file"; [[ -z "$paused_state" ]] || rm -f "$paused_state"; [[ -z "${FAKE_CONTROLLER_PROVENANCE_FILE:-}" ]] || rm -f "$FAKE_CONTROLLER_PROVENANCE_FILE"; [[ -z "${FAKE_CONTROLLER_IMAGE_ID_FILE:-}" ]] || rm -f "$FAKE_CONTROLLER_IMAGE_ID_FILE"; [[ -z "${FAKE_CONTROLLER_ENV_FILE:-}" ]] || rm -f "$FAKE_CONTROLLER_ENV_FILE"
+        rm -f "$state"; [[ -z "$status_file" ]] || rm -f "$status_file"; [[ -z "$paused_state" ]] || rm -f "$paused_state"
+        if [[ -n "$stopped_state" ]]; then
+          : >"$stopped_state"
+        else
+          [[ -z "${FAKE_CONTROLLER_PROVENANCE_FILE:-}" ]] || rm -f "$FAKE_CONTROLLER_PROVENANCE_FILE"; [[ -z "${FAKE_CONTROLLER_IMAGE_ID_FILE:-}" ]] || rm -f "$FAKE_CONTROLLER_IMAGE_ID_FILE"; [[ -z "${FAKE_CONTROLLER_ENV_FILE:-}" ]] || rm -f "$FAKE_CONTROLLER_ENV_FILE"
+        fi
         [[ -z "${FAKE_ACTIVE_MANAGED_AFTER_STOP:-}" ]] || : >"$FAKE_ACTIVE_MANAGED_AFTER_STOP"
         ;;
-      down|rm) rm -f "$state"; [[ -z "$status_file" ]] || rm -f "$status_file"; [[ -z "$paused_state" ]] || rm -f "$paused_state"; [[ -z "${FAKE_CONTROLLER_PROVENANCE_FILE:-}" ]] || rm -f "$FAKE_CONTROLLER_PROVENANCE_FILE"; [[ -z "${FAKE_CONTROLLER_IMAGE_ID_FILE:-}" ]] || rm -f "$FAKE_CONTROLLER_IMAGE_ID_FILE"; [[ -z "${FAKE_CONTROLLER_ENV_FILE:-}" ]] || rm -f "$FAKE_CONTROLLER_ENV_FILE" ;;
+      down|rm) rm -f "$state"; [[ -z "$status_file" ]] || rm -f "$status_file"; [[ -z "$paused_state" ]] || rm -f "$paused_state"; [[ -z "$stopped_state" ]] || rm -f "$stopped_state"; [[ -z "${FAKE_CONTROLLER_PROVENANCE_FILE:-}" ]] || rm -f "$FAKE_CONTROLLER_PROVENANCE_FILE"; [[ -z "${FAKE_CONTROLLER_IMAGE_ID_FILE:-}" ]] || rm -f "$FAKE_CONTROLLER_IMAGE_ID_FILE"; [[ -z "${FAKE_CONTROLLER_ENV_FILE:-}" ]] || rm -f "$FAKE_CONTROLLER_ENV_FILE" ;;
       pause)
         if [[ -n "$paused_state" && -f "$paused_state" ]]; then
           printf 'Error response from daemon: container is already paused\n' >&2
@@ -865,6 +877,34 @@ expect_success "$installer" --rollback >/dev/null
 printf '%s\n' "$engine_ref" >"$FAKE_RUNNER_IMAGE_STATE"
 printf '%s\n' "$prior_runner_image_id" >"$FAKE_RUNNER_IMAGE_ID_STATE"
 unset FAKE_COMPOSE_LOG
+absent_controller_output=$tmp/absent-controller-timer-rollback.out
+export FAKE_COMPOSE_LOG=$tmp/absent-controller-timer-rollback-compose.log
+export FAKE_STOPPED_CONTROLLER_STATE=$tmp/stopped-controller
+: >"$FAKE_COMPOSE_LOG"
+prior_controller_image_id=$(<"$FAKE_CONTROLLER_IMAGE_ID_STATE")
+rm -f "$FAKE_CONTROLLER_IMAGE_STATE" "$FAKE_CONTROLLER_IMAGE_ID_STATE"
+export FAKE_FAIL_TIMER_ENABLE=1
+if "$installer" --upgrade --config-repo "$config_repo" --config-identity fixture-org/fleet-config --controller example-ci-01 --ref "$ref_two" >"$absent_controller_output" 2>&1; then
+  fail 'absent-controller timer activation failure unexpectedly succeeded'
+fi
+unset FAKE_FAIL_TIMER_ENABLE
+grep -Fq 'ROLLBACK_RESTORED' "$absent_controller_output" || fail "absent-controller timer failure did not restore the checkpoint: $(<"$absent_controller_output")"
+[[ ! -e "$FAKE_CONTROLLER_IMAGE_STATE" && ! -e "$FAKE_CONTROLLER_IMAGE_ID_STATE" ]] || fail 'timer-failure rollback retained a controller tag that was previously absent'
+[[ -f "$FAKE_DOCKER_STATE" && ! -f "$FAKE_STOPPED_CONTROLLER_STATE" ]] || fail 'timer-failure rollback did not restart the prior controller'
+candidate_up_line=$(grep -n -m1 '^up|' "$FAKE_COMPOSE_LOG" | cut -d: -f1 || true)
+rollback_stop_line=$(grep -n '^stop|' "$FAKE_COMPOSE_LOG" | tail -n1 | cut -d: -f1 || true)
+candidate_rm_line=$(grep -n -m1 '^rm|' "$FAKE_COMPOSE_LOG" | cut -d: -f1 || true)
+controller_rm_line=$(grep -n -m1 "^image-rm|$FAKE_CONTROLLER_IMAGE$" "$FAKE_COMPOSE_LOG" | cut -d: -f1 || true)
+rollback_up_line=$(grep -n '^up|' "$FAKE_COMPOSE_LOG" | tail -n1 | cut -d: -f1 || true)
+[[ -n "$candidate_up_line" && -n "$rollback_stop_line" && -n "$candidate_rm_line" && -n "$controller_rm_line" && -n "$rollback_up_line" \
+  && "$candidate_up_line" -lt "$rollback_stop_line" && "$rollback_stop_line" -lt "$candidate_rm_line" && "$candidate_rm_line" -lt "$controller_rm_line" && "$controller_rm_line" -lt "$rollback_up_line" ]] \
+  || fail 'timer-failure rollback did not remove the stopped candidate before restoring image state and restarting'
+printf '%s\n' "$engine_ref" >"$FAKE_CONTROLLER_IMAGE_STATE"
+printf '%s\n' "$prior_controller_image_id" >"$FAKE_CONTROLLER_IMAGE_ID_STATE"
+printf '%s\n' "$prior_controller_image_id" >"$FAKE_CONTROLLER_IMAGE_ID_FILE"
+printf '%s\n' "$engine_ref" >"$FAKE_CONTROLLER_PROVENANCE_FILE"
+unset FAKE_STOPPED_CONTROLLER_STATE FAKE_COMPOSE_LOG
+[[ ${CI_FLEET_TEST_STOP_AFTER_ABSENT_CONTROLLER_ROLLBACK:-0} != 1 ]] || { printf 'ABSENT_CONTROLLER_ROLLBACK_REGRESSION_OK\n'; exit 0; }
 restartable_tag_build_output=$tmp/restartable-tag-build.out
 export FAKE_COMPOSE_LOG=$tmp/restartable-tag-build-compose.log
 : >"$FAKE_COMPOSE_LOG"

From ae7399019607c923aa9bc184006b9037615ee598 Mon Sep 17 00:00:00 2001
From: Nickfost <1572453+Nickfost@users.noreply.github.com>
Date: Fri, 4 Sep 2026 05:45:33 -0500
Subject: [PATCH 12/14] fix: preserve live image for absent controller tag

---
 scripts/install-worker-controller.sh      | 52 +++++++++++++++++------
 scripts/test-install-worker-controller.sh | 27 +++++++++---
 2 files changed, 60 insertions(+), 19 deletions(-)

diff --git a/scripts/install-worker-controller.sh b/scripts/install-worker-controller.sh
index 61d92d0a..15f8930d 100755
--- a/scripts/install-worker-controller.sh
+++ b/scripts/install-worker-controller.sh
@@ -700,7 +700,7 @@ build_candidate() {
 }
 
 load_checkpoint_images() {
-  local environment=$1 image_ids=${2:-} output expected=2
+  local environment=$1 image_ids=${2:-} output
   output=$(python3 - "$environment" "$image_ids" "$repo_root/scripts" <<'PY'
 import re
 import sys
@@ -719,19 +719,23 @@ for key in image_keys:
 if sys.argv[2]:
     id_keys = tuple(f"{key}_ID" for key in image_keys)
     values = parse_env(Path(sys.argv[2]), allow_unknown=True)
-    if set(values) != set(id_keys) or any(values[key] != "absent" and not re.fullmatch(r"sha256:[0-9a-f]{64}", values[key]) for key in id_keys):
+    live_key = "CI_FLEET_CONTROLLER_LIVE_IMAGE_ID"
+    if not set(id_keys).issubset(values) or not set(values) <= {*id_keys, live_key} or any(values[key] != "absent" and not re.fullmatch(r"sha256:[0-9a-f]{64}", values[key]) for key in id_keys):
+        raise SystemExit(1)
+    if live_key in values and (values[id_keys[1]] != "absent" or not re.fullmatch(r"sha256:[0-9a-f]{64}", values[live_key])):
         raise SystemExit(1)
     for key in id_keys:
         print(values[key])
+    if live_key in values:
+        print(values[live_key])
 PY
   ) || return 1
-  [[ -z "$image_ids" ]] || expected=4
   mapfile -t checkpoint_images <<<"$output"
-  [[ ${#checkpoint_images[@]} == "$expected" ]]
+  [[ ${#checkpoint_images[@]} == 2 || -n "$image_ids" && ( ${#checkpoint_images[@]} == 4 || ${#checkpoint_images[@]} == 5 ) ]]
 }
 
 make_checkpoint() {
-  local timestamp target unit timer final_checkpoint staged_checkpoint expected_owner=0 runner_id controller_id
+  local timestamp target unit timer final_checkpoint staged_checkpoint expected_owner=0 runner_id controller_id controller_live_id=
   timestamp=$(date -u +%Y%m%dT%H%M%SZ)
   final_checkpoint=$checkpoints_dir/${timestamp}-$$
   install -d -m 0700 "$checkpoints_dir"
@@ -753,10 +757,16 @@ make_checkpoint() {
     if ! controller_id=$(docker image inspect --format '{{.Id}}' "${checkpoint_images[1]}" 2>/dev/null); then
       docker info >/dev/null 2>&1 || die 'Docker daemon is unavailable'
       controller_id=absent
+      if ! controller_live_id=$(docker inspect --format '{{.Image}}' "$controller_container" 2>/dev/null); then
+        docker info >/dev/null 2>&1 || die 'Docker daemon is unavailable'
+        controller_live_id=
+      fi
     fi
     [[ "$runner_id" == absent || "$runner_id" =~ ^sha256:[0-9a-f]{64}$ ]] || die 'installed runner image ID is invalid'
     [[ "$controller_id" == absent || "$controller_id" =~ ^sha256:[0-9a-f]{64}$ ]] || die 'installed controller image ID is invalid'
+    [[ -z "$controller_live_id" || "$controller_live_id" =~ ^sha256:[0-9a-f]{64}$ ]] || die 'live controller image ID is invalid'
     printf 'CI_FLEET_RUNNER_IMAGE_ID=%s\nCI_FLEET_CONTROLLER_IMAGE_ID=%s\n' "$runner_id" "$controller_id" >"$checkpoint_dir/image-ids.env"
+    [[ -z "$controller_live_id" ]] || printf 'CI_FLEET_CONTROLLER_LIVE_IMAGE_ID=%s\n' "$controller_live_id" >>"$checkpoint_dir/image-ids.env"
     chmod 0600 "$checkpoint_dir/image-ids.env"
   fi
   [[ ! -f "$state_file" ]] || install -m 0600 "$state_file" "$checkpoint_dir/install-state.json"
@@ -1024,6 +1034,7 @@ restore_systemd_snapshot() {
 
 restore_checkpoint() {
   local target restored_state actual index expected_owner=0 failed=0 checkpoint_release='' drain_env=$rendered_env drain_release='' restore_images=false new_format=false
+  local remove_restored_controller_tag=false
   local format_marker=$checkpoint_dir/format-version image_ids=$checkpoint_dir/image-ids.env
   [[ -n "$checkpoint_dir" && -d "$checkpoint_dir" ]] || return 1
   [[ "$testing" != 1 ]] || expected_owner=$(id -u)
@@ -1104,15 +1115,22 @@ restore_checkpoint() {
   if $restore_images; then
     for index in 0 1; do
       if [[ ${checkpoint_images[index + 2]} == absent ]]; then
-        if docker image inspect --format '{{.Id}}' "${checkpoint_images[index]}" >/dev/null 2>&1; then
-          docker image rm "${checkpoint_images[index]}" >/dev/null || failed=1
-        else
-          docker info >/dev/null 2>&1 || failed=1
-        fi
-        if docker image inspect --format '{{.Id}}' "${checkpoint_images[index]}" >/dev/null 2>&1; then
-          failed=1
+        if [[ "$index" == 1 && ${#checkpoint_images[@]} == 5 ]]; then
+          docker image tag "${checkpoint_images[4]}" "${checkpoint_images[index]}" || failed=1
+          actual=$(docker image inspect --format '{{.Id}}' "${checkpoint_images[index]}" 2>/dev/null) || failed=1
+          [[ "$actual" == "${checkpoint_images[4]}" ]] || failed=1
+          remove_restored_controller_tag=true
         else
-          docker info >/dev/null 2>&1 || failed=1
+          if docker image inspect --format '{{.Id}}' "${checkpoint_images[index]}" >/dev/null 2>&1; then
+            docker image rm "${checkpoint_images[index]}" >/dev/null || failed=1
+          else
+            docker info >/dev/null 2>&1 || failed=1
+          fi
+          if docker image inspect --format '{{.Id}}' "${checkpoint_images[index]}" >/dev/null 2>&1; then
+            failed=1
+          else
+            docker info >/dev/null 2>&1 || failed=1
+          fi
         fi
       else
         docker image tag "${checkpoint_images[index + 2]}" "${checkpoint_images[index]}" || failed=1
@@ -1134,6 +1152,14 @@ restore_checkpoint() {
       fi
     fi
   fi
+  if $remove_restored_controller_tag && ((failed == 0)); then
+    docker image rm "${checkpoint_images[1]}" >/dev/null || failed=1
+    if docker image inspect --format '{{.Id}}' "${checkpoint_images[1]}" >/dev/null 2>&1; then
+      failed=1
+    else
+      docker info >/dev/null 2>&1 || failed=1
+    fi
+  fi
   set -e
   trap on_error ERR
   if ((failed != 0)); then
diff --git a/scripts/test-install-worker-controller.sh b/scripts/test-install-worker-controller.sh
index da9a0a53..e7a8db81 100755
--- a/scripts/test-install-worker-controller.sh
+++ b/scripts/test-install-worker-controller.sh
@@ -157,6 +157,11 @@ case "${1:-}" in
     fi
     case "$command" in
       up)
+        controller_image=$(awk -F= '$1 == "CI_FLEET_CONTROLLER_IMAGE" {print substr($0, index($0, "=") + 1)}' "$env_file")
+        "$0" image inspect --format '{{.Id}}' "$controller_image" >/dev/null 2>&1 || {
+          printf 'configured controller image is unavailable: %s\n' "$controller_image" >&2
+          exit 49
+        }
         if [[ -n "${FAKE_DELAY_UP_ONCE:-}" && -f "$FAKE_DELAY_UP_ONCE" ]]; then
           delay_marker=$FAKE_DELAY_UP_ONCE
           rm -f "$delay_marker"
@@ -881,24 +886,34 @@ absent_controller_output=$tmp/absent-controller-timer-rollback.out
 export FAKE_COMPOSE_LOG=$tmp/absent-controller-timer-rollback-compose.log
 export FAKE_STOPPED_CONTROLLER_STATE=$tmp/stopped-controller
 : >"$FAKE_COMPOSE_LOG"
-prior_controller_image_id=$(<"$FAKE_CONTROLLER_IMAGE_ID_STATE")
+prior_controller_image_id=sha256:4444444444444444444444444444444444444444444444444444444444444444
+printf '%s\n' "$prior_controller_image_id" >>"$FAKE_AVAILABLE_IMAGE_IDS"
+printf '%s\n' "$prior_controller_image_id" >"$FAKE_CONTROLLER_IMAGE_ID_STATE"
+printf '%s\n' "$prior_controller_image_id" >"$FAKE_CONTROLLER_IMAGE_ID_FILE"
 rm -f "$FAKE_CONTROLLER_IMAGE_STATE" "$FAKE_CONTROLLER_IMAGE_ID_STATE"
+[[ $(<"$FAKE_CONTROLLER_IMAGE_ID_FILE") == "$prior_controller_image_id" ]] || fail 'untagged running controller lost its exact image ID'
 export FAKE_FAIL_TIMER_ENABLE=1
 if "$installer" --upgrade --config-repo "$config_repo" --config-identity fixture-org/fleet-config --controller example-ci-01 --ref "$ref_two" >"$absent_controller_output" 2>&1; then
   fail 'absent-controller timer activation failure unexpectedly succeeded'
 fi
 unset FAKE_FAIL_TIMER_ENABLE
-grep -Fq 'ROLLBACK_RESTORED' "$absent_controller_output" || fail "absent-controller timer failure did not restore the checkpoint: $(<"$absent_controller_output")"
+grep -Fq 'ROLLBACK_RESTORED' "$absent_controller_output" || fail "absent-controller timer failure did not restore the checkpoint: $(<"$absent_controller_output"); compose log: $(<"$FAKE_COMPOSE_LOG")"
+checkpoint_path=$(awk '$1 == "CHECKPOINT_CREATED" {sub(/^path=/, "", $2); value=$2} END {print value}' "$absent_controller_output")
+grep -Fxq 'CI_FLEET_CONTROLLER_IMAGE_ID=absent' "$checkpoint_path/image-ids.env" || fail 'checkpoint did not preserve the absent controller tag'
+grep -Fxq "CI_FLEET_CONTROLLER_LIVE_IMAGE_ID=$prior_controller_image_id" "$checkpoint_path/image-ids.env" || fail 'checkpoint omitted the untagged live controller image ID'
 [[ ! -e "$FAKE_CONTROLLER_IMAGE_STATE" && ! -e "$FAKE_CONTROLLER_IMAGE_ID_STATE" ]] || fail 'timer-failure rollback retained a controller tag that was previously absent'
 [[ -f "$FAKE_DOCKER_STATE" && ! -f "$FAKE_STOPPED_CONTROLLER_STATE" ]] || fail 'timer-failure rollback did not restart the prior controller'
+[[ $(<"$FAKE_CONTROLLER_IMAGE_ID_FILE") == "$prior_controller_image_id" ]] || fail 'timer-failure rollback did not recreate the exact prior controller image'
 candidate_up_line=$(grep -n -m1 '^up|' "$FAKE_COMPOSE_LOG" | cut -d: -f1 || true)
 rollback_stop_line=$(grep -n '^stop|' "$FAKE_COMPOSE_LOG" | tail -n1 | cut -d: -f1 || true)
 candidate_rm_line=$(grep -n -m1 '^rm|' "$FAKE_COMPOSE_LOG" | cut -d: -f1 || true)
-controller_rm_line=$(grep -n -m1 "^image-rm|$FAKE_CONTROLLER_IMAGE$" "$FAKE_COMPOSE_LOG" | cut -d: -f1 || true)
+controller_tag_line=$(grep -n -m1 "^image-tag|$prior_controller_image_id|$FAKE_CONTROLLER_IMAGE$" "$FAKE_COMPOSE_LOG" | cut -d: -f1 || true)
 rollback_up_line=$(grep -n '^up|' "$FAKE_COMPOSE_LOG" | tail -n1 | cut -d: -f1 || true)
-[[ -n "$candidate_up_line" && -n "$rollback_stop_line" && -n "$candidate_rm_line" && -n "$controller_rm_line" && -n "$rollback_up_line" \
-  && "$candidate_up_line" -lt "$rollback_stop_line" && "$rollback_stop_line" -lt "$candidate_rm_line" && "$candidate_rm_line" -lt "$controller_rm_line" && "$controller_rm_line" -lt "$rollback_up_line" ]] \
-  || fail 'timer-failure rollback did not remove the stopped candidate before restoring image state and restarting'
+controller_rm_line=$(grep -n "^image-rm|$FAKE_CONTROLLER_IMAGE$" "$FAKE_COMPOSE_LOG" | tail -n1 | cut -d: -f1 || true)
+[[ -n "$candidate_up_line" && -n "$rollback_stop_line" && -n "$candidate_rm_line" && -n "$controller_tag_line" && -n "$rollback_up_line" && -n "$controller_rm_line" \
+  && "$candidate_up_line" -lt "$rollback_stop_line" && "$rollback_stop_line" -lt "$candidate_rm_line" && "$candidate_rm_line" -lt "$controller_tag_line" \
+  && "$controller_tag_line" -lt "$rollback_up_line" && "$rollback_up_line" -lt "$controller_rm_line" ]] \
+  || fail 'timer-failure rollback did not remove the candidate, recreate the prior controller, and remove its temporary tag'
 printf '%s\n' "$engine_ref" >"$FAKE_CONTROLLER_IMAGE_STATE"
 printf '%s\n' "$prior_controller_image_id" >"$FAKE_CONTROLLER_IMAGE_ID_STATE"
 printf '%s\n' "$prior_controller_image_id" >"$FAKE_CONTROLLER_IMAGE_ID_FILE"

From 81682202c2cbc54fb5457c656cc0d3fbc8d7bc60 Mon Sep 17 00:00:00 2001
From: Nick <1572453+Nickfost@users.noreply.github.com>
Date: Fri, 4 Sep 2026 07:45:58 -0500
Subject: [PATCH 13/14] fix: drain controllers without current release link

---
 scripts/install-worker-controller.sh      | 15 ++++++++----
 scripts/test-install-worker-controller.sh | 30 ++++++++++++++++++++---
 2 files changed, 37 insertions(+), 8 deletions(-)

diff --git a/scripts/install-worker-controller.sh b/scripts/install-worker-controller.sh
index 15f8930d..8a31cd21 100755
--- a/scripts/install-worker-controller.sh
+++ b/scripts/install-worker-controller.sh
@@ -888,7 +888,7 @@ try_drain_current() {
 }
 
 drain_current() {
-  try_drain_current || die "$drain_error"
+  try_drain_current "$@" || die "$drain_error"
 }
 
 install_systemd_units() {
@@ -1247,7 +1247,7 @@ perform_converge() {
   elif [[ "$mode" == adopt ]]; then
     die '--adopt requires a trusted installed controller identity'
   fi
-  drain_current
+  drain_current false "$rendered_env" "$release_dir"
   if [[ "$testing" == 1 && -n ${CI_FLEET_TEST_PAUSE_AFTER_DRAIN_FILE:-} ]]; then
     : >"$CI_FLEET_TEST_PAUSE_AFTER_DRAIN_FILE"
     while [[ ! -f "$CI_FLEET_TEST_PAUSE_AFTER_DRAIN_FILE.continue" ]]; do sleep 0.05; done
@@ -1275,12 +1275,17 @@ perform_rollback() {
 }
 
 perform_uninstall() {
-  local old_release=
+  local candidate old_release='' old_ref=''
   load_installed_controller_identity
-  old_release=$(current_runtime_release)
+  for candidate in "$(current_runtime_release)" "$(readlink -f "$manager_current" 2>/dev/null || true)" "$repo_root"; do
+    [[ -n "$candidate" && -f "$candidate/.ci-fleet-engine-ref" ]] || continue
+    old_ref=$(<"$candidate/.ci-fleet-engine-ref")
+    if [[ "$old_ref" =~ ^[0-9a-f]{40}$ ]] && runtime_release_complete "$candidate" "$old_ref"; then old_release=$candidate; break; fi
+  done
+  [[ -n "$old_release" ]] || die 'a trusted complete release is required to uninstall the controller'
   make_checkpoint
   transaction_active=true
-  drain_current
+  drain_current false "$rendered_env" "$old_release"
   remove_inactive_managed_runners
   if [[ -n "$old_release" && -f "$rendered_env" ]]; then
     compose "$old_release" "$rendered_env" down --remove-orphans || true
diff --git a/scripts/test-install-worker-controller.sh b/scripts/test-install-worker-controller.sh
index e7a8db81..8512d6b3 100755
--- a/scripts/test-install-worker-controller.sh
+++ b/scripts/test-install-worker-controller.sh
@@ -1110,9 +1110,19 @@ expect_success "$installer" --rollback >/dev/null
 grep -Fq 'CI_FLEET_MAX_RUNNERS=1' "$root/etc/ci-fleet/ci-fleet.env" || fail 'rollback did not restore capacity one'
 
 ref_three=$(write_config drained 2 2)
+missing_current_output=$tmp/missing-current-restartable.out
+export FAKE_COMPOSE_LOG=$tmp/missing-current-restartable-compose.log
+: >"$FAKE_COMPOSE_LOG"
+rm -f "$root/opt/ci-fleet/current"
 printf 'dead\n' >"$FAKE_CONTROLLER_STATUS_FILE"
-expect_success "$installer" --upgrade "${base_args[@]}" --ref "$ref_three" >/dev/null
+if ! "$installer" --upgrade "${base_args[@]}" --ref "$ref_three" >"$missing_current_output" 2>&1; then
+  grep -Fq 'cannot stop restartable controller state without its runtime release: dead' "$missing_current_output" || fail "missing-current convergence failed unexpectedly: $(<"$missing_current_output")"
+  if grep -Eq '^(stop|build|up|down|rm)\|' "$FAKE_COMPOSE_LOG"; then fail 'missing-current convergence mutated the controller before validating a drain release'; fi
+  fail "missing-current convergence did not use the validated candidate release: $(<"$missing_current_output")"
+fi
+unset FAKE_COMPOSE_LOG
 [[ ! -f "$FAKE_DOCKER_STATE" && ! -f "$FAKE_CONTROLLER_STATUS_FILE" ]] || fail 'non-active convergence retained a dead controller'
+[[ $(readlink -f "$root/opt/ci-fleet/current") == "$root/opt/ci-fleet/releases/$engine_ref" ]] || fail 'missing-current convergence did not repair the current release link'
 grep -Fq 'CI_FLEET_CONTROLLER_STATE=drained' "$root/etc/ci-fleet/ci-fleet.env" || fail 'drained state was not rendered'
 grep -Fq 'CI_FLEET_MAX_RUNNERS=0' "$root/etc/ci-fleet/ci-fleet.env" || fail 'drained controller retained effective capacity'
 [[ ! -f "$FAKE_DOCKER_STATE" ]] || fail 'drained controller remained running'
@@ -1147,16 +1157,30 @@ export FAKE_ALL_RUNNER_STATE=$tmp/uninstall-stopped-managed-runner
 mkdir -p "$root/var/lib/ci-fleet/health"
 printf '{"status":"healthy"}\n' >"$root/var/lib/ci-fleet/health/latest.json"
 : >"$FAKE_DOCKER_PS_LOG"
-expect_success "$installer" --uninstall >/dev/null
+uninstall_output=$tmp/dangling-current-uninstall.out
+export FAKE_COMPOSE_LOG=$tmp/dangling-current-uninstall-compose.log
+: >"$FAKE_COMPOSE_LOG"
+export FAKE_STOPPED_CONTROLLER_STATE=$tmp/uninstall-created-controller
+: >"$FAKE_STOPPED_CONTROLLER_STATE"
+printf 'created\n' >"$FAKE_CONTROLLER_STATUS_FILE"
+ln -sfn "$root/opt/ci-fleet/releases/missing/release" "$root/opt/ci-fleet/current"
+if ! "$installer" --uninstall >"$uninstall_output" 2>&1; then
+  grep -Fq 'cannot stop restartable controller state without its runtime release: created' "$uninstall_output" || fail "dangling-current uninstall failed unexpectedly: $(<"$uninstall_output")"
+  if grep -Eq '^(stop|build|up|down|rm)\|' "$FAKE_COMPOSE_LOG"; then fail 'dangling-current uninstall mutated the controller before validating a drain release'; fi
+  fail "dangling-current uninstall did not use a validated installed release: $(<"$uninstall_output")"
+fi
+unset FAKE_STOPPED_CONTROLLER_STATE FAKE_COMPOSE_LOG
+grep -Fq 'UNINSTALL_OK' "$uninstall_output" || fail 'dangling-current uninstall did not complete'
 [[ ! -f "$FAKE_RUNNER_STATE_ONCE" ]] || fail 'uninstall did not wait for an orphaned managed runner'
 [[ ! -f "$FAKE_ALL_RUNNER_STATE" ]] || fail 'uninstall retained stopped managed runners'
 grep -Fq 'label=io.randomdevelopment.ci-fleet.instance=example-ci-01' "$FAKE_DOCKER_PS_LOG" || fail 'uninstall runner cleanup was not scoped to the installed instance'
 if grep -Eq 'label=io.randomdevelopment.ci-fleet.instance=$' "$FAKE_DOCKER_PS_LOG"; then fail 'uninstall runner cleanup used an empty instance filter'; fi
 unset FAKE_RUNNER_STATE_ONCE FAKE_ALL_RUNNER_STATE
-[[ ! -e "$root/opt/ci-fleet/current" && ! -e "$root/var/lib/ci-fleet/install-state.json" ]] || fail 'uninstall left active installation state'
+[[ ! -e "$root/opt/ci-fleet/current" && ! -L "$root/opt/ci-fleet/current" && ! -e "$root/var/lib/ci-fleet/install-state.json" ]] || fail 'uninstall left active installation state'
 [[ -f "$host_config" && -f "$pem" ]] || fail 'uninstall removed preserved host credentials'
 [[ -f "$root/etc/ci-fleet/monitoring.env" ]] || fail 'uninstall removed host-local monitoring configuration'
 [[ ! -e "$root/var/lib/ci-fleet/health" ]] || fail 'uninstall retained fleet-owned health state'
+[[ ${CI_FLEET_TEST_STOP_AFTER_CURRENT_LINK_FALLBACK:-0} != 1 ]] || { printf 'CURRENT_LINK_FALLBACK_REGRESSIONS_OK\n'; exit 0; }
 
 adopt_root=$tmp/adopt-host
 export CI_FLEET_ROOT_PREFIX=$adopt_root

From e473725020b4be98d41502192ccc2e24e963811f Mon Sep 17 00:00:00 2001
From: Nickfost <1572453+Nickfost@users.noreply.github.com>
Date: Fri, 4 Sep 2026 09:18:09 -0500
Subject: [PATCH 14/14] fix: preserve exact rollback image state

---
 scripts/install-worker-controller.sh      | 64 +++++++++++++----------
 scripts/test-install-worker-controller.sh | 61 ++++++++++++++++++++-
 2 files changed, 97 insertions(+), 28 deletions(-)

diff --git a/scripts/install-worker-controller.sh b/scripts/install-worker-controller.sh
index 8a31cd21..4e3104cb 100755
--- a/scripts/install-worker-controller.sh
+++ b/scripts/install-worker-controller.sh
@@ -722,7 +722,7 @@ if sys.argv[2]:
     live_key = "CI_FLEET_CONTROLLER_LIVE_IMAGE_ID"
     if not set(id_keys).issubset(values) or not set(values) <= {*id_keys, live_key} or any(values[key] != "absent" and not re.fullmatch(r"sha256:[0-9a-f]{64}", values[key]) for key in id_keys):
         raise SystemExit(1)
-    if live_key in values and (values[id_keys[1]] != "absent" or not re.fullmatch(r"sha256:[0-9a-f]{64}", values[live_key])):
+    if live_key in values and (not re.fullmatch(r"sha256:[0-9a-f]{64}", values[live_key]) or values[live_key] == values[id_keys[1]]):
         raise SystemExit(1)
     for key in id_keys:
         print(values[key])
@@ -757,10 +757,12 @@ make_checkpoint() {
     if ! controller_id=$(docker image inspect --format '{{.Id}}' "${checkpoint_images[1]}" 2>/dev/null); then
       docker info >/dev/null 2>&1 || die 'Docker daemon is unavailable'
       controller_id=absent
-      if ! controller_live_id=$(docker inspect --format '{{.Image}}' "$controller_container" 2>/dev/null); then
-        docker info >/dev/null 2>&1 || die 'Docker daemon is unavailable'
-        controller_live_id=
-      fi
+    fi
+    if controller_live_id=$(docker inspect --format '{{.Image}}' "$controller_container" 2>/dev/null); then
+      [[ "$controller_live_id" != "$controller_id" ]] || controller_live_id=
+    else
+      docker info >/dev/null 2>&1 || die 'Docker daemon is unavailable'
+      controller_live_id=
     fi
     [[ "$runner_id" == absent || "$runner_id" =~ ^sha256:[0-9a-f]{64}$ ]] || die 'installed runner image ID is invalid'
     [[ "$controller_id" == absent || "$controller_id" =~ ^sha256:[0-9a-f]{64}$ ]] || die 'installed controller image ID is invalid'
@@ -1034,7 +1036,7 @@ restore_systemd_snapshot() {
 
 restore_checkpoint() {
   local target restored_state actual index expected_owner=0 failed=0 checkpoint_release='' drain_env=$rendered_env drain_release='' restore_images=false new_format=false
-  local remove_restored_controller_tag=false
+  local restore_controller_tag_after_start=false
   local format_marker=$checkpoint_dir/format-version image_ids=$checkpoint_dir/image-ids.env
   [[ -n "$checkpoint_dir" && -d "$checkpoint_dir" ]] || return 1
   [[ "$testing" != 1 ]] || expected_owner=$(id -u)
@@ -1075,6 +1077,10 @@ restore_checkpoint() {
     note "ROLLBACK_FAILED reason=$drain_error"
     return 1
   fi
+  if ! remove_inactive_managed_runners; then
+    note 'ROLLBACK_FAILED reason=could not remove inactive managed runners'
+    return 1
+  fi
   if [[ -f "$checkpoint_dir/install-state.json" || -f "$checkpoint_dir/ci-fleet.env" ]]; then
     load_installed_controller_identity "$checkpoint_dir/install-state.json" "$checkpoint_dir/ci-fleet.env"
   fi
@@ -1114,23 +1120,21 @@ restore_checkpoint() {
   fi
   if $restore_images; then
     for index in 0 1; do
-      if [[ ${checkpoint_images[index + 2]} == absent ]]; then
-        if [[ "$index" == 1 && ${#checkpoint_images[@]} == 5 ]]; then
-          docker image tag "${checkpoint_images[4]}" "${checkpoint_images[index]}" || failed=1
-          actual=$(docker image inspect --format '{{.Id}}' "${checkpoint_images[index]}" 2>/dev/null) || failed=1
-          [[ "$actual" == "${checkpoint_images[4]}" ]] || failed=1
-          remove_restored_controller_tag=true
+      if [[ "$index" == 1 && ${#checkpoint_images[@]} == 5 ]]; then
+        docker image tag "${checkpoint_images[4]}" "${checkpoint_images[index]}" || failed=1
+        actual=$(docker image inspect --format '{{.Id}}' "${checkpoint_images[index]}" 2>/dev/null) || failed=1
+        [[ "$actual" == "${checkpoint_images[4]}" ]] || failed=1
+        restore_controller_tag_after_start=true
+      elif [[ ${checkpoint_images[index + 2]} == absent ]]; then
+        if docker image inspect --format '{{.Id}}' "${checkpoint_images[index]}" >/dev/null 2>&1; then
+          docker image rm "${checkpoint_images[index]}" >/dev/null || failed=1
         else
-          if docker image inspect --format '{{.Id}}' "${checkpoint_images[index]}" >/dev/null 2>&1; then
-            docker image rm "${checkpoint_images[index]}" >/dev/null || failed=1
-          else
-            docker info >/dev/null 2>&1 || failed=1
-          fi
-          if docker image inspect --format '{{.Id}}' "${checkpoint_images[index]}" >/dev/null 2>&1; then
-            failed=1
-          else
-            docker info >/dev/null 2>&1 || failed=1
-          fi
+          docker info >/dev/null 2>&1 || failed=1
+        fi
+        if docker image inspect --format '{{.Id}}' "${checkpoint_images[index]}" >/dev/null 2>&1; then
+          failed=1
+        else
+          docker info >/dev/null 2>&1 || failed=1
         fi
       else
         docker image tag "${checkpoint_images[index + 2]}" "${checkpoint_images[index]}" || failed=1
@@ -1152,12 +1156,18 @@ restore_checkpoint() {
       fi
     fi
   fi
-  if $remove_restored_controller_tag && ((failed == 0)); then
-    docker image rm "${checkpoint_images[1]}" >/dev/null || failed=1
-    if docker image inspect --format '{{.Id}}' "${checkpoint_images[1]}" >/dev/null 2>&1; then
-      failed=1
+  if $restore_controller_tag_after_start && ((failed == 0)); then
+    if [[ ${checkpoint_images[3]} == absent ]]; then
+      docker image rm "${checkpoint_images[1]}" >/dev/null || failed=1
+      if docker image inspect --format '{{.Id}}' "${checkpoint_images[1]}" >/dev/null 2>&1; then
+        failed=1
+      else
+        docker info >/dev/null 2>&1 || failed=1
+      fi
     else
-      docker info >/dev/null 2>&1 || failed=1
+      docker image tag "${checkpoint_images[3]}" "${checkpoint_images[1]}" || failed=1
+      actual=$(docker image inspect --format '{{.Id}}' "${checkpoint_images[1]}" 2>/dev/null) || failed=1
+      [[ "$actual" == "${checkpoint_images[3]}" ]] || failed=1
     fi
   fi
   set -e
diff --git a/scripts/test-install-worker-controller.sh b/scripts/test-install-worker-controller.sh
index 8512d6b3..e8498dcb 100755
--- a/scripts/test-install-worker-controller.sh
+++ b/scripts/test-install-worker-controller.sh
@@ -113,14 +113,20 @@ case "${1:-}" in
       rm)
         image=${3:-}
         if [[ "$image" == "${FAKE_RUNNER_IMAGE:-}" || "$image" == "${FAKE_PRIOR_RUNNER_IMAGE:-}" || "$image" == "${FAKE_PREVIOUS_RUNNER_IMAGE:-}" ]]; then
+          image_kind=runner
           image_state=${FAKE_RUNNER_IMAGE_STATE:-}
           image_id_state=${FAKE_RUNNER_IMAGE_ID_STATE:-}
         elif [[ "$image" == "${FAKE_CONTROLLER_IMAGE:-}" || "$image" == "${FAKE_PRIOR_CONTROLLER_IMAGE:-}" || "$image" == "${FAKE_PREVIOUS_CONTROLLER_IMAGE:-}" ]]; then
+          image_kind=controller
           image_state=${FAKE_CONTROLLER_IMAGE_STATE:-}
           image_id_state=${FAKE_CONTROLLER_IMAGE_ID_STATE:-}
         else
           exit 1
         fi
+        if [[ "$image_kind" == runner && -n "${FAKE_ALL_RUNNER_STATE:-}" && -f "$FAKE_ALL_RUNNER_STATE" ]]; then
+          [[ -z "${FAKE_COMPOSE_LOG:-}" ]] || printf 'image-rm-blocked|%s\n' "$image" >>"$FAKE_COMPOSE_LOG"
+          exit 48
+        fi
         if [[ -n "$stopped_state" && -f "$stopped_state" && -n "${FAKE_CONTROLLER_IMAGE_ID_FILE:-}" && -f "$FAKE_CONTROLLER_IMAGE_ID_FILE" && -f "$image_id_state" ]] \
           && cmp -s "$FAKE_CONTROLLER_IMAGE_ID_FILE" "$image_id_state"; then
           [[ -z "${FAKE_COMPOSE_LOG:-}" ]] || printf 'image-rm-blocked|%s\n' "$image" >>"$FAKE_COMPOSE_LOG"
@@ -134,6 +140,7 @@ case "${1:-}" in
     ;;
   rm)
     (($# >= 2)) || exit 1
+    [[ -z "${FAKE_COMPOSE_LOG:-}" ]] || printf 'container-rm|%s\n' "$*" >>"$FAKE_COMPOSE_LOG"
     [[ -z "${FAKE_ALL_RUNNER_STATE:-}" ]] || rm -f "$FAKE_ALL_RUNNER_STATE"
     ;;
   volume|network)
@@ -241,6 +248,7 @@ if [[ "${1:-}" == enable && "${2:-}" == --now && ! -f "${CI_FLEET_ROOT_PREFIX:-}
   exit 98
 fi
 if [[ -n "${FAKE_FAIL_TIMER_ENABLE:-}" && "${1:-}" == enable && "${2:-}" == --now && "$*" == *ci-fleet-reconcile.timer* ]]; then
+  [[ -z "${FAKE_CREATE_STOPPED_RUNNER_ON_TIMER_FAILURE:-}" || -z "${FAKE_ALL_RUNNER_STATE:-}" ]] || : >"$FAKE_ALL_RUNNER_STATE"
   exit 97
 fi
 if [[ -n "${FAKE_DISABLED_TIMER:-}" && ( "${1:-}" == is-enabled || "${1:-}" == is-active ) && $# == 3 && "${3:-}" == "$FAKE_DISABLED_TIMER" ]]; then
@@ -583,7 +591,7 @@ printf '%040d\n' 0 >"$FAKE_CONTROLLER_PROVENANCE_FILE"
 expect_failure 'DRIFT controller_runtime' "$installer" --check "${base_args[@]}" --ref "$ref_one"
 expect_success "$installer" --install "${base_args[@]}" --ref "$ref_one" >/dev/null
 [[ $(<"$FAKE_CONTROLLER_PROVENANCE_FILE") == "$engine_ref" ]] || fail 'controller convergence did not restore running image provenance'
-printf 'sha256:%040d\n' 0 >"$FAKE_CONTROLLER_IMAGE_ID_FILE"
+printf 'sha256:%064d\n' 0 >"$FAKE_CONTROLLER_IMAGE_ID_FILE"
 expect_failure 'DRIFT controller_runtime' "$installer" --check "${base_args[@]}" --ref "$ref_one"
 expect_success "$installer" --install "${base_args[@]}" --ref "$ref_one" >/dev/null
 [[ $(<"$FAKE_CONTROLLER_IMAGE_ID_FILE") == "$FAKE_CANDIDATE_CONTROLLER_IMAGE_ID" ]] || fail 'controller convergence did not restore live image identity'
@@ -920,6 +928,57 @@ printf '%s\n' "$prior_controller_image_id" >"$FAKE_CONTROLLER_IMAGE_ID_FILE"
 printf '%s\n' "$engine_ref" >"$FAKE_CONTROLLER_PROVENANCE_FILE"
 unset FAKE_STOPPED_CONTROLLER_STATE FAKE_COMPOSE_LOG
 [[ ${CI_FLEET_TEST_STOP_AFTER_ABSENT_CONTROLLER_ROLLBACK:-0} != 1 ]] || { printf 'ABSENT_CONTROLLER_ROLLBACK_REGRESSION_OK\n'; exit 0; }
+drifted_controller_output=$tmp/drifted-controller-timer-rollback.out
+export FAKE_COMPOSE_LOG=$tmp/drifted-controller-timer-rollback-compose.log
+: >"$FAKE_COMPOSE_LOG"
+drifted_live_controller_image_id=sha256:5555555555555555555555555555555555555555555555555555555555555555
+drifted_tag_controller_image_id=sha256:6666666666666666666666666666666666666666666666666666666666666666
+printf '%s\n%s\n' "$drifted_live_controller_image_id" "$drifted_tag_controller_image_id" >>"$FAKE_AVAILABLE_IMAGE_IDS"
+printf '%s\n' "$engine_ref" >"$FAKE_CONTROLLER_IMAGE_STATE"
+printf '%s\n' "$drifted_tag_controller_image_id" >"$FAKE_CONTROLLER_IMAGE_ID_STATE"
+printf '%s\n' "$drifted_live_controller_image_id" >"$FAKE_CONTROLLER_IMAGE_ID_FILE"
+export FAKE_FAIL_TIMER_ENABLE=1
+if "$installer" --upgrade --config-repo "$config_repo" --config-identity fixture-org/fleet-config --controller example-ci-01 --ref "$ref_two" >"$drifted_controller_output" 2>&1; then
+  fail 'drifted-controller timer activation failure unexpectedly succeeded'
+fi
+unset FAKE_FAIL_TIMER_ENABLE
+grep -Fq 'CHECKPOINT_CREATED' "$drifted_controller_output" || fail 'drifted-controller timer failure changed state before checkpointing'
+grep -Fq 'DRAIN_OK managed_runners=0' "$drifted_controller_output" || fail 'drifted-controller timer failure did not drain before activation'
+grep -Fq 'ROLLBACK_RESTORED' "$drifted_controller_output" || fail "drifted-controller timer failure did not restore the checkpoint: $(<"$drifted_controller_output"); compose log: $(<"$FAKE_COMPOSE_LOG")"
+[[ $(<"$FAKE_CONTROLLER_IMAGE_ID_STATE") == "$drifted_tag_controller_image_id" ]] || fail 'drifted-controller rollback did not preserve the prior tag mapping'
+[[ $(<"$FAKE_CONTROLLER_IMAGE_ID_FILE") == "$drifted_live_controller_image_id" ]] || fail 'drifted-controller rollback did not recreate the exact prior controller image'
+checkpoint_path=$(awk '$1 == "CHECKPOINT_CREATED" {sub(/^path=/, "", $2); value=$2} END {print value}' "$drifted_controller_output")
+grep -Fxq "CI_FLEET_CONTROLLER_IMAGE_ID=$drifted_tag_controller_image_id" "$checkpoint_path/image-ids.env" || fail 'checkpoint omitted the drifted controller tag image ID'
+grep -Fxq "CI_FLEET_CONTROLLER_LIVE_IMAGE_ID=$drifted_live_controller_image_id" "$checkpoint_path/image-ids.env" || fail 'checkpoint omitted the drifted live controller image ID'
+unset FAKE_COMPOSE_LOG
+[[ ${CI_FLEET_TEST_STOP_AFTER_DRIFTED_CONTROLLER_ROLLBACK:-0} != 1 ]] || { printf 'DRIFTED_CONTROLLER_ROLLBACK_REGRESSION_OK\n'; exit 0; }
+printf '%s\n' "$drifted_tag_controller_image_id" >"$FAKE_CONTROLLER_IMAGE_ID_FILE"
+stopped_runner_output=$tmp/stopped-runner-timer-rollback.out
+export FAKE_COMPOSE_LOG=$tmp/stopped-runner-timer-rollback-compose.log
+export FAKE_ALL_RUNNER_STATE=$tmp/stopped-candidate-runner
+: >"$FAKE_COMPOSE_LOG"
+rm -f "$FAKE_RUNNER_IMAGE_STATE" "$FAKE_RUNNER_IMAGE_ID_STATE" "$FAKE_ALL_RUNNER_STATE"
+export FAKE_CREATE_STOPPED_RUNNER_ON_TIMER_FAILURE=1
+export FAKE_FAIL_TIMER_ENABLE=1
+if "$installer" --upgrade --config-repo "$config_repo" --config-identity fixture-org/fleet-config --controller example-ci-01 --ref "$ref_two" >"$stopped_runner_output" 2>&1; then
+  fail 'stopped-runner timer activation failure unexpectedly succeeded'
+fi
+unset FAKE_CREATE_STOPPED_RUNNER_ON_TIMER_FAILURE FAKE_FAIL_TIMER_ENABLE
+grep -Fq 'CHECKPOINT_CREATED' "$stopped_runner_output" || fail 'stopped-runner timer failure changed state before checkpointing'
+grep -Fq 'DRAIN_OK managed_runners=0' "$stopped_runner_output" || fail 'stopped-runner timer failure did not drain before activation'
+grep -Fq 'ROLLBACK_RESTORED' "$stopped_runner_output" || fail "stopped candidate runner blocked absent-tag rollback: $(<"$stopped_runner_output"); compose log: $(<"$FAKE_COMPOSE_LOG")"
+checkpoint_path=$(awk '$1 == "CHECKPOINT_CREATED" {sub(/^path=/, "", $2); value=$2} END {print value}' "$stopped_runner_output")
+grep -Fxq 'CI_FLEET_RUNNER_IMAGE_ID=absent' "$checkpoint_path/image-ids.env" || fail 'stopped-runner checkpoint did not preserve the absent runner tag'
+[[ ! -e "$FAKE_ALL_RUNNER_STATE" ]] || fail 'rollback retained the stopped candidate runner'
+[[ ! -e "$FAKE_RUNNER_IMAGE_STATE" && ! -e "$FAKE_RUNNER_IMAGE_ID_STATE" ]] || fail 'stopped-runner rollback retained a runner tag that was previously absent'
+container_rm_line=$(grep -n -m1 '^container-rm|' "$FAKE_COMPOSE_LOG" | cut -d: -f1 || true)
+runner_rm_line=$(grep -n -m1 "^image-rm|$FAKE_RUNNER_IMAGE$" "$FAKE_COMPOSE_LOG" | cut -d: -f1 || true)
+rollback_up_line=$(grep -n '^up|' "$FAKE_COMPOSE_LOG" | tail -n1 | cut -d: -f1 || true)
+[[ -n "$container_rm_line" && -n "$runner_rm_line" && -n "$rollback_up_line" && "$container_rm_line" -lt "$runner_rm_line" && "$runner_rm_line" -lt "$rollback_up_line" ]] || fail 'rollback did not remove the stopped candidate runner before restoring images and restarting the controller'
+printf '%s\n' "$engine_ref" >"$FAKE_RUNNER_IMAGE_STATE"
+printf '%s\n' "$prior_runner_image_id" >"$FAKE_RUNNER_IMAGE_ID_STATE"
+unset FAKE_ALL_RUNNER_STATE FAKE_COMPOSE_LOG
+[[ ${CI_FLEET_TEST_STOP_AFTER_STOPPED_RUNNER_ROLLBACK:-0} != 1 ]] || { printf 'STOPPED_RUNNER_ROLLBACK_REGRESSION_OK\n'; exit 0; }
 restartable_tag_build_output=$tmp/restartable-tag-build.out
 export FAKE_COMPOSE_LOG=$tmp/restartable-tag-build-compose.log
 : >"$FAKE_COMPOSE_LOG"