fix: keep candidate builds outside upgrade transactions

controller/main.go

@@ -76,7 +76,9 @@ func run(ctx context.Context) error {
 	scaler := &Scaler{runners: newRunnerState(), dockerClient: docker, scalesetClient: client, logger: logger, config: cfg, scaleSetID: set.ID}
 	if err := scaler.recoverStale(ctx); err != nil { return err }
 	scaler.writeStatus()
-	go scaler.publishStatus(ctx, time.Minute)
+	statusTicker := time.NewTicker(time.Minute)
+	defer statusTicker.Stop()
+	go scaler.publishStatus(ctx, statusTicker.C)
 	defer scaler.shutdown(context.WithoutCancel(ctx))
 	hostname, err := os.Hostname()
 	if err != nil { return fmt.Errorf("get hostname: %w", err) }

controller/status.go

@@ -53,14 +53,12 @@ func (s *Scaler) writeStatus() {
 	if err != nil { s.logger.Warn("write controller status", "error", err) }
 }
 
-func (s *Scaler) publishStatus(ctx context.Context, interval time.Duration) {
-	ticker := time.NewTicker(interval)
-	defer ticker.Stop()
+func (s *Scaler) publishStatus(ctx context.Context, ticks <-chan time.Time) {
 	for {
 		select {
 		case <-ctx.Done():
 			return
-		case <-ticker.C:
+		case <-ticks:
 			s.writeStatus()
 		}
 	}

controller/status_test.go

@@ -113,26 +113,19 @@ func TestStatusPublisherRefreshesIdleSnapshot(t *testing.T) {
 	}
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	done := make(chan struct{})
-	go func() {
-		scaler.publishStatus(ctx, time.Millisecond)
-		close(done)
-	}()
-	deadline := time.After(time.Second)
-	for {
-		if _, err := os.Stat(path); err == nil {
-			cancel()
-			select {
-			case <-done:
-			case <-time.After(time.Second):
-				t.Fatal("idle status publisher did not stop")
-			}
-			return
-		}
-		select {
-		case <-deadline:
-			t.Fatal("idle status publisher did not refresh snapshot")
-		case <-time.After(time.Millisecond):
-		}
+	original := encodeControllerStatus
+	encodeControllerStatus = func(file *os.File, value controllerStatus) error {
+		err := original(file, value)
+		cancel()
+		return err
 	}
+	defer func() { encodeControllerStatus = original }()
+	ticks := make(chan time.Time, 1)
+	ticks <- time.Now()
+	scaler.publishStatus(ctx, ticks)
+	body, err := os.ReadFile(path)
+	if err != nil { t.Fatal("idle status publisher did not refresh snapshot") }
+	var got controllerStatus
+	if err := json.Unmarshal(body, &got); err != nil { t.Fatal(err) }
+	if got.Current != 0 || got.Busy != 0 { t.Fatalf("published non-idle status: %+v", got) }
 }

docs/ADDING-A-HOST.md

@@ -102,7 +102,7 @@ sudo ./scripts/install-worker-controller.sh \
   --controller example-ci-01
 ```
 
-Use `--adopt` when converting an existing manual controller. The installer validates the complete configuration, renders host-local runtime state, installs the pinned engine, creates a controller checkpoint, builds images, installs maintenance timers, and verifies health.
+Use `--adopt` when converting an existing manual controller. The installer validates the complete configuration, renders host-local runtime state, installs the pinned engine, and validates the candidate Compose configuration. It may build images with a distinct inert runner tag before creating a checkpoint and draining. A build that would retag the runner image used by the active controller waits until after checkpoint and drain. Managed preflight still runs after drain, before activation. A failed pre-transaction build may leave candidate image or layer artifacts, but it does not change installed state or stop the active controller.
 
 The configuration repository credential, when required, must be read-only and host-side. Credentials are never accepted in command arguments.
 

docs/DESIRED-STATE.md

@@ -136,12 +136,13 @@ The installer:
 3. selects exactly one logical controller;
 4. renders `/etc/ci-fleet/ci-fleet.env` without secret values;
 5. fetches and verifies the pinned public engine commit;
-6. creates a root-only controller checkpoint;
-7. drains the current controller and waits for every managed runner to finish, including orphaned runners left after a stopped or crashed controller;
-8. runs managed preflight and builds the pinned runner and controller images;
-9. installs health, cleanup, and pinned-state drift unit definitions;
-10. starts the controller only when its desired state is active and verifies runtime health;
-11. atomically records redacted installation state, then enables the maintenance timers.
+6. validates the candidate Compose configuration and builds images before the transaction when no active installed controller uses the candidate runner tag;
+7. creates a root-only controller checkpoint;
+8. drains the current controller and waits for every managed runner to finish, including orphaned runners left after a stopped or crashed controller;
+9. runs managed preflight and, when the candidate would retag the active controller's runner image, builds the images after drain;
+10. installs health, cleanup, and pinned-state drift unit definitions;
+11. starts the controller only when its desired state is active and verifies runtime health;
+12. atomically records redacted installation state, then enables the maintenance timers.
 
 A successful second `--install` run reports `NO_CHANGE` and performs no unnecessary replacement. A successful engine upgrade advances both the runtime release and the maintenance installer-manager to the same pinned commit; rollback restores both.
 
@@ -240,7 +241,7 @@ Legacy project-specific hosts remain until CI, promotion, and deployment no long
 
 ## Failure and recovery behavior
 
-Before mutation, the installer records the prior rendered environment, installation metadata, runtime release, installer-manager release, and maintenance unit/timer state under `/var/lib/ci-fleet/checkpoints`. Each checkpoint is staged and atomically renamed with a completion marker; rollback ignores partial staging directories. Build and validation happen before the active release changes. A failed activation or health check drains the candidate, restores those artifacts, restarts the prior controller only when no managed runner is active, and verifies prior-release health before reporting rollback success. A host-local installer lock serializes every check and mutation. Runtime and installer-manager releases are staged on their respective target filesystems and renamed atomically so a failed copy cannot masquerade as an installed immutable release.
+Before mutation, the installer records the prior rendered environment, installation metadata, runtime release, installer-manager release, and maintenance unit/timer state under `/var/lib/ci-fleet/checkpoints`. Each checkpoint is staged and atomically renamed with a completion marker; rollback ignores partial staging directories. Compose validation happens before the checkpoint. A build using a distinct inert runner tag may also happen before the checkpoint; failure may leave candidate image or layer artifacts, but installed state and the active controller remain unchanged. A build that would retag the active controller's runner image happens only after checkpoint and drain. A failed activation or health check drains the candidate, restores those artifacts, restarts the prior controller only when no managed runner is active, and verifies prior-release health before reporting rollback success. A host-local installer lock serializes every check and mutation. Runtime and installer-manager releases are staged on their respective target filesystems and renamed atomically so a failed copy cannot masquerade as an installed immutable release.
 
 Installer checkpoints and machine backups serve different failure classes. A checkpoint rolls back a single failed reconciliation. Recoverability of the machine itself is governed by each host's declared failure boundary (see [Adding a host](ADDING-A-HOST.md)): a disposable controller needs no machine backup at all — recovery is rebuilding from reviewed Git-authored desired state — while a non-disposable host follows its own documented local backup policy.
 

host/systemd/ci-fleet-reconcile.service

@@ -9,6 +9,10 @@ Type=oneshot
 User=root
 WorkingDirectory=/opt/ci-fleet/manager/current
 ExecStart=/opt/ci-fleet/manager/current/scripts/remote-reconcile.sh
+# One hour for the installer lock, plus one hour for a cold build and transaction.
+TimeoutStartSec=2h
+# Allow checkpoint restoration after TimeoutStartSec sends SIGTERM.
+TimeoutStopSec=15min
 Restart=no
 # 0=noop, 3=drift/invalid — timer retries
 SuccessExitStatus=0 3