diff --git a/changelog.d/fixed-14929-upgrade-hold-clock.md b/changelog.d/fixed-14929-upgrade-hold-clock.md new file mode 100644 index 0000000000..85a214a6e2 --- /dev/null +++ b/changelog.d/fixed-14929-upgrade-hold-clock.md @@ -0,0 +1 @@ +- Stale upgrade recovery preserves the original elapsed time when a release channel advances, so promotions cannot restart the silent-drain hold or postpone stuck-upgrade alerts. diff --git a/src/Dockerfile b/src/Dockerfile index d5a4cd69a9..65b7110236 100644 --- a/src/Dockerfile +++ b/src/Dockerfile @@ -300,8 +300,10 @@ RUN npm install -g --ignore-scripts --no-fund --no-audit \ # SDK nests as its own dependency is never executed (its native binary would # reject the proxy CA). Tolerant of failure like the copilot layer: discovery # falls back to the raw HTTP probe when the SDK is absent. +# Bound this optional install so a stalled registry download cannot consume +# the overlayfs exec guard's entire job timeout before it runs its assertion. ARG COPILOT_SDK_VERSION=1.0.8 -RUN npm install -g --ignore-scripts --no-fund --no-audit \ +RUN timeout 120s npm install -g --ignore-scripts --no-fund --no-audit \ "@github/copilot-sdk@${COPILOT_SDK_VERSION}" && npm cache clean --force || \ echo "WARN: GitHub Copilot SDK install failed — skipping" diff --git a/src/pkg/hub/saas_upgrade.go b/src/pkg/hub/saas_upgrade.go index da1645c01f..3c98d6fa0f 100644 --- a/src/pkg/hub/saas_upgrade.go +++ b/src/pkg/hub/saas_upgrade.go @@ -1251,17 +1251,20 @@ func (s *HubServer) triggerAutoUpgrades() { s.mu.Lock() for i := range s.registry.Hives { if s.registry.Hives[i].ID == h.ID { - // Route through beginUpgrade so the start time - // SURVIVES a same-target re-arm (the crash-loop retry - // case): re-arming delivery for the same target must - // not reset the elapsed clock, or a thrashing upgrade - // never crosses staleUpgradeTimeout and the stuck- - // upgrade alert never fires. A genuinely advanced - // target (recoverTarget != old UpgradeTarget) is a new - // upgrade and DOES get a fresh clock. A lost/zero start - // is re-stamped either way, which self-heals the - // ibm-alchemy zero-timestamp wedge. + // Recovery continues the existing attempt, even when a + // channel promotion advances its target. Adopt the target + // before beginUpgrade so it preserves the original clock: + // promotions must not restart the silent-drain hold or + // postpone stuck-upgrade alerting. A lost/zero clock is + // still stamped by beginUpgrade. + s.registry.Hives[i].UpgradeTarget = recoverTarget s.beginUpgrade(i, recoverTarget) + if recoverTarget != upgradeTarget { + // The promoted target still causes a rollout. Record it + // separately so its restart is not counted as a crash, + // without resetting the attempt's hold clock. + s.alerts.noteUpgrade(h.ID, time.Now()) + } break } } diff --git a/src/pkg/hub/upgrade_recovery_test.go b/src/pkg/hub/upgrade_recovery_test.go index 8ad294d02f..8c51192057 100644 --- a/src/pkg/hub/upgrade_recovery_test.go +++ b/src/pkg/hub/upgrade_recovery_test.go @@ -406,3 +406,67 @@ func TestUpgradeStartedAtRoundTripsThroughRegistryAndRecovery(t *testing.T) { t.Fatalf("recovery must keep the rehydrated clock, got %v want %v", s.registry.Hives[0].UpgradeStartedAt, started) } } + +// Channel promotions during stale recovery must not extend the silent-drain +// window. The deadline belongs to the original attempt, not its latest target. +func TestTriggerAutoUpgradesPromotionPreservesHoldDeadline(t *testing.T) { + cleanup := helperSetupTempDirs(t) + defer cleanup() + resetSHACaches(t) + stubChannelImageVerificationUnavailable(t) + resetCommitOrderState(t) + stubCommitCompare(func(base, head string, logger *slog.Logger) (string, error) { + return "behind", nil + }) + seedLatestSHA(t, "v5", "bbbbbbb") + if err := saveSaaSHive(&SaaSHive{ + ID: "stable-hold", Owner: "alice", AutoUpgrade: true, + Status: "running", ClusterID: "remote", TrackedChannel: "stable", + }); err != nil { + t.Fatalf("save hive: %v", err) + } + started := time.Now().Add(-23 * time.Minute) + s := &HubServer{ + logger: slog.Default(), heartbeatUpgrade: make(map[string]string), + alerts: newAlertState(), + clusters: map[string]ClusterConfig{"remote": {ID: "remote"}}, + } + s.registry.Hives = []RegistryEntry{{ + ID: "stable-hold", GitBranch: "v5", GitHash: "1111111", + ImageRef: "ghcr.io/hivecommons/hive:stable", + Upgrading: true, UpgradeTarget: "2222222", UpgradeStartedAt: started, + LastHeartbeat: rfc3339At(time.Now()), + }} + s.alerts.noteUpgrade("stable-hold", started) + for promotion, target := range []string{"3333333", "4444444"} { + stubChannelRevisions(t, map[string]string{"stable": target}) + s.triggerAutoUpgrades() + entry := s.registry.Hives[0] + if !entry.Upgrading || entry.UpgradeTarget != target || s.heartbeatUpgrade[entry.ID] != target { + t.Fatalf("promotion must advance registry and delivery targets to %s: %+v", target, entry) + } + if !entry.UpgradeStartedAt.Equal(started) { + t.Fatalf("promotion reset the original start: got %v, want %v", entry.UpgradeStartedAt, started) + } + // Distinct starts after each promotion are planned rollouts, even + // though the preserved attempt clock is beyond the upgrade grace. + restarted := time.Now().Add(time.Duration(promotion+1) * time.Minute) + if crashes := s.alerts.observeStart(entry.ID, rfc3339At(restarted), restarted); crashes != 0 { + t.Fatalf("promotion restart must not count as a crash: got %d", crashes) + } + if ev := evaluateOrphanedUpgrade(&entry, started.Add(silentDrainClearAfter-time.Second), ""); ev.orphaned { + t.Fatalf("original silent-drain window must still hold: %+v", ev) + } + entry.LastHeartbeat = rfc3339At(started.Add(silentDrainClearAfter)) + if ev := evaluateOrphanedUpgrade(&entry, started.Add(silentDrainClearAfter+time.Second), ""); !ev.orphaned || ev.converged { + t.Fatalf("promotion must not extend the original silent-drain deadline: %+v", ev) + } + } + // Recovery must still self-heal a lost clock when advancing the target. + s.registry.Hives[0].UpgradeStartedAt = time.Time{} + stubChannelRevisions(t, map[string]string{"stable": "5555555"}) + s.triggerAutoUpgrades() + if entry := s.registry.Hives[0]; entry.UpgradeStartedAt.IsZero() || entry.UpgradeTarget != "5555555" { + t.Fatalf("recovery must stamp a lost clock and advance the target: %+v", entry) + } +}