From 23f57c197f794472fea3ec2c72187c5cc75daec4 Mon Sep 17 00:00:00 2001 From: thatguywob Date: Wed, 28 Oct 2026 13:25:23 +0000 Subject: [PATCH 1/3] =?UTF-8?q?=F0=9F=90=9B=20fix:=20preserve=20upgrade=20?= =?UTF-8?q?hold=20clock=20during=20target=20recovery?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Keep stale recovery on the original attempt clock when a channel promotion advances its target. Cover repeated stable promotions, the silent-drain deadline, and lost-clock recovery. Hive-Run: hivecommons/hive#14929 Hive-Plan: preserve-stale-recovery-clock Hive-Spec: hivecommons/hive#14929#target-advancement-hold Signed-off-by: thatguywob --- changelog.d/fixed-14929-upgrade-hold-clock.md | 1 + src/pkg/hub/saas_upgrade.go | 17 +++--- src/pkg/hub/upgrade_recovery_test.go | 56 +++++++++++++++++++ 3 files changed, 64 insertions(+), 10 deletions(-) create mode 100644 changelog.d/fixed-14929-upgrade-hold-clock.md diff --git a/changelog.d/fixed-14929-upgrade-hold-clock.md b/changelog.d/fixed-14929-upgrade-hold-clock.md new file mode 100644 index 0000000000..85a214a6e2 --- /dev/null +++ b/changelog.d/fixed-14929-upgrade-hold-clock.md @@ -0,0 +1 @@ +- Stale upgrade recovery preserves the original elapsed time when a release channel advances, so promotions cannot restart the silent-drain hold or postpone stuck-upgrade alerts. diff --git a/src/pkg/hub/saas_upgrade.go b/src/pkg/hub/saas_upgrade.go index da1645c01f..114397a310 100644 --- a/src/pkg/hub/saas_upgrade.go +++ b/src/pkg/hub/saas_upgrade.go @@ -1251,16 +1251,13 @@ func (s *HubServer) triggerAutoUpgrades() { s.mu.Lock() for i := range s.registry.Hives { if s.registry.Hives[i].ID == h.ID { - // Route through beginUpgrade so the start time - // SURVIVES a same-target re-arm (the crash-loop retry - // case): re-arming delivery for the same target must - // not reset the elapsed clock, or a thrashing upgrade - // never crosses staleUpgradeTimeout and the stuck- - // upgrade alert never fires. A genuinely advanced - // target (recoverTarget != old UpgradeTarget) is a new - // upgrade and DOES get a fresh clock. A lost/zero start - // is re-stamped either way, which self-heals the - // ibm-alchemy zero-timestamp wedge. + // Recovery continues the existing attempt, even when a + // channel promotion advances its target. Adopt the target + // before beginUpgrade so it preserves the original clock: + // promotions must not restart the silent-drain hold or + // postpone stuck-upgrade alerting. A lost/zero clock is + // still stamped by beginUpgrade. + s.registry.Hives[i].UpgradeTarget = recoverTarget s.beginUpgrade(i, recoverTarget) break } diff --git a/src/pkg/hub/upgrade_recovery_test.go b/src/pkg/hub/upgrade_recovery_test.go index 8ad294d02f..418a47c9be 100644 --- a/src/pkg/hub/upgrade_recovery_test.go +++ b/src/pkg/hub/upgrade_recovery_test.go @@ -406,3 +406,59 @@ func TestUpgradeStartedAtRoundTripsThroughRegistryAndRecovery(t *testing.T) { t.Fatalf("recovery must keep the rehydrated clock, got %v want %v", s.registry.Hives[0].UpgradeStartedAt, started) } } + +// Channel promotions during stale recovery must not extend the silent-drain +// window. The deadline belongs to the original attempt, not its latest target. +func TestTriggerAutoUpgradesPromotionPreservesHoldDeadline(t *testing.T) { + cleanup := helperSetupTempDirs(t) + defer cleanup() + resetSHACaches(t) + stubChannelImageVerificationUnavailable(t) + resetCommitOrderState(t) + stubCommitCompare(func(base, head string, logger *slog.Logger) (string, error) { + return "behind", nil + }) + seedLatestSHA(t, "v5", "bbbbbbb") + if err := saveSaaSHive(&SaaSHive{ + ID: "stable-hold", Owner: "alice", AutoUpgrade: true, + Status: "running", ClusterID: "remote", TrackedChannel: "stable", + }); err != nil { + t.Fatalf("save hive: %v", err) + } + started := time.Now().Add(-23 * time.Minute) + s := &HubServer{ + logger: slog.Default(), heartbeatUpgrade: make(map[string]string), + clusters: map[string]ClusterConfig{"remote": {ID: "remote"}}, + } + s.registry.Hives = []RegistryEntry{{ + ID: "stable-hold", GitBranch: "v5", GitHash: "1111111", + ImageRef: "ghcr.io/hivecommons/hive:stable", + Upgrading: true, UpgradeTarget: "2222222", UpgradeStartedAt: started, + LastHeartbeat: rfc3339At(time.Now()), + }} + for _, target := range []string{"3333333", "4444444"} { + stubChannelRevisions(t, map[string]string{"stable": target}) + s.triggerAutoUpgrades() + entry := s.registry.Hives[0] + if !entry.Upgrading || entry.UpgradeTarget != target || s.heartbeatUpgrade[entry.ID] != target { + t.Fatalf("promotion must advance registry and delivery targets to %s: %+v", target, entry) + } + if !entry.UpgradeStartedAt.Equal(started) { + t.Fatalf("promotion reset the original start: got %v, want %v", entry.UpgradeStartedAt, started) + } + if ev := evaluateOrphanedUpgrade(&entry, started.Add(silentDrainClearAfter-time.Second), ""); ev.orphaned { + t.Fatalf("original silent-drain window must still hold: %+v", ev) + } + entry.LastHeartbeat = rfc3339At(started.Add(silentDrainClearAfter)) + if ev := evaluateOrphanedUpgrade(&entry, started.Add(silentDrainClearAfter+time.Second), ""); !ev.orphaned || ev.converged { + t.Fatalf("promotion must not extend the original silent-drain deadline: %+v", ev) + } + } + // Recovery must still self-heal a lost clock when advancing the target. + s.registry.Hives[0].UpgradeStartedAt = time.Time{} + stubChannelRevisions(t, map[string]string{"stable": "5555555"}) + s.triggerAutoUpgrades() + if entry := s.registry.Hives[0]; entry.UpgradeStartedAt.IsZero() || entry.UpgradeTarget != "5555555" { + t.Fatalf("recovery must stamp a lost clock and advance the target: %+v", entry) + } +} From 2c7340f089eb78b6c58a4bb30c1e94d7ef5c6b13 Mon Sep 17 00:00:00 2001 From: thatguywob Date: Wed, 28 Oct 2026 13:43:46 +0000 Subject: [PATCH 2/3] =?UTF-8?q?=F0=9F=90=9B=20fix:=20retain=20rollout=20ma?= =?UTF-8?q?rks=20during=20upgrade=20target=20recovery?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: thatguywob --- src/pkg/hub/saas_upgrade.go | 6 ++++++ src/pkg/hub/upgrade_recovery_test.go | 10 +++++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/pkg/hub/saas_upgrade.go b/src/pkg/hub/saas_upgrade.go index 114397a310..3c98d6fa0f 100644 --- a/src/pkg/hub/saas_upgrade.go +++ b/src/pkg/hub/saas_upgrade.go @@ -1259,6 +1259,12 @@ func (s *HubServer) triggerAutoUpgrades() { // still stamped by beginUpgrade. s.registry.Hives[i].UpgradeTarget = recoverTarget s.beginUpgrade(i, recoverTarget) + if recoverTarget != upgradeTarget { + // The promoted target still causes a rollout. Record it + // separately so its restart is not counted as a crash, + // without resetting the attempt's hold clock. + s.alerts.noteUpgrade(h.ID, time.Now()) + } break } } diff --git a/src/pkg/hub/upgrade_recovery_test.go b/src/pkg/hub/upgrade_recovery_test.go index 418a47c9be..8c51192057 100644 --- a/src/pkg/hub/upgrade_recovery_test.go +++ b/src/pkg/hub/upgrade_recovery_test.go @@ -428,6 +428,7 @@ func TestTriggerAutoUpgradesPromotionPreservesHoldDeadline(t *testing.T) { started := time.Now().Add(-23 * time.Minute) s := &HubServer{ logger: slog.Default(), heartbeatUpgrade: make(map[string]string), + alerts: newAlertState(), clusters: map[string]ClusterConfig{"remote": {ID: "remote"}}, } s.registry.Hives = []RegistryEntry{{ @@ -436,7 +437,8 @@ func TestTriggerAutoUpgradesPromotionPreservesHoldDeadline(t *testing.T) { Upgrading: true, UpgradeTarget: "2222222", UpgradeStartedAt: started, LastHeartbeat: rfc3339At(time.Now()), }} - for _, target := range []string{"3333333", "4444444"} { + s.alerts.noteUpgrade("stable-hold", started) + for promotion, target := range []string{"3333333", "4444444"} { stubChannelRevisions(t, map[string]string{"stable": target}) s.triggerAutoUpgrades() entry := s.registry.Hives[0] @@ -446,6 +448,12 @@ func TestTriggerAutoUpgradesPromotionPreservesHoldDeadline(t *testing.T) { if !entry.UpgradeStartedAt.Equal(started) { t.Fatalf("promotion reset the original start: got %v, want %v", entry.UpgradeStartedAt, started) } + // Distinct starts after each promotion are planned rollouts, even + // though the preserved attempt clock is beyond the upgrade grace. + restarted := time.Now().Add(time.Duration(promotion+1) * time.Minute) + if crashes := s.alerts.observeStart(entry.ID, rfc3339At(restarted), restarted); crashes != 0 { + t.Fatalf("promotion restart must not count as a crash: got %d", crashes) + } if ev := evaluateOrphanedUpgrade(&entry, started.Add(silentDrainClearAfter-time.Second), ""); ev.orphaned { t.Fatalf("original silent-drain window must still hold: %+v", ev) } From cd3a7b42d9fe4c0850b57039fef511c80b7629c6 Mon Sep 17 00:00:00 2001 From: thatguywob Date: Wed, 28 Oct 2026 14:05:03 +0000 Subject: [PATCH 3/3] =?UTF-8?q?=F0=9F=90=9B=20fix:=20bound=20optional=20Co?= =?UTF-8?q?pilot=20SDK=20image=20install?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: thatguywob --- src/Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Dockerfile b/src/Dockerfile index d5a4cd69a9..65b7110236 100644 --- a/src/Dockerfile +++ b/src/Dockerfile @@ -300,8 +300,10 @@ RUN npm install -g --ignore-scripts --no-fund --no-audit \ # SDK nests as its own dependency is never executed (its native binary would # reject the proxy CA). Tolerant of failure like the copilot layer: discovery # falls back to the raw HTTP probe when the SDK is absent. +# Bound this optional install so a stalled registry download cannot consume +# the overlayfs exec guard's entire job timeout before it runs its assertion. ARG COPILOT_SDK_VERSION=1.0.8 -RUN npm install -g --ignore-scripts --no-fund --no-audit \ +RUN timeout 120s npm install -g --ignore-scripts --no-fund --no-audit \ "@github/copilot-sdk@${COPILOT_SDK_VERSION}" && npm cache clean --force || \ echo "WARN: GitHub Copilot SDK install failed — skipping"