Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/fixed-14929-upgrade-hold-clock.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- Stale upgrade recovery preserves the original elapsed time when a release channel advances, so promotions cannot restart the silent-drain hold or postpone stuck-upgrade alerts.
4 changes: 3 additions & 1 deletion src/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -300,8 +300,10 @@ RUN npm install -g --ignore-scripts --no-fund --no-audit \
# SDK nests as its own dependency is never executed (its native binary would
# reject the proxy CA). Tolerant of failure like the copilot layer: discovery
# falls back to the raw HTTP probe when the SDK is absent.
# Bound this optional install so a stalled registry download cannot consume
# the overlayfs exec guard's entire job timeout before it runs its assertion.
ARG COPILOT_SDK_VERSION=1.0.8
RUN npm install -g --ignore-scripts --no-fund --no-audit \
RUN timeout 120s npm install -g --ignore-scripts --no-fund --no-audit \
"@github/copilot-sdk@${COPILOT_SDK_VERSION}" && npm cache clean --force || \
echo "WARN: GitHub Copilot SDK install failed — skipping"

Expand Down
23 changes: 13 additions & 10 deletions src/pkg/hub/saas_upgrade.go
Original file line number Diff line number Diff line change
Expand Up @@ -1251,17 +1251,20 @@ func (s *HubServer) triggerAutoUpgrades() {
s.mu.Lock()
for i := range s.registry.Hives {
if s.registry.Hives[i].ID == h.ID {
// Route through beginUpgrade so the start time
// SURVIVES a same-target re-arm (the crash-loop retry
// case): re-arming delivery for the same target must
// not reset the elapsed clock, or a thrashing upgrade
// never crosses staleUpgradeTimeout and the stuck-
// upgrade alert never fires. A genuinely advanced
// target (recoverTarget != old UpgradeTarget) is a new
// upgrade and DOES get a fresh clock. A lost/zero start
// is re-stamped either way, which self-heals the
// ibm-alchemy zero-timestamp wedge.
// Recovery continues the existing attempt, even when a
// channel promotion advances its target. Adopt the target
// before beginUpgrade so it preserves the original clock:
// promotions must not restart the silent-drain hold or
// postpone stuck-upgrade alerting. A lost/zero clock is
// still stamped by beginUpgrade.
s.registry.Hives[i].UpgradeTarget = recoverTarget
s.beginUpgrade(i, recoverTarget)
if recoverTarget != upgradeTarget {
// The promoted target still causes a rollout. Record it
// separately so its restart is not counted as a crash,
// without resetting the attempt's hold clock.
s.alerts.noteUpgrade(h.ID, time.Now())
}
break
}
}
Expand Down
64 changes: 64 additions & 0 deletions src/pkg/hub/upgrade_recovery_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -406,3 +406,67 @@ func TestUpgradeStartedAtRoundTripsThroughRegistryAndRecovery(t *testing.T) {
t.Fatalf("recovery must keep the rehydrated clock, got %v want %v", s.registry.Hives[0].UpgradeStartedAt, started)
}
}

// Channel promotions during stale recovery must not extend the silent-drain
// window. The deadline belongs to the original attempt, not its latest target.
func TestTriggerAutoUpgradesPromotionPreservesHoldDeadline(t *testing.T) {
cleanup := helperSetupTempDirs(t)
defer cleanup()
resetSHACaches(t)
stubChannelImageVerificationUnavailable(t)
resetCommitOrderState(t)
stubCommitCompare(func(base, head string, logger *slog.Logger) (string, error) {
return "behind", nil
})
seedLatestSHA(t, "v5", "bbbbbbb")
if err := saveSaaSHive(&SaaSHive{
ID: "stable-hold", Owner: "alice", AutoUpgrade: true,
Status: "running", ClusterID: "remote", TrackedChannel: "stable",
}); err != nil {
t.Fatalf("save hive: %v", err)
}
started := time.Now().Add(-23 * time.Minute)
s := &HubServer{
logger: slog.Default(), heartbeatUpgrade: make(map[string]string),
alerts: newAlertState(),
clusters: map[string]ClusterConfig{"remote": {ID: "remote"}},
}
s.registry.Hives = []RegistryEntry{{
ID: "stable-hold", GitBranch: "v5", GitHash: "1111111",
ImageRef: "ghcr.io/hivecommons/hive:stable",
Upgrading: true, UpgradeTarget: "2222222", UpgradeStartedAt: started,
LastHeartbeat: rfc3339At(time.Now()),
}}
s.alerts.noteUpgrade("stable-hold", started)
for promotion, target := range []string{"3333333", "4444444"} {
stubChannelRevisions(t, map[string]string{"stable": target})
s.triggerAutoUpgrades()
entry := s.registry.Hives[0]
if !entry.Upgrading || entry.UpgradeTarget != target || s.heartbeatUpgrade[entry.ID] != target {
t.Fatalf("promotion must advance registry and delivery targets to %s: %+v", target, entry)
}
if !entry.UpgradeStartedAt.Equal(started) {
t.Fatalf("promotion reset the original start: got %v, want %v", entry.UpgradeStartedAt, started)
}
// Distinct starts after each promotion are planned rollouts, even
// though the preserved attempt clock is beyond the upgrade grace.
restarted := time.Now().Add(time.Duration(promotion+1) * time.Minute)
if crashes := s.alerts.observeStart(entry.ID, rfc3339At(restarted), restarted); crashes != 0 {
t.Fatalf("promotion restart must not count as a crash: got %d", crashes)
}
if ev := evaluateOrphanedUpgrade(&entry, started.Add(silentDrainClearAfter-time.Second), ""); ev.orphaned {
t.Fatalf("original silent-drain window must still hold: %+v", ev)
}
entry.LastHeartbeat = rfc3339At(started.Add(silentDrainClearAfter))
if ev := evaluateOrphanedUpgrade(&entry, started.Add(silentDrainClearAfter+time.Second), ""); !ev.orphaned || ev.converged {
t.Fatalf("promotion must not extend the original silent-drain deadline: %+v", ev)
}
}
// Recovery must still self-heal a lost clock when advancing the target.
s.registry.Hives[0].UpgradeStartedAt = time.Time{}
stubChannelRevisions(t, map[string]string{"stable": "5555555"})
s.triggerAutoUpgrades()
if entry := s.registry.Hives[0]; entry.UpgradeStartedAt.IsZero() || entry.UpgradeTarget != "5555555" {
t.Fatalf("recovery must stamp a lost clock and advance the target: %+v", entry)
}
}
Loading