diff --git a/changelog.d/fixed-upgrade-delay-warnings.md b/changelog.d/fixed-upgrade-delay-warnings.md
new file mode 100644
index 0000000000..c0e0b57cc0
--- /dev/null
+++ b/changelog.d/fixed-upgrade-delay-warnings.md
@@ -0,0 +1 @@
+- Show temporary upgrade holds as Delayed instead of missed windows, and clear resolved delay warnings after a successful upgrade on both dashboards, including older history entries.
diff --git a/src/pkg/dashboard/api_version.go b/src/pkg/dashboard/api_version.go
index bffdd673db..881d5d667d 100644
--- a/src/pkg/dashboard/api_version.go
+++ b/src/pkg/dashboard/api_version.go
@@ -95,7 +95,13 @@ func (s *Server) handleVersion(w http.ResponseWriter, r *http.Request) {
resp["upgradeProgress"] = progress
}
if policy != nil && len(policy.UpgradeHistory) > 0 {
- resp["upgradeHistory"] = policy.UpgradeHistory
+ var upgradedAt time.Time
+ if upgradeOutcomeRec != nil {
+ upgradedAt = upgradeOutcomeRec.CompletedAt
+ }
+ if history := spoke.UpgradeHistoryForDisplay(policy.UpgradeHistory, upgradedAt); len(history) > 0 {
+ resp["upgradeHistory"] = history
+ }
}
if cacheAge > dashboardVersionTipCacheTTL || cached == "" {
diff --git a/src/pkg/dashboard/static-src/css/topbar.css b/src/pkg/dashboard/static-src/css/topbar.css
index ef84522944..93334c2f74 100644
--- a/src/pkg/dashboard/static-src/css/topbar.css
+++ b/src/pkg/dashboard/static-src/css/topbar.css
@@ -38,7 +38,7 @@
.oc-version-manual-pill { white-space: nowrap; padding: 1px 7px; border-radius: var(--r-pill); border: 1px solid var(--border); color: var(--muted); font-size: var(--fs-xs); }
.oc-version-upgrade-note { color: var(--muted); font-size: var(--fs-xs); overflow-wrap: anywhere; }
.oc-version-state-pill { white-space: nowrap; padding: var(--sp-1) var(--sp-3); border-radius: var(--r-pill); border: 1px solid var(--border); color: var(--muted); font-size: var(--fs-xs); font-variant-numeric: tabular-nums; }
- .oc-version-state-pill.is-draining { color: var(--amber); border-color: color-mix(in srgb, var(--amber) 40%, transparent); }
+ .oc-version-state-pill.is-draining, .oc-version-state-pill.is-delayed { color: var(--amber); border-color: color-mix(in srgb, var(--amber) 40%, transparent); }
.oc-version-state-pill.is-missed { color: var(--red, var(--danger)); border-color: color-mix(in srgb, var(--red, var(--danger)) 40%, transparent); }
.oc-version-state-pill.is-done { color: var(--green, var(--success)); }
.oc-version-history { list-style: none; margin: 0; padding: 0; font-size: var(--fs-xs); color: var(--muted); display: flex; flex-direction: column; gap: 2px; }
diff --git a/src/pkg/dashboard/static-src/js/version.js b/src/pkg/dashboard/static-src/js/version.js
index 42df507c93..a294b2db8b 100644
--- a/src/pkg/dashboard/static-src/js/version.js
+++ b/src/pkg/dashboard/static-src/js/version.js
@@ -228,7 +228,7 @@
state as a pill — with the drain countdown and the agents holding the
upgrade — and lists the history so a missed window is never silent. */
function versionStateLabels() {
- return { scheduled: 'Scheduled', draining: 'Draining', pulling: 'Pulling', rolling: 'Rolling', verifying: 'Verifying', done: 'Upgraded', missed: 'Missed window' };
+ return { scheduled: 'Scheduled', delayed: 'Delayed', draining: 'Draining', pulling: 'Pulling', rolling: 'Rolling', verifying: 'Verifying', done: 'Upgraded', missed: 'Missed window' };
}
function versionStateLabel(state) {
@@ -261,7 +261,7 @@
const agents = versionAgentsInTurnText(st);
const left = typeof st.drainRemainingS === 'number' && st.drainRemainingS > 0 ? versionFormatDuration(st.drainRemainingS) + ' left' : 'deadline reached';
text += (agents ? ' · ' + agents : '') + ' · ' + left;
- } else if (st.state === 'missed' && st.reason) {
+ } else if ((st.state === 'missed' || st.state === 'delayed') && st.reason) {
text += ' · ' + st.reason;
} else if (st.state === 'scheduled' && st.nextWindowAt) {
const at = new Date(st.nextWindowAt);
@@ -1427,4 +1427,3 @@
}, VERSION_CHECK_INTERVAL_MS);
}
// startVersionCheck() — deferred to _deferredInit
-
diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html
index 06da9262d6..934d50c71a 100644
--- a/src/pkg/dashboard/static/index.html
+++ b/src/pkg/dashboard/static/index.html
@@ -404,7 +404,7 @@
.oc-version-manual-pill { white-space: nowrap; padding: 1px 7px; border-radius: var(--r-pill); border: 1px solid var(--border); color: var(--muted); font-size: var(--fs-xs); }
.oc-version-upgrade-note { color: var(--muted); font-size: var(--fs-xs); overflow-wrap: anywhere; }
.oc-version-state-pill { white-space: nowrap; padding: var(--sp-1) var(--sp-3); border-radius: var(--r-pill); border: 1px solid var(--border); color: var(--muted); font-size: var(--fs-xs); font-variant-numeric: tabular-nums; }
- .oc-version-state-pill.is-draining { color: var(--amber); border-color: color-mix(in srgb, var(--amber) 40%, transparent); }
+ .oc-version-state-pill.is-draining, .oc-version-state-pill.is-delayed { color: var(--amber); border-color: color-mix(in srgb, var(--amber) 40%, transparent); }
.oc-version-state-pill.is-missed { color: var(--red, var(--danger)); border-color: color-mix(in srgb, var(--red, var(--danger)) 40%, transparent); }
.oc-version-state-pill.is-done { color: var(--green, var(--success)); }
.oc-version-history { list-style: none; margin: 0; padding: 0; font-size: var(--fs-xs); color: var(--muted); display: flex; flex-direction: column; gap: 2px; }
@@ -45721,7 +45721,7 @@
Configure LiteLLM
state as a pill — with the drain countdown and the agents holding the
upgrade — and lists the history so a missed window is never silent. */
function versionStateLabels() {
- return { scheduled: 'Scheduled', draining: 'Draining', pulling: 'Pulling', rolling: 'Rolling', verifying: 'Verifying', done: 'Upgraded', missed: 'Missed window' };
+ return { scheduled: 'Scheduled', delayed: 'Delayed', draining: 'Draining', pulling: 'Pulling', rolling: 'Rolling', verifying: 'Verifying', done: 'Upgraded', missed: 'Missed window' };
}
function versionStateLabel(state) {
@@ -45754,7 +45754,7 @@ Configure LiteLLM
const agents = versionAgentsInTurnText(st);
const left = typeof st.drainRemainingS === 'number' && st.drainRemainingS > 0 ? versionFormatDuration(st.drainRemainingS) + ' left' : 'deadline reached';
text += (agents ? ' · ' + agents : '') + ' · ' + left;
- } else if (st.state === 'missed' && st.reason) {
+ } else if ((st.state === 'missed' || st.state === 'delayed') && st.reason) {
text += ' · ' + st.reason;
} else if (st.state === 'scheduled' && st.nextWindowAt) {
const at = new Date(st.nextWindowAt);
@@ -46920,7 +46920,6 @@ Configure LiteLLM
}, VERSION_CHECK_INTERVAL_MS);
}
// startVersionCheck() — deferred to _deferredInit
-
var _eventSource = null;
var _eventSourceReconnectTimer = null;
var _sseLastFrameAtMs = 0;
diff --git a/src/pkg/dashboard/upgrade_progress.go b/src/pkg/dashboard/upgrade_progress.go
index ba51d32f6b..dfbedf0d7c 100644
--- a/src/pkg/dashboard/upgrade_progress.go
+++ b/src/pkg/dashboard/upgrade_progress.go
@@ -63,6 +63,11 @@ func buildUpgradeProgress(in upgradeProgressInput) *spoke.UpgradeProgress {
nextWindow = in.Policy.NextWindowAt
history = in.Policy.UpgradeHistory
}
+ var upgradedAt time.Time
+ if in.Outcome != nil {
+ upgradedAt = in.Outcome.CompletedAt
+ }
+ history = spoke.UpgradeHistoryForDisplay(history, upgradedAt)
if d := in.Drain; d != nil {
turns := d.Turns
@@ -114,7 +119,9 @@ func buildUpgradeProgress(in upgradeProgressInput) *spoke.UpgradeProgress {
return p
}
- if o := in.Outcome; o != nil && !o.CompletedAt.IsZero() && !in.StartedAt.IsZero() && !o.CompletedAt.Before(in.StartedAt.Add(-time.Minute)) {
+ last, hasLast := latestClosedAttempt(history)
+ if o := in.Outcome; o != nil && !o.CompletedAt.IsZero() && !in.StartedAt.IsZero() &&
+ !o.CompletedAt.Before(in.StartedAt.Add(-time.Minute)) && (!hasLast || !last.EndedAt.After(o.CompletedAt)) {
return &spoke.UpgradeProgress{
State: spoke.UpgradeStateDone,
Target: shortSHADashboard(o.TargetSHA),
@@ -126,7 +133,7 @@ func buildUpgradeProgress(in upgradeProgressInput) *spoke.UpgradeProgress {
}
}
- if last, ok := latestClosedAttempt(history); ok && now.Sub(last.EndedAt) <= upgradeHistoryRecentFor {
+ if hasLast && now.Sub(last.EndedAt) <= upgradeHistoryRecentFor {
p := &spoke.UpgradeProgress{
State: last.Outcome,
Target: last.Target,
@@ -140,6 +147,8 @@ func buildUpgradeProgress(in upgradeProgressInput) *spoke.UpgradeProgress {
switch last.Outcome {
case spoke.UpgradeOutcomeDone:
p.Detail = "upgraded to " + orDashDashboard(last.Target) + " " + ago
+ case spoke.UpgradeOutcomeDelayed:
+ p.Detail = "delayed: " + last.Reason
default:
if last.Trigger == spoke.UpgradeTriggerWindow {
p.Detail = "missed window " + ago + ": " + last.Reason
diff --git a/src/pkg/dashboard/upgrade_progress_test.go b/src/pkg/dashboard/upgrade_progress_test.go
index 8fd681f2ea..16f4e43c6c 100644
--- a/src/pkg/dashboard/upgrade_progress_test.go
+++ b/src/pkg/dashboard/upgrade_progress_test.go
@@ -54,13 +54,43 @@ func TestBuildUpgradeProgressStates(t *testing.T) {
state: spoke.UpgradeStateDone, detail: "upgraded to abc1234 9m ago",
},
{
- name: "old outcome ignored, hub history missed window shown",
+ name: "old outcome ignored, legacy hub window shown as delayed",
in: upgradeProgressInput{StartedAt: now.Add(-10 * time.Minute),
Outcome: &upgradeOutcome{TargetSHA: "abc1234", CompletedAt: now.Add(-48 * time.Hour)},
Policy: &spoke.HeartbeatUpgradePolicy{HubManaged: true, NextWindowAt: next, UpgradeHistory: []spoke.UpgradeAttempt{
{Trigger: spoke.UpgradeTriggerWindow, Outcome: spoke.UpgradeOutcomeMissed, Reason: "spoke still draining",
StartedAt: now.Add(-3 * time.Hour), EndedAt: now.Add(-3 * time.Hour)}}}},
- state: spoke.UpgradeStateMissed, detail: "missed window 3h0m ago: spoke still draining",
+ state: spoke.UpgradeStateDelayed, detail: "delayed: spoke still draining",
+ },
+ {
+ name: "landed outcome clears stale hub delay before next heartbeat",
+ in: upgradeProgressInput{StartedAt: now.Add(-10 * time.Minute),
+ Outcome: &upgradeOutcome{TargetSHA: "abc1234", CompletedAt: now.Add(-9 * time.Minute)},
+ Policy: &spoke.HeartbeatUpgradePolicy{HubManaged: true, UpgradeHistory: []spoke.UpgradeAttempt{
+ {Trigger: spoke.UpgradeTriggerWindow, Outcome: spoke.UpgradeOutcomeMissed, Reason: "merge debounce",
+ StartedAt: now.Add(-15 * time.Minute), EndedAt: now.Add(-12 * time.Minute)},
+ }}},
+ state: spoke.UpgradeStateDone, detail: "upgraded to abc1234",
+ },
+ {
+ name: "new delay is not hidden by this boot's earlier success",
+ in: upgradeProgressInput{StartedAt: now.Add(-10 * time.Minute),
+ Outcome: &upgradeOutcome{TargetSHA: "abc1234", CompletedAt: now.Add(-9 * time.Minute)},
+ Policy: &spoke.HeartbeatUpgradePolicy{HubManaged: true, UpgradeHistory: []spoke.UpgradeAttempt{
+ {Trigger: spoke.UpgradeTriggerWindow, Outcome: spoke.UpgradeOutcomeDelayed, Reason: "merge debounce",
+ StartedAt: now.Add(-2 * time.Minute), EndedAt: now.Add(-time.Minute)},
+ }}},
+ state: spoke.UpgradeStateDelayed, detail: "merge debounce",
+ },
+ {
+ name: "genuine hub failure remains missed",
+ in: upgradeProgressInput{
+ StartedAt: now.Add(-10 * time.Minute),
+ Outcome: &upgradeOutcome{TargetSHA: "abc1234", CompletedAt: now.Add(-9 * time.Minute)},
+ Policy: &spoke.HeartbeatUpgradePolicy{HubManaged: true, UpgradeHistory: []spoke.UpgradeAttempt{
+ {Trigger: spoke.UpgradeTriggerScheduled, Outcome: spoke.UpgradeOutcomeMissed, Reason: "ErrImagePull", EndedAt: now.Add(-time.Minute)},
+ }}},
+ state: spoke.UpgradeStateMissed, detail: "ErrImagePull",
},
{
name: "scheduled with window",
diff --git a/src/pkg/dashboard/upgrade_progress_ui_test.go b/src/pkg/dashboard/upgrade_progress_ui_test.go
index 40250cc65a..a40e29e60c 100644
--- a/src/pkg/dashboard/upgrade_progress_ui_test.go
+++ b/src/pkg/dashboard/upgrade_progress_ui_test.go
@@ -4,6 +4,7 @@ import (
"encoding/json"
"net/http"
"net/http/httptest"
+ "os"
"path/filepath"
"strings"
"testing"
@@ -55,10 +56,32 @@ func TestVersionCarriesUpgradeProgressAndHistory(t *testing.T) {
if len(p.AgentsInTurn) != 1 || p.AgentsInTurn[0].Agent != "reviewer" || p.AgentsInTurn[0].ElapsedS != 2280 {
t.Fatalf("agentsInTurn = %+v", p.AgentsInTurn)
}
- if len(body.UpgradeHistory) != 1 || body.UpgradeHistory[0].Outcome != spoke.UpgradeOutcomeMissed || body.UpgradeHistory[0].Reason != "drain lapsed" {
+ if len(body.UpgradeHistory) != 1 || body.UpgradeHistory[0].Outcome != spoke.UpgradeOutcomeDelayed || body.UpgradeHistory[0].Reason != "drain lapsed" {
t.Fatalf("upgradeHistory = %+v", body.UpgradeHistory)
}
+ // The spoke knows the rollout landed before the hub's next heartbeat.
+ s.SetUpgradeDrainSource(nil)
+ outcome, err := json.Marshal(upgradeOutcome{TargetSHA: "abc1234", CompletedAt: time.Now()})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(upgradeOutcomePath, outcome, 0o600); err != nil {
+ t.Fatal(err)
+ }
+ rec = httptest.NewRecorder()
+ s.handleVersion(rec, httptest.NewRequest(http.MethodGet, "/api/version", nil))
+ body.UpgradeHistory = nil
+ if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil {
+ t.Fatal(err)
+ }
+ if body.UpgradeProgress == nil || body.UpgradeProgress.State != spoke.UpgradeStateDone || len(body.UpgradeHistory) != 0 {
+ t.Fatalf("successful upgrade still shows a delay: %+v", body)
+ }
+ if err := os.Remove(upgradeOutcomePath); err != nil {
+ t.Fatal(err)
+ }
+
// No policy, no drain, nothing on disk: the keys are absent rather than
// null so older SPA builds see exactly the payload they did before.
s2 := &Server{status: &StatusPayload{}, startedAt: time.Now()}
@@ -88,6 +111,7 @@ func TestIndexRendersUpgradeProgressPillAndHistory(t *testing.T) {
"st.agentsInTurn",
".oc-version-state-pill {",
".oc-version-state-pill.is-draining",
+ ".oc-version-state-pill.is-delayed",
".oc-version-state-pill.is-missed",
".oc-version-history {",
"row('Progress', statePill",
@@ -98,7 +122,7 @@ func TestIndexRendersUpgradeProgressPillAndHistory(t *testing.T) {
}
}
labels := jsFunc(t, html, "versionStateLabels")
- for _, state := range []string{"scheduled", "draining", "pulling", "rolling", "verifying", "done", "missed"} {
+ for _, state := range []string{"scheduled", "delayed", "draining", "pulling", "rolling", "verifying", "done", "missed"} {
if !strings.Contains(labels, state+":") {
t.Errorf("versionStateLabels lacks %q", state)
}
diff --git a/src/pkg/hub/assets/dashboard.html b/src/pkg/hub/assets/dashboard.html
index 7ba7a93a6c..b1dff8d84a 100644
--- a/src/pkg/hub/assets/dashboard.html
+++ b/src/pkg/hub/assets/dashboard.html
@@ -693,7 +693,7 @@
/* Scheduled-upgrade progress pill, history badge and interrupt action
on the hive card's Version cell (#14950). */
.upgrade-progress-pill { display: inline-block; padding: var(--sp-1) var(--sp-3); border-radius: var(--r-pill); font-size: var(--fs-2xs); background: var(--surface); color: var(--muted); border: 1px solid var(--border); white-space: nowrap; font-variant-numeric: tabular-nums; }
- .upgrade-progress-pill.is-draining { color: var(--amber); }
+ .upgrade-progress-pill.is-draining, .upgrade-progress-pill.is-delayed { color: var(--amber); }
.upgrade-progress-pill.is-missed, .upgrade-progress-pill.has-missed { color: var(--red); }
.upgrade-progress-pill.is-done { color: var(--green); }
.upgrade-progress-history { margin-left: var(--sp-2); cursor: help; }
@@ -9940,8 +9940,8 @@ 0 ? formatUpgradeElapsed(st.drainRemainingS * 1000) + ' left' : 'deadline reached';
extra = ' · ' + left;
if (agents) tipExtra = '\n' + agents;
- } else if (st.state === 'missed' && st.reason) {
+ } else if ((st.state === 'missed' || st.state === 'delayed') && st.reason) {
tipExtra = '\n' + st.reason;
}
var tip = (st.detail || label) + tipExtra;
diff --git a/src/pkg/hub/spoke/upgrade_history.go b/src/pkg/hub/spoke/upgrade_history.go
new file mode 100644
index 0000000000..0c18b2f575
--- /dev/null
+++ b/src/pkg/hub/spoke/upgrade_history.go
@@ -0,0 +1,30 @@
+package spoke
+
+import "time"
+
+// UpgradeHistoryForDisplay keeps the audit history intact while removing
+// resolved schedule holds. Older hubs recorded those holds as missed windows.
+// upgradedAt lets a spoke clear a hold before its next successful heartbeat.
+func UpgradeHistoryForDisplay(history []UpgradeAttempt, upgradedAt time.Time) []UpgradeAttempt {
+ for _, a := range history {
+ if a.Outcome == UpgradeOutcomeDone && a.EndedAt.After(upgradedAt) {
+ upgradedAt = a.EndedAt
+ }
+ }
+ var rows []UpgradeAttempt
+ for _, a := range history {
+ if a.Outcome == UpgradeOutcomeDelayed ||
+ (a.Trigger == UpgradeTriggerWindow && a.Outcome == UpgradeOutcomeMissed) {
+ at := a.EndedAt
+ if at.IsZero() {
+ at = a.StartedAt
+ }
+ if !upgradedAt.IsZero() && !at.After(upgradedAt) {
+ continue
+ }
+ a.Outcome = UpgradeOutcomeDelayed
+ }
+ rows = append(rows, a)
+ }
+ return rows
+}
diff --git a/src/pkg/hub/spoke/upgrade_history_test.go b/src/pkg/hub/spoke/upgrade_history_test.go
new file mode 100644
index 0000000000..e30202a8b1
--- /dev/null
+++ b/src/pkg/hub/spoke/upgrade_history_test.go
@@ -0,0 +1,62 @@
+package spoke
+
+import (
+ "reflect"
+ "testing"
+ "time"
+)
+
+func TestUpgradeHistoryForDisplay(t *testing.T) {
+ now := time.Date(2026, 10, 30, 12, 0, 0, 0, time.UTC)
+ hold := UpgradeAttempt{
+ Trigger: UpgradeTriggerWindow, Outcome: UpgradeOutcomeMissed, Reason: "merge debounce",
+ StartedAt: now.Add(-10 * time.Minute), EndedAt: now.Add(-5 * time.Minute),
+ }
+ done := UpgradeAttempt{Outcome: UpgradeOutcomeDone, EndedAt: now}
+ failed := UpgradeAttempt{Trigger: UpgradeTriggerScheduled, Outcome: UpgradeOutcomeMissed, EndedAt: now.Add(time.Minute), Reason: "ErrImagePull"}
+ delayed := hold
+ delayed.Outcome = UpgradeOutcomeDelayed
+ newDelay := delayed
+ newDelay.EndedAt = now.Add(time.Minute)
+ unfinished := UpgradeAttempt{Trigger: UpgradeTriggerScheduled, StartedAt: now}
+ noEnd := hold
+ noEnd.EndedAt = time.Time{}
+
+ cases := []struct {
+ name string
+ history []UpgradeAttempt
+ upgradedAt time.Time
+ want []UpgradeAttempt
+ }{
+ {name: "empty"},
+ {name: "legacy window is delayed", history: []UpgradeAttempt{hold}, want: []UpgradeAttempt{delayed}},
+ {name: "new delay stays delayed", history: []UpgradeAttempt{delayed}, want: []UpgradeAttempt{delayed}},
+ {name: "success resolves old window", history: []UpgradeAttempt{hold, done}, want: []UpgradeAttempt{done}},
+ {name: "unsorted history", history: []UpgradeAttempt{done, hold}, want: []UpgradeAttempt{done}},
+ {name: "local success clears stale hub history", history: []UpgradeAttempt{hold}, upgradedAt: now},
+ {name: "success at hold timestamp resolves it", history: []UpgradeAttempt{hold}, upgradedAt: hold.EndedAt},
+ {name: "real failure survives", history: []UpgradeAttempt{hold, failed, done}, want: []UpgradeAttempt{failed, done}},
+ {name: "new delay after success survives", history: []UpgradeAttempt{done, newDelay}, want: []UpgradeAttempt{done, newDelay}},
+ {name: "local success does not hide a newer delay", history: []UpgradeAttempt{newDelay}, upgradedAt: now, want: []UpgradeAttempt{newDelay}},
+ {name: "open attempt survives", history: []UpgradeAttempt{unfinished, done}, want: []UpgradeAttempt{unfinished, done}},
+ {name: "missing end falls back to start", history: []UpgradeAttempt{noEnd}, upgradedAt: now},
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ original := append([]UpgradeAttempt(nil), tc.history...)
+ got := UpgradeHistoryForDisplay(tc.history, tc.upgradedAt)
+ if !reflect.DeepEqual(got, tc.want) {
+ t.Fatalf("display history = %+v, want %+v", got, tc.want)
+ }
+ if !reflect.DeepEqual(tc.history, original) {
+ t.Fatal("presentation must not rewrite the audit history")
+ }
+ if len(got) > 0 {
+ got[0].Reason = "changed by caller"
+ if !reflect.DeepEqual(tc.history, original) {
+ t.Fatal("returned history must not alias the input")
+ }
+ }
+ })
+ }
+}
diff --git a/src/pkg/hub/spoke/upgrade_progress.go b/src/pkg/hub/spoke/upgrade_progress.go
index b0ffc69496..fc0f04535b 100644
--- a/src/pkg/hub/spoke/upgrade_progress.go
+++ b/src/pkg/hub/spoke/upgrade_progress.go
@@ -8,8 +8,8 @@ import (
)
// Upgrade progress states. One scheduled upgrade walks
-// scheduled → draining → pulling → rolling → verifying → done, or ends in
-// missed. The hub derives the state from its registry latch plus the spoke's
+// scheduled (or delayed) → draining → pulling → rolling → verifying → done,
+// or ends in missed. The hub derives the state from its registry latch plus the spoke's
// heartbeat; the spoke derives the same vocabulary from what it knows locally
// (its drain latch, upgrade marker and the hub's policy), so both dashboards
// speak the same words about the same upgrade.
@@ -17,6 +17,8 @@ const (
// UpgradeStateScheduled — nothing in flight; NextWindowAt says when the
// hub will next consider this hive (or "instant"/"next poll").
UpgradeStateScheduled = "scheduled"
+ // UpgradeStateDelayed — a scheduled upgrade is waiting for a gate to clear.
+ UpgradeStateDelayed = "delayed"
// UpgradeStateDraining — the spoke received the instruction and is
// waiting for in-flight agent turns before it restarts.
UpgradeStateDraining = "draining"
@@ -34,14 +36,15 @@ const (
UpgradeStateDone = "done"
// UpgradeStateMissed — the most recent attempt or window did not land;
// Reason says why (drain lapsed, hub restarted, unreachable, image pull
- // failed, held by the schedule gate, …).
+ // failed, …). Temporary schedule holds are delayed, not missed.
UpgradeStateMissed = "missed"
)
// Upgrade attempt outcomes recorded in UpgradeAttempt.Outcome.
const (
- UpgradeOutcomeDone = "done"
- UpgradeOutcomeMissed = "missed"
+ UpgradeOutcomeDone = "done"
+ UpgradeOutcomeMissed = "missed"
+ UpgradeOutcomeDelayed = "delayed"
)
// Upgrade attempt triggers recorded in UpgradeAttempt.Trigger.
@@ -100,7 +103,7 @@ type UpgradeProgress struct {
// hive's agent count so the UI can say "2/5 agents in turn".
AgentsInTurn []UpgradeTurn `json:"agentsInTurn,omitempty"`
AgentsTotal int `json:"agentsTotal,omitempty"`
- // Reason explains a missed state, or qualifies an in-flight one.
+ // Reason explains a delayed/missed state, or qualifies an in-flight one.
Reason string `json:"reason,omitempty"`
// Detail is the one-line human summary the pill shows.
Detail string `json:"detail"`
diff --git a/src/pkg/hub/upgrade_delay_ui_test.go b/src/pkg/hub/upgrade_delay_ui_test.go
new file mode 100644
index 0000000000..c4236b4bab
--- /dev/null
+++ b/src/pkg/hub/upgrade_delay_ui_test.go
@@ -0,0 +1,44 @@
+package hub
+
+import (
+ "os/exec"
+ "strings"
+ "testing"
+)
+
+func TestUpgradeDelayWarningRendering(t *testing.T) {
+ node, err := exec.LookPath("node")
+ if err != nil {
+ t.Skip("node unavailable")
+ }
+ parts := []string{
+ "const assert = require('node:assert/strict');",
+ "function esc(s) { return String(s || ''); }",
+ "function escAttr(s) { return esc(s); }",
+ }
+ for _, name := range []string{"UPGRADE_PROGRESS_PILL_STATES", "UPGRADE_PROGRESS_LABELS", "UPGRADE_HISTORY_MAX_ROWS"} {
+ parts = append(parts, extractHubJSVar(t, dashboardHTML, name)+";")
+ }
+ for _, name := range []string{"upgradeProgressLabel", "upgradeAgentsInTurnText", "formatUpgradeElapsed", "renderUpgradeHistory", "renderUpgradeProgress"} {
+ parts = append(parts, jsFunc(t, name))
+ }
+ parts = append(parts, `
+const hold = {trigger: 'window', outcome: 'delayed', reason: 'merge debounce', endedAt: '2026-10-30T13:04:00Z'};
+const done = {outcome: 'done', target: 'abc1234', endedAt: '2026-10-30T13:08:00Z'};
+const failed = {trigger: 'scheduled', outcome: 'missed', reason: 'ErrImagePull', endedAt: '2026-10-30T13:09:00Z'};
+let html = renderUpgradeProgress({upgradeState: {state: 'delayed', reason: 'merge debounce'}, upgradeHistory: [hold]}, false);
+assert.match(html, /is-delayed/);
+assert.match(html, />Delayed);
+assert.match(html, /merge debounce/);
+assert.doesNotMatch(html, /missed|history/);
+assert.equal(renderUpgradeProgress({upgradeState: {state: 'done'}, upgradeHistory: [done]}, false), '');
+assert.equal(renderUpgradeProgress({upgradeState: {state: 'done'}, upgradeHistory: [hold, done]}, false), '');
+assert.equal(renderUpgradeProgress({upgradeState: {state: 'scheduled'}, upgradeHistory: [{...hold, outcome: 'missed'}, done]}, false), '');
+assert.equal(renderUpgradeHistory({upgradeHistory: [done, {...failed, endedAt: '2026-10-30T13:00:00Z'}]}), '');
+assert.match(renderUpgradeHistory({upgradeHistory: [done, failed]}), /1 missed/);
+assert.match(renderUpgradeHistory({upgradeHistory: [failed]}), /ErrImagePull/);
+`)
+ if out, err := exec.Command(node, "-e", strings.Join(parts, "\n")).CombinedOutput(); err != nil {
+ t.Fatalf("upgrade delay rendering failed: %v\n%s", err, out)
+ }
+}
diff --git a/src/pkg/hub/upgrade_history.go b/src/pkg/hub/upgrade_history.go
index a1d0b4ddf6..bb25851b52 100644
--- a/src/pkg/hub/upgrade_history.go
+++ b/src/pkg/hub/upgrade_history.go
@@ -61,7 +61,7 @@ func upgradeHistoryPath(hiveID string) string {
return filepath.Join(upgradeHistoryDir, hiveID+".json")
}
-// recent returns a copy of the hive's attempts, oldest first.
+// recent returns a display copy, oldest first, without resolved schedule holds.
func (u *upgradeHistoryStore) recent(hiveID string) []spoke.UpgradeAttempt {
if u == nil {
return nil
@@ -72,7 +72,7 @@ func (u *upgradeHistoryStore) recent(hiveID string) []spoke.UpgradeAttempt {
if len(src) == 0 {
return nil
}
- return append([]spoke.UpgradeAttempt(nil), src...)
+ return spoke.UpgradeHistoryForDisplay(src, time.Time{})
}
// open returns the hive's currently open attempt, if any.
@@ -173,7 +173,7 @@ func (u *upgradeHistoryStore) end(hiveID, outcome, reason, from, target string,
u.persistLocked(hiveID)
}
-// windowMissed records a daily/weekly window the hub held instead of firing.
+// windowMissed records a daily/weekly window as delayed, not a failed attempt.
// One row per window: a hold reason that changes while the same window is
// open updates the row rather than adding another, and an attempt already
// open for this hive means the window did fire, so nothing is recorded.
@@ -190,6 +190,7 @@ func (u *upgradeHistoryStore) windowMissed(hiveID, from, reason string, at time.
return
}
if last.Trigger == spoke.UpgradeTriggerWindow && sameUpgradeWindow(last.StartedAt, at) {
+ last.Outcome = spoke.UpgradeOutcomeDelayed
last.Reason = reason
last.EndedAt = at.UTC()
u.byID[hiveID] = list
@@ -202,7 +203,7 @@ func (u *upgradeHistoryStore) windowMissed(hiveID, from, reason string, at time.
Trigger: spoke.UpgradeTriggerWindow,
StartedAt: at.UTC(),
EndedAt: at.UTC(),
- Outcome: spoke.UpgradeOutcomeMissed,
+ Outcome: spoke.UpgradeOutcomeDelayed,
Reason: reason,
})
u.byID[hiveID] = trimUpgradeHistory(list, upgradeHistoryMax())
diff --git a/src/pkg/hub/upgrade_history_test.go b/src/pkg/hub/upgrade_history_test.go
index eff10ea6e3..fd476ef06e 100644
--- a/src/pkg/hub/upgrade_history_test.go
+++ b/src/pkg/hub/upgrade_history_test.go
@@ -84,7 +84,7 @@ func TestUpgradeHistoryWindowMissedOneRowPerDay(t *testing.T) {
if len(got) != 1 {
t.Fatalf("same window must dedupe to one row, got %d", len(got))
}
- if got[0].Trigger != spoke.UpgradeTriggerWindow || got[0].Outcome != spoke.UpgradeOutcomeMissed || got[0].Reason != "spoke still draining" {
+ if got[0].Trigger != spoke.UpgradeTriggerWindow || got[0].Outcome != spoke.UpgradeOutcomeDelayed || got[0].Reason != "spoke still draining" {
t.Fatalf("window row = %+v", got[0])
}
u.windowMissed("h", "551ffc3", "spoke still draining", day1.Add(24*time.Hour))
diff --git a/src/pkg/hub/upgrade_progress.go b/src/pkg/hub/upgrade_progress.go
index f1745c6065..fe90a30b8e 100644
--- a/src/pkg/hub/upgrade_progress.go
+++ b/src/pkg/hub/upgrade_progress.go
@@ -37,7 +37,7 @@ type upgradeProgressInput struct {
// Upgrading + spoke reports upgrading → pulling
// Upgrading + no beat since instruction→ rolling
// Upgrading (older build, silent) → pulling, qualified
-// open window held by the gate → missed (hold reason)
+// open window held by the gate → delayed (hold reason)
// last history row ended recently → that row's outcome
// daily/weekly schedule → scheduled (next window)
// instant / spoke-managed → scheduled (next poll)
@@ -124,8 +124,9 @@ func upgradeProgressFor(in upgradeProgressInput) *spoke.UpgradeProgress {
return p
}
- // Nothing in flight. A window the gate is holding right now is a live
- // miss, before any history row.
+ history := spoke.UpgradeHistoryForDisplay(in.History, time.Time{})
+ last, hasLast := latestClosedAttempt(history)
+ // A newer closed attempt supersedes a stale schedule hold.
daily := false
if in.AutoUpgrade {
switch effectiveAutoUpgradeSchedule(in.Mode, in.Channel) {
@@ -137,18 +138,19 @@ func upgradeProgressFor(in upgradeProgressInput) *spoke.UpgradeProgress {
if daily {
nextWindow = nextScheduledAutoUpgradeAt(in.Mode, in.LastFired, now)
}
- if daily && in.HoldReason != "" && scheduledUpgradeWindowOpen(in.Mode, in.LastFired, now) {
+ if daily && in.HoldReason != "" && scheduledUpgradeWindowOpen(in.Mode, in.LastFired, now) &&
+ (!hasLast || in.HoldAt.After(last.EndedAt)) {
return &spoke.UpgradeProgress{
- State: spoke.UpgradeStateMissed,
+ State: spoke.UpgradeStateDelayed,
From: historyRef(e.GitHash),
NextWindowAt: nextWindow,
Reason: in.HoldReason,
- Detail: "window held: " + in.HoldReason,
+ Detail: "delayed: " + in.HoldReason,
EndedAt: rfc(in.HoldAt),
}
}
- if last, ok := latestClosedAttempt(in.History); ok && now.Sub(last.EndedAt) <= upgradeHistoryRecentFor {
+ if hasLast && now.Sub(last.EndedAt) <= upgradeHistoryRecentFor {
p := &spoke.UpgradeProgress{
State: last.Outcome,
Target: last.Target,
@@ -162,6 +164,8 @@ func upgradeProgressFor(in upgradeProgressInput) *spoke.UpgradeProgress {
switch last.Outcome {
case spoke.UpgradeOutcomeDone:
p.Detail = "upgraded to " + orDash(last.Target) + " " + ago
+ case spoke.UpgradeOutcomeDelayed:
+ p.Detail = "delayed: " + last.Reason
default:
if last.Trigger == spoke.UpgradeTriggerWindow {
p.Detail = "missed window " + ago + ": " + last.Reason
diff --git a/src/pkg/hub/upgrade_progress_markup_test.go b/src/pkg/hub/upgrade_progress_markup_test.go
index cbddff3f7e..a6e2e649ea 100644
--- a/src/pkg/hub/upgrade_progress_markup_test.go
+++ b/src/pkg/hub/upgrade_progress_markup_test.go
@@ -31,7 +31,7 @@ func TestDashboardRendersUpgradeProgressPillAndHistory(t *testing.T) {
}
// Every state the hub can emit has a human label.
labels := extractHubJSVar(t, dashboardHTML, "UPGRADE_PROGRESS_LABELS")
- for _, state := range []string{"scheduled", "draining", "pulling", "rolling", "verifying", "done", "missed"} {
+ for _, state := range []string{"scheduled", "delayed", "draining", "pulling", "rolling", "verifying", "done", "missed"} {
if !strings.Contains(labels, state+":") {
t.Errorf("UPGRADE_PROGRESS_LABELS lacks %q: %s", state, labels)
}
diff --git a/src/pkg/hub/upgrade_progress_test.go b/src/pkg/hub/upgrade_progress_test.go
index afd23afa1a..3ddf0d9ace 100644
--- a/src/pkg/hub/upgrade_progress_test.go
+++ b/src/pkg/hub/upgrade_progress_test.go
@@ -83,19 +83,53 @@ func TestUpgradeProgressForStates(t *testing.T) {
state: spoke.UpgradeStatePulling, detail: "draining silently",
},
{
- name: "open window held by the gate is missed now",
+ name: "open window held by the gate is delayed",
in: upgradeProgressInput{Entry: &RegistryEntry{GitHash: "551ffc3"},
AutoUpgrade: true, Mode: AutoUpgradeModeDaily, Channel: ReleaseChannelStable,
HoldReason: "hub restarted; stuck-upgrade latch recovering", HoldAt: now.Add(-10 * time.Minute)},
- state: spoke.UpgradeStateMissed, detail: "window held", nextWin: true,
+ state: spoke.UpgradeStateDelayed, detail: "delayed", nextWin: true,
},
{
- name: "recent missed window from history",
+ name: "legacy missed window is delayed",
in: upgradeProgressInput{Entry: &RegistryEntry{GitHash: "551ffc3"},
AutoUpgrade: true, Mode: AutoUpgradeModeDaily, LastFired: now.Format(autoUpgradeDateFormat),
History: []spoke.UpgradeAttempt{{Trigger: spoke.UpgradeTriggerWindow, Outcome: spoke.UpgradeOutcomeMissed,
Reason: "drain lapsed", StartedAt: now.Add(-2 * time.Hour), EndedAt: now.Add(-2 * time.Hour)}}},
- state: spoke.UpgradeStateMissed, detail: "missed window 2h0m ago: drain lapsed", nextWin: true,
+ state: spoke.UpgradeStateDelayed, detail: "delayed: drain lapsed", nextWin: true,
+ },
+ {
+ name: "successful upgrade clears a stale live hold and legacy window",
+ in: upgradeProgressInput{Entry: &RegistryEntry{GitHash: "abc1234"},
+ AutoUpgrade: true, Mode: AutoUpgradeModeDaily, HoldReason: "merge debounce", HoldAt: now.Add(-10 * time.Minute),
+ History: []spoke.UpgradeAttempt{
+ {Trigger: spoke.UpgradeTriggerWindow, Outcome: spoke.UpgradeOutcomeMissed, Reason: "merge debounce", EndedAt: now.Add(-10 * time.Minute)},
+ {Target: "abc1234", Outcome: spoke.UpgradeOutcomeDone, EndedAt: now.Add(-5 * time.Minute)},
+ }},
+ state: spoke.UpgradeStateDone, detail: "upgraded to abc1234", nextWin: true,
+ },
+ {
+ name: "new hold after a successful upgrade remains visible",
+ in: upgradeProgressInput{Entry: &RegistryEntry{GitHash: "abc1234"},
+ AutoUpgrade: true, Mode: AutoUpgradeModeDaily, HoldReason: "merge debounce", HoldAt: now.Add(-time.Minute),
+ History: []spoke.UpgradeAttempt{
+ {Target: "abc1234", Outcome: spoke.UpgradeOutcomeDone, EndedAt: now.Add(-5 * time.Minute)},
+ }},
+ state: spoke.UpgradeStateDelayed, detail: "merge debounce", nextWin: true,
+ },
+ {
+ name: "genuine failed attempt is not relabeled",
+ in: upgradeProgressInput{Entry: &RegistryEntry{GitHash: "551ffc3"},
+ History: []spoke.UpgradeAttempt{{Trigger: spoke.UpgradeTriggerScheduled, Outcome: spoke.UpgradeOutcomeMissed,
+ Reason: "ErrImagePull", EndedAt: now.Add(-time.Minute)}}},
+ state: spoke.UpgradeStateMissed, detail: "ErrImagePull",
+ },
+ {
+ name: "stale hold does not mask a later failed attempt",
+ in: upgradeProgressInput{Entry: &RegistryEntry{GitHash: "551ffc3"},
+ AutoUpgrade: true, Mode: AutoUpgradeModeDaily, HoldReason: "merge debounce", HoldAt: now.Add(-10 * time.Minute),
+ History: []spoke.UpgradeAttempt{{Trigger: spoke.UpgradeTriggerScheduled, Outcome: spoke.UpgradeOutcomeMissed,
+ Reason: "ErrImagePull", EndedAt: now.Add(-time.Minute)}}},
+ state: spoke.UpgradeStateMissed, detail: "ErrImagePull", nextWin: true,
},
{
name: "recent done from history",
@@ -161,7 +195,7 @@ func TestUpgradeProgressInFlight(t *testing.T) {
t.Fatalf("%s should be in flight", st)
}
}
- for _, st := range []string{spoke.UpgradeStateScheduled, spoke.UpgradeStateDone, spoke.UpgradeStateMissed} {
+ for _, st := range []string{spoke.UpgradeStateScheduled, spoke.UpgradeStateDelayed, spoke.UpgradeStateDone, spoke.UpgradeStateMissed} {
if (&spoke.UpgradeProgress{State: st}).InFlight() {
t.Fatalf("%s should not be in flight", st)
}
@@ -190,9 +224,14 @@ func TestHeartbeatUpgradePolicyCarriesWindowAndHistory(t *testing.T) {
if _, err := time.Parse(time.RFC3339, policy.NextWindowAt); err != nil {
t.Fatalf("NextWindowAt not RFC3339: %q", policy.NextWindowAt)
}
- if len(policy.UpgradeHistory) != 1 || policy.UpgradeHistory[0].Reason != "hub restarted" {
+ if len(policy.UpgradeHistory) != 1 || policy.UpgradeHistory[0].Outcome != spoke.UpgradeOutcomeDelayed || policy.UpgradeHistory[0].Reason != "hub restarted" {
t.Fatalf("history not delivered on the policy: %+v", policy.UpgradeHistory)
}
+ s.upgradeHistory.end("fma", spoke.UpgradeOutcomeDone, "", "551ffc3", "5193426", time.Now())
+ policy = s.heartbeatUpgradePolicy(payload, sh, false, false, "v5", "")
+ if policy == nil || len(policy.UpgradeHistory) != 1 || policy.UpgradeHistory[0].Outcome != spoke.UpgradeOutcomeDone {
+ t.Fatalf("successful upgrade must remove the delay from the policy: %+v", policy)
+ }
}
func TestUpgradeProgressForRowReadsSaaSHiveAndHistory(t *testing.T) {