diff --git a/changelog.d/fixed-upgrade-delay-warnings.md b/changelog.d/fixed-upgrade-delay-warnings.md new file mode 100644 index 0000000000..c0e0b57cc0 --- /dev/null +++ b/changelog.d/fixed-upgrade-delay-warnings.md @@ -0,0 +1 @@ +- Show temporary upgrade holds as Delayed instead of missed windows, and clear resolved delay warnings after a successful upgrade on both dashboards, including older history entries. diff --git a/src/pkg/dashboard/api_version.go b/src/pkg/dashboard/api_version.go index bffdd673db..881d5d667d 100644 --- a/src/pkg/dashboard/api_version.go +++ b/src/pkg/dashboard/api_version.go @@ -95,7 +95,13 @@ func (s *Server) handleVersion(w http.ResponseWriter, r *http.Request) { resp["upgradeProgress"] = progress } if policy != nil && len(policy.UpgradeHistory) > 0 { - resp["upgradeHistory"] = policy.UpgradeHistory + var upgradedAt time.Time + if upgradeOutcomeRec != nil { + upgradedAt = upgradeOutcomeRec.CompletedAt + } + if history := spoke.UpgradeHistoryForDisplay(policy.UpgradeHistory, upgradedAt); len(history) > 0 { + resp["upgradeHistory"] = history + } } if cacheAge > dashboardVersionTipCacheTTL || cached == "" { diff --git a/src/pkg/dashboard/static-src/css/topbar.css b/src/pkg/dashboard/static-src/css/topbar.css index ef84522944..93334c2f74 100644 --- a/src/pkg/dashboard/static-src/css/topbar.css +++ b/src/pkg/dashboard/static-src/css/topbar.css @@ -38,7 +38,7 @@ .oc-version-manual-pill { white-space: nowrap; padding: 1px 7px; border-radius: var(--r-pill); border: 1px solid var(--border); color: var(--muted); font-size: var(--fs-xs); } .oc-version-upgrade-note { color: var(--muted); font-size: var(--fs-xs); overflow-wrap: anywhere; } .oc-version-state-pill { white-space: nowrap; padding: var(--sp-1) var(--sp-3); border-radius: var(--r-pill); border: 1px solid var(--border); color: var(--muted); font-size: var(--fs-xs); font-variant-numeric: tabular-nums; } - .oc-version-state-pill.is-draining { color: var(--amber); border-color: color-mix(in srgb, var(--amber) 40%, transparent); } + .oc-version-state-pill.is-draining, .oc-version-state-pill.is-delayed { color: var(--amber); border-color: color-mix(in srgb, var(--amber) 40%, transparent); } .oc-version-state-pill.is-missed { color: var(--red, var(--danger)); border-color: color-mix(in srgb, var(--red, var(--danger)) 40%, transparent); } .oc-version-state-pill.is-done { color: var(--green, var(--success)); } .oc-version-history { list-style: none; margin: 0; padding: 0; font-size: var(--fs-xs); color: var(--muted); display: flex; flex-direction: column; gap: 2px; } diff --git a/src/pkg/dashboard/static-src/js/version.js b/src/pkg/dashboard/static-src/js/version.js index 42df507c93..a294b2db8b 100644 --- a/src/pkg/dashboard/static-src/js/version.js +++ b/src/pkg/dashboard/static-src/js/version.js @@ -228,7 +228,7 @@ state as a pill — with the drain countdown and the agents holding the upgrade — and lists the history so a missed window is never silent. */ function versionStateLabels() { - return { scheduled: 'Scheduled', draining: 'Draining', pulling: 'Pulling', rolling: 'Rolling', verifying: 'Verifying', done: 'Upgraded', missed: 'Missed window' }; + return { scheduled: 'Scheduled', delayed: 'Delayed', draining: 'Draining', pulling: 'Pulling', rolling: 'Rolling', verifying: 'Verifying', done: 'Upgraded', missed: 'Missed window' }; } function versionStateLabel(state) { @@ -261,7 +261,7 @@ const agents = versionAgentsInTurnText(st); const left = typeof st.drainRemainingS === 'number' && st.drainRemainingS > 0 ? versionFormatDuration(st.drainRemainingS) + ' left' : 'deadline reached'; text += (agents ? ' · ' + agents : '') + ' · ' + left; - } else if (st.state === 'missed' && st.reason) { + } else if ((st.state === 'missed' || st.state === 'delayed') && st.reason) { text += ' · ' + st.reason; } else if (st.state === 'scheduled' && st.nextWindowAt) { const at = new Date(st.nextWindowAt); @@ -1427,4 +1427,3 @@ }, VERSION_CHECK_INTERVAL_MS); } // startVersionCheck() — deferred to _deferredInit - diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html index 06da9262d6..934d50c71a 100644 --- a/src/pkg/dashboard/static/index.html +++ b/src/pkg/dashboard/static/index.html @@ -404,7 +404,7 @@ .oc-version-manual-pill { white-space: nowrap; padding: 1px 7px; border-radius: var(--r-pill); border: 1px solid var(--border); color: var(--muted); font-size: var(--fs-xs); } .oc-version-upgrade-note { color: var(--muted); font-size: var(--fs-xs); overflow-wrap: anywhere; } .oc-version-state-pill { white-space: nowrap; padding: var(--sp-1) var(--sp-3); border-radius: var(--r-pill); border: 1px solid var(--border); color: var(--muted); font-size: var(--fs-xs); font-variant-numeric: tabular-nums; } - .oc-version-state-pill.is-draining { color: var(--amber); border-color: color-mix(in srgb, var(--amber) 40%, transparent); } + .oc-version-state-pill.is-draining, .oc-version-state-pill.is-delayed { color: var(--amber); border-color: color-mix(in srgb, var(--amber) 40%, transparent); } .oc-version-state-pill.is-missed { color: var(--red, var(--danger)); border-color: color-mix(in srgb, var(--red, var(--danger)) 40%, transparent); } .oc-version-state-pill.is-done { color: var(--green, var(--success)); } .oc-version-history { list-style: none; margin: 0; padding: 0; font-size: var(--fs-xs); color: var(--muted); display: flex; flex-direction: column; gap: 2px; } @@ -45721,7 +45721,7 @@

Configure LiteLLM

state as a pill — with the drain countdown and the agents holding the upgrade — and lists the history so a missed window is never silent. */ function versionStateLabels() { - return { scheduled: 'Scheduled', draining: 'Draining', pulling: 'Pulling', rolling: 'Rolling', verifying: 'Verifying', done: 'Upgraded', missed: 'Missed window' }; + return { scheduled: 'Scheduled', delayed: 'Delayed', draining: 'Draining', pulling: 'Pulling', rolling: 'Rolling', verifying: 'Verifying', done: 'Upgraded', missed: 'Missed window' }; } function versionStateLabel(state) { @@ -45754,7 +45754,7 @@

Configure LiteLLM

const agents = versionAgentsInTurnText(st); const left = typeof st.drainRemainingS === 'number' && st.drainRemainingS > 0 ? versionFormatDuration(st.drainRemainingS) + ' left' : 'deadline reached'; text += (agents ? ' · ' + agents : '') + ' · ' + left; - } else if (st.state === 'missed' && st.reason) { + } else if ((st.state === 'missed' || st.state === 'delayed') && st.reason) { text += ' · ' + st.reason; } else if (st.state === 'scheduled' && st.nextWindowAt) { const at = new Date(st.nextWindowAt); @@ -46920,7 +46920,6 @@

Configure LiteLLM

}, VERSION_CHECK_INTERVAL_MS); } // startVersionCheck() — deferred to _deferredInit - var _eventSource = null; var _eventSourceReconnectTimer = null; var _sseLastFrameAtMs = 0; diff --git a/src/pkg/dashboard/upgrade_progress.go b/src/pkg/dashboard/upgrade_progress.go index ba51d32f6b..dfbedf0d7c 100644 --- a/src/pkg/dashboard/upgrade_progress.go +++ b/src/pkg/dashboard/upgrade_progress.go @@ -63,6 +63,11 @@ func buildUpgradeProgress(in upgradeProgressInput) *spoke.UpgradeProgress { nextWindow = in.Policy.NextWindowAt history = in.Policy.UpgradeHistory } + var upgradedAt time.Time + if in.Outcome != nil { + upgradedAt = in.Outcome.CompletedAt + } + history = spoke.UpgradeHistoryForDisplay(history, upgradedAt) if d := in.Drain; d != nil { turns := d.Turns @@ -114,7 +119,9 @@ func buildUpgradeProgress(in upgradeProgressInput) *spoke.UpgradeProgress { return p } - if o := in.Outcome; o != nil && !o.CompletedAt.IsZero() && !in.StartedAt.IsZero() && !o.CompletedAt.Before(in.StartedAt.Add(-time.Minute)) { + last, hasLast := latestClosedAttempt(history) + if o := in.Outcome; o != nil && !o.CompletedAt.IsZero() && !in.StartedAt.IsZero() && + !o.CompletedAt.Before(in.StartedAt.Add(-time.Minute)) && (!hasLast || !last.EndedAt.After(o.CompletedAt)) { return &spoke.UpgradeProgress{ State: spoke.UpgradeStateDone, Target: shortSHADashboard(o.TargetSHA), @@ -126,7 +133,7 @@ func buildUpgradeProgress(in upgradeProgressInput) *spoke.UpgradeProgress { } } - if last, ok := latestClosedAttempt(history); ok && now.Sub(last.EndedAt) <= upgradeHistoryRecentFor { + if hasLast && now.Sub(last.EndedAt) <= upgradeHistoryRecentFor { p := &spoke.UpgradeProgress{ State: last.Outcome, Target: last.Target, @@ -140,6 +147,8 @@ func buildUpgradeProgress(in upgradeProgressInput) *spoke.UpgradeProgress { switch last.Outcome { case spoke.UpgradeOutcomeDone: p.Detail = "upgraded to " + orDashDashboard(last.Target) + " " + ago + case spoke.UpgradeOutcomeDelayed: + p.Detail = "delayed: " + last.Reason default: if last.Trigger == spoke.UpgradeTriggerWindow { p.Detail = "missed window " + ago + ": " + last.Reason diff --git a/src/pkg/dashboard/upgrade_progress_test.go b/src/pkg/dashboard/upgrade_progress_test.go index 8fd681f2ea..16f4e43c6c 100644 --- a/src/pkg/dashboard/upgrade_progress_test.go +++ b/src/pkg/dashboard/upgrade_progress_test.go @@ -54,13 +54,43 @@ func TestBuildUpgradeProgressStates(t *testing.T) { state: spoke.UpgradeStateDone, detail: "upgraded to abc1234 9m ago", }, { - name: "old outcome ignored, hub history missed window shown", + name: "old outcome ignored, legacy hub window shown as delayed", in: upgradeProgressInput{StartedAt: now.Add(-10 * time.Minute), Outcome: &upgradeOutcome{TargetSHA: "abc1234", CompletedAt: now.Add(-48 * time.Hour)}, Policy: &spoke.HeartbeatUpgradePolicy{HubManaged: true, NextWindowAt: next, UpgradeHistory: []spoke.UpgradeAttempt{ {Trigger: spoke.UpgradeTriggerWindow, Outcome: spoke.UpgradeOutcomeMissed, Reason: "spoke still draining", StartedAt: now.Add(-3 * time.Hour), EndedAt: now.Add(-3 * time.Hour)}}}}, - state: spoke.UpgradeStateMissed, detail: "missed window 3h0m ago: spoke still draining", + state: spoke.UpgradeStateDelayed, detail: "delayed: spoke still draining", + }, + { + name: "landed outcome clears stale hub delay before next heartbeat", + in: upgradeProgressInput{StartedAt: now.Add(-10 * time.Minute), + Outcome: &upgradeOutcome{TargetSHA: "abc1234", CompletedAt: now.Add(-9 * time.Minute)}, + Policy: &spoke.HeartbeatUpgradePolicy{HubManaged: true, UpgradeHistory: []spoke.UpgradeAttempt{ + {Trigger: spoke.UpgradeTriggerWindow, Outcome: spoke.UpgradeOutcomeMissed, Reason: "merge debounce", + StartedAt: now.Add(-15 * time.Minute), EndedAt: now.Add(-12 * time.Minute)}, + }}}, + state: spoke.UpgradeStateDone, detail: "upgraded to abc1234", + }, + { + name: "new delay is not hidden by this boot's earlier success", + in: upgradeProgressInput{StartedAt: now.Add(-10 * time.Minute), + Outcome: &upgradeOutcome{TargetSHA: "abc1234", CompletedAt: now.Add(-9 * time.Minute)}, + Policy: &spoke.HeartbeatUpgradePolicy{HubManaged: true, UpgradeHistory: []spoke.UpgradeAttempt{ + {Trigger: spoke.UpgradeTriggerWindow, Outcome: spoke.UpgradeOutcomeDelayed, Reason: "merge debounce", + StartedAt: now.Add(-2 * time.Minute), EndedAt: now.Add(-time.Minute)}, + }}}, + state: spoke.UpgradeStateDelayed, detail: "merge debounce", + }, + { + name: "genuine hub failure remains missed", + in: upgradeProgressInput{ + StartedAt: now.Add(-10 * time.Minute), + Outcome: &upgradeOutcome{TargetSHA: "abc1234", CompletedAt: now.Add(-9 * time.Minute)}, + Policy: &spoke.HeartbeatUpgradePolicy{HubManaged: true, UpgradeHistory: []spoke.UpgradeAttempt{ + {Trigger: spoke.UpgradeTriggerScheduled, Outcome: spoke.UpgradeOutcomeMissed, Reason: "ErrImagePull", EndedAt: now.Add(-time.Minute)}, + }}}, + state: spoke.UpgradeStateMissed, detail: "ErrImagePull", }, { name: "scheduled with window", diff --git a/src/pkg/dashboard/upgrade_progress_ui_test.go b/src/pkg/dashboard/upgrade_progress_ui_test.go index 40250cc65a..a40e29e60c 100644 --- a/src/pkg/dashboard/upgrade_progress_ui_test.go +++ b/src/pkg/dashboard/upgrade_progress_ui_test.go @@ -4,6 +4,7 @@ import ( "encoding/json" "net/http" "net/http/httptest" + "os" "path/filepath" "strings" "testing" @@ -55,10 +56,32 @@ func TestVersionCarriesUpgradeProgressAndHistory(t *testing.T) { if len(p.AgentsInTurn) != 1 || p.AgentsInTurn[0].Agent != "reviewer" || p.AgentsInTurn[0].ElapsedS != 2280 { t.Fatalf("agentsInTurn = %+v", p.AgentsInTurn) } - if len(body.UpgradeHistory) != 1 || body.UpgradeHistory[0].Outcome != spoke.UpgradeOutcomeMissed || body.UpgradeHistory[0].Reason != "drain lapsed" { + if len(body.UpgradeHistory) != 1 || body.UpgradeHistory[0].Outcome != spoke.UpgradeOutcomeDelayed || body.UpgradeHistory[0].Reason != "drain lapsed" { t.Fatalf("upgradeHistory = %+v", body.UpgradeHistory) } + // The spoke knows the rollout landed before the hub's next heartbeat. + s.SetUpgradeDrainSource(nil) + outcome, err := json.Marshal(upgradeOutcome{TargetSHA: "abc1234", CompletedAt: time.Now()}) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(upgradeOutcomePath, outcome, 0o600); err != nil { + t.Fatal(err) + } + rec = httptest.NewRecorder() + s.handleVersion(rec, httptest.NewRequest(http.MethodGet, "/api/version", nil)) + body.UpgradeHistory = nil + if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil { + t.Fatal(err) + } + if body.UpgradeProgress == nil || body.UpgradeProgress.State != spoke.UpgradeStateDone || len(body.UpgradeHistory) != 0 { + t.Fatalf("successful upgrade still shows a delay: %+v", body) + } + if err := os.Remove(upgradeOutcomePath); err != nil { + t.Fatal(err) + } + // No policy, no drain, nothing on disk: the keys are absent rather than // null so older SPA builds see exactly the payload they did before. s2 := &Server{status: &StatusPayload{}, startedAt: time.Now()} @@ -88,6 +111,7 @@ func TestIndexRendersUpgradeProgressPillAndHistory(t *testing.T) { "st.agentsInTurn", ".oc-version-state-pill {", ".oc-version-state-pill.is-draining", + ".oc-version-state-pill.is-delayed", ".oc-version-state-pill.is-missed", ".oc-version-history {", "row('Progress', statePill", @@ -98,7 +122,7 @@ func TestIndexRendersUpgradeProgressPillAndHistory(t *testing.T) { } } labels := jsFunc(t, html, "versionStateLabels") - for _, state := range []string{"scheduled", "draining", "pulling", "rolling", "verifying", "done", "missed"} { + for _, state := range []string{"scheduled", "delayed", "draining", "pulling", "rolling", "verifying", "done", "missed"} { if !strings.Contains(labels, state+":") { t.Errorf("versionStateLabels lacks %q", state) } diff --git a/src/pkg/hub/assets/dashboard.html b/src/pkg/hub/assets/dashboard.html index 7ba7a93a6c..b1dff8d84a 100644 --- a/src/pkg/hub/assets/dashboard.html +++ b/src/pkg/hub/assets/dashboard.html @@ -693,7 +693,7 @@ /* Scheduled-upgrade progress pill, history badge and interrupt action on the hive card's Version cell (#14950). */ .upgrade-progress-pill { display: inline-block; padding: var(--sp-1) var(--sp-3); border-radius: var(--r-pill); font-size: var(--fs-2xs); background: var(--surface); color: var(--muted); border: 1px solid var(--border); white-space: nowrap; font-variant-numeric: tabular-nums; } - .upgrade-progress-pill.is-draining { color: var(--amber); } + .upgrade-progress-pill.is-draining, .upgrade-progress-pill.is-delayed { color: var(--amber); } .upgrade-progress-pill.is-missed, .upgrade-progress-pill.has-missed { color: var(--red); } .upgrade-progress-pill.is-done { color: var(--green); } .upgrade-progress-history { margin-left: var(--sp-2); cursor: help; } @@ -9940,8 +9940,8 @@

0 ? formatUpgradeElapsed(st.drainRemainingS * 1000) + ' left' : 'deadline reached'; extra = ' · ' + left; if (agents) tipExtra = '\n' + agents; - } else if (st.state === 'missed' && st.reason) { + } else if ((st.state === 'missed' || st.state === 'delayed') && st.reason) { tipExtra = '\n' + st.reason; } var tip = (st.detail || label) + tipExtra; diff --git a/src/pkg/hub/spoke/upgrade_history.go b/src/pkg/hub/spoke/upgrade_history.go new file mode 100644 index 0000000000..0c18b2f575 --- /dev/null +++ b/src/pkg/hub/spoke/upgrade_history.go @@ -0,0 +1,30 @@ +package spoke + +import "time" + +// UpgradeHistoryForDisplay keeps the audit history intact while removing +// resolved schedule holds. Older hubs recorded those holds as missed windows. +// upgradedAt lets a spoke clear a hold before its next successful heartbeat. +func UpgradeHistoryForDisplay(history []UpgradeAttempt, upgradedAt time.Time) []UpgradeAttempt { + for _, a := range history { + if a.Outcome == UpgradeOutcomeDone && a.EndedAt.After(upgradedAt) { + upgradedAt = a.EndedAt + } + } + var rows []UpgradeAttempt + for _, a := range history { + if a.Outcome == UpgradeOutcomeDelayed || + (a.Trigger == UpgradeTriggerWindow && a.Outcome == UpgradeOutcomeMissed) { + at := a.EndedAt + if at.IsZero() { + at = a.StartedAt + } + if !upgradedAt.IsZero() && !at.After(upgradedAt) { + continue + } + a.Outcome = UpgradeOutcomeDelayed + } + rows = append(rows, a) + } + return rows +} diff --git a/src/pkg/hub/spoke/upgrade_history_test.go b/src/pkg/hub/spoke/upgrade_history_test.go new file mode 100644 index 0000000000..e30202a8b1 --- /dev/null +++ b/src/pkg/hub/spoke/upgrade_history_test.go @@ -0,0 +1,62 @@ +package spoke + +import ( + "reflect" + "testing" + "time" +) + +func TestUpgradeHistoryForDisplay(t *testing.T) { + now := time.Date(2026, 10, 30, 12, 0, 0, 0, time.UTC) + hold := UpgradeAttempt{ + Trigger: UpgradeTriggerWindow, Outcome: UpgradeOutcomeMissed, Reason: "merge debounce", + StartedAt: now.Add(-10 * time.Minute), EndedAt: now.Add(-5 * time.Minute), + } + done := UpgradeAttempt{Outcome: UpgradeOutcomeDone, EndedAt: now} + failed := UpgradeAttempt{Trigger: UpgradeTriggerScheduled, Outcome: UpgradeOutcomeMissed, EndedAt: now.Add(time.Minute), Reason: "ErrImagePull"} + delayed := hold + delayed.Outcome = UpgradeOutcomeDelayed + newDelay := delayed + newDelay.EndedAt = now.Add(time.Minute) + unfinished := UpgradeAttempt{Trigger: UpgradeTriggerScheduled, StartedAt: now} + noEnd := hold + noEnd.EndedAt = time.Time{} + + cases := []struct { + name string + history []UpgradeAttempt + upgradedAt time.Time + want []UpgradeAttempt + }{ + {name: "empty"}, + {name: "legacy window is delayed", history: []UpgradeAttempt{hold}, want: []UpgradeAttempt{delayed}}, + {name: "new delay stays delayed", history: []UpgradeAttempt{delayed}, want: []UpgradeAttempt{delayed}}, + {name: "success resolves old window", history: []UpgradeAttempt{hold, done}, want: []UpgradeAttempt{done}}, + {name: "unsorted history", history: []UpgradeAttempt{done, hold}, want: []UpgradeAttempt{done}}, + {name: "local success clears stale hub history", history: []UpgradeAttempt{hold}, upgradedAt: now}, + {name: "success at hold timestamp resolves it", history: []UpgradeAttempt{hold}, upgradedAt: hold.EndedAt}, + {name: "real failure survives", history: []UpgradeAttempt{hold, failed, done}, want: []UpgradeAttempt{failed, done}}, + {name: "new delay after success survives", history: []UpgradeAttempt{done, newDelay}, want: []UpgradeAttempt{done, newDelay}}, + {name: "local success does not hide a newer delay", history: []UpgradeAttempt{newDelay}, upgradedAt: now, want: []UpgradeAttempt{newDelay}}, + {name: "open attempt survives", history: []UpgradeAttempt{unfinished, done}, want: []UpgradeAttempt{unfinished, done}}, + {name: "missing end falls back to start", history: []UpgradeAttempt{noEnd}, upgradedAt: now}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + original := append([]UpgradeAttempt(nil), tc.history...) + got := UpgradeHistoryForDisplay(tc.history, tc.upgradedAt) + if !reflect.DeepEqual(got, tc.want) { + t.Fatalf("display history = %+v, want %+v", got, tc.want) + } + if !reflect.DeepEqual(tc.history, original) { + t.Fatal("presentation must not rewrite the audit history") + } + if len(got) > 0 { + got[0].Reason = "changed by caller" + if !reflect.DeepEqual(tc.history, original) { + t.Fatal("returned history must not alias the input") + } + } + }) + } +} diff --git a/src/pkg/hub/spoke/upgrade_progress.go b/src/pkg/hub/spoke/upgrade_progress.go index b0ffc69496..fc0f04535b 100644 --- a/src/pkg/hub/spoke/upgrade_progress.go +++ b/src/pkg/hub/spoke/upgrade_progress.go @@ -8,8 +8,8 @@ import ( ) // Upgrade progress states. One scheduled upgrade walks -// scheduled → draining → pulling → rolling → verifying → done, or ends in -// missed. The hub derives the state from its registry latch plus the spoke's +// scheduled (or delayed) → draining → pulling → rolling → verifying → done, +// or ends in missed. The hub derives the state from its registry latch plus the spoke's // heartbeat; the spoke derives the same vocabulary from what it knows locally // (its drain latch, upgrade marker and the hub's policy), so both dashboards // speak the same words about the same upgrade. @@ -17,6 +17,8 @@ const ( // UpgradeStateScheduled — nothing in flight; NextWindowAt says when the // hub will next consider this hive (or "instant"/"next poll"). UpgradeStateScheduled = "scheduled" + // UpgradeStateDelayed — a scheduled upgrade is waiting for a gate to clear. + UpgradeStateDelayed = "delayed" // UpgradeStateDraining — the spoke received the instruction and is // waiting for in-flight agent turns before it restarts. UpgradeStateDraining = "draining" @@ -34,14 +36,15 @@ const ( UpgradeStateDone = "done" // UpgradeStateMissed — the most recent attempt or window did not land; // Reason says why (drain lapsed, hub restarted, unreachable, image pull - // failed, held by the schedule gate, …). + // failed, …). Temporary schedule holds are delayed, not missed. UpgradeStateMissed = "missed" ) // Upgrade attempt outcomes recorded in UpgradeAttempt.Outcome. const ( - UpgradeOutcomeDone = "done" - UpgradeOutcomeMissed = "missed" + UpgradeOutcomeDone = "done" + UpgradeOutcomeMissed = "missed" + UpgradeOutcomeDelayed = "delayed" ) // Upgrade attempt triggers recorded in UpgradeAttempt.Trigger. @@ -100,7 +103,7 @@ type UpgradeProgress struct { // hive's agent count so the UI can say "2/5 agents in turn". AgentsInTurn []UpgradeTurn `json:"agentsInTurn,omitempty"` AgentsTotal int `json:"agentsTotal,omitempty"` - // Reason explains a missed state, or qualifies an in-flight one. + // Reason explains a delayed/missed state, or qualifies an in-flight one. Reason string `json:"reason,omitempty"` // Detail is the one-line human summary the pill shows. Detail string `json:"detail"` diff --git a/src/pkg/hub/upgrade_delay_ui_test.go b/src/pkg/hub/upgrade_delay_ui_test.go new file mode 100644 index 0000000000..c4236b4bab --- /dev/null +++ b/src/pkg/hub/upgrade_delay_ui_test.go @@ -0,0 +1,44 @@ +package hub + +import ( + "os/exec" + "strings" + "testing" +) + +func TestUpgradeDelayWarningRendering(t *testing.T) { + node, err := exec.LookPath("node") + if err != nil { + t.Skip("node unavailable") + } + parts := []string{ + "const assert = require('node:assert/strict');", + "function esc(s) { return String(s || ''); }", + "function escAttr(s) { return esc(s); }", + } + for _, name := range []string{"UPGRADE_PROGRESS_PILL_STATES", "UPGRADE_PROGRESS_LABELS", "UPGRADE_HISTORY_MAX_ROWS"} { + parts = append(parts, extractHubJSVar(t, dashboardHTML, name)+";") + } + for _, name := range []string{"upgradeProgressLabel", "upgradeAgentsInTurnText", "formatUpgradeElapsed", "renderUpgradeHistory", "renderUpgradeProgress"} { + parts = append(parts, jsFunc(t, name)) + } + parts = append(parts, ` +const hold = {trigger: 'window', outcome: 'delayed', reason: 'merge debounce', endedAt: '2026-10-30T13:04:00Z'}; +const done = {outcome: 'done', target: 'abc1234', endedAt: '2026-10-30T13:08:00Z'}; +const failed = {trigger: 'scheduled', outcome: 'missed', reason: 'ErrImagePull', endedAt: '2026-10-30T13:09:00Z'}; +let html = renderUpgradeProgress({upgradeState: {state: 'delayed', reason: 'merge debounce'}, upgradeHistory: [hold]}, false); +assert.match(html, /is-delayed/); +assert.match(html, />Delayed