Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/fixed-upgrade-delay-warnings.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- Show temporary upgrade holds as Delayed instead of missed windows, and clear resolved delay warnings after a successful upgrade on both dashboards, including older history entries.
8 changes: 7 additions & 1 deletion src/pkg/dashboard/api_version.go
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,13 @@ func (s *Server) handleVersion(w http.ResponseWriter, r *http.Request) {
resp["upgradeProgress"] = progress
}
if policy != nil && len(policy.UpgradeHistory) > 0 {
resp["upgradeHistory"] = policy.UpgradeHistory
var upgradedAt time.Time
if upgradeOutcomeRec != nil {
upgradedAt = upgradeOutcomeRec.CompletedAt
}
if history := spoke.UpgradeHistoryForDisplay(policy.UpgradeHistory, upgradedAt); len(history) > 0 {
resp["upgradeHistory"] = history
}
}

if cacheAge > dashboardVersionTipCacheTTL || cached == "" {
Expand Down
2 changes: 1 addition & 1 deletion src/pkg/dashboard/static-src/css/topbar.css
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@
.oc-version-manual-pill { white-space: nowrap; padding: 1px 7px; border-radius: var(--r-pill); border: 1px solid var(--border); color: var(--muted); font-size: var(--fs-xs); }
.oc-version-upgrade-note { color: var(--muted); font-size: var(--fs-xs); overflow-wrap: anywhere; }
.oc-version-state-pill { white-space: nowrap; padding: var(--sp-1) var(--sp-3); border-radius: var(--r-pill); border: 1px solid var(--border); color: var(--muted); font-size: var(--fs-xs); font-variant-numeric: tabular-nums; }
.oc-version-state-pill.is-draining { color: var(--amber); border-color: color-mix(in srgb, var(--amber) 40%, transparent); }
.oc-version-state-pill.is-draining, .oc-version-state-pill.is-delayed { color: var(--amber); border-color: color-mix(in srgb, var(--amber) 40%, transparent); }
.oc-version-state-pill.is-missed { color: var(--red, var(--danger)); border-color: color-mix(in srgb, var(--red, var(--danger)) 40%, transparent); }
.oc-version-state-pill.is-done { color: var(--green, var(--success)); }
.oc-version-history { list-style: none; margin: 0; padding: 0; font-size: var(--fs-xs); color: var(--muted); display: flex; flex-direction: column; gap: 2px; }
Expand Down
5 changes: 2 additions & 3 deletions src/pkg/dashboard/static-src/js/version.js
Original file line number Diff line number Diff line change
Expand Up @@ -228,7 +228,7 @@
state as a pill — with the drain countdown and the agents holding the
upgrade — and lists the history so a missed window is never silent. */
function versionStateLabels() {
return { scheduled: 'Scheduled', draining: 'Draining', pulling: 'Pulling', rolling: 'Rolling', verifying: 'Verifying', done: 'Upgraded', missed: 'Missed window' };
return { scheduled: 'Scheduled', delayed: 'Delayed', draining: 'Draining', pulling: 'Pulling', rolling: 'Rolling', verifying: 'Verifying', done: 'Upgraded', missed: 'Missed window' };
}

function versionStateLabel(state) {
Expand Down Expand Up @@ -261,7 +261,7 @@
const agents = versionAgentsInTurnText(st);
const left = typeof st.drainRemainingS === 'number' && st.drainRemainingS > 0 ? versionFormatDuration(st.drainRemainingS) + ' left' : 'deadline reached';
text += (agents ? ' · ' + agents : '') + ' · ' + left;
} else if (st.state === 'missed' && st.reason) {
} else if ((st.state === 'missed' || st.state === 'delayed') && st.reason) {
text += ' · ' + st.reason;
} else if (st.state === 'scheduled' && st.nextWindowAt) {
const at = new Date(st.nextWindowAt);
Expand Down Expand Up @@ -1427,4 +1427,3 @@
}, VERSION_CHECK_INTERVAL_MS);
}
// startVersionCheck() — deferred to _deferredInit

7 changes: 3 additions & 4 deletions src/pkg/dashboard/static/index.html
Original file line number Diff line number Diff line change
Expand Up @@ -404,7 +404,7 @@
.oc-version-manual-pill { white-space: nowrap; padding: 1px 7px; border-radius: var(--r-pill); border: 1px solid var(--border); color: var(--muted); font-size: var(--fs-xs); }
.oc-version-upgrade-note { color: var(--muted); font-size: var(--fs-xs); overflow-wrap: anywhere; }
.oc-version-state-pill { white-space: nowrap; padding: var(--sp-1) var(--sp-3); border-radius: var(--r-pill); border: 1px solid var(--border); color: var(--muted); font-size: var(--fs-xs); font-variant-numeric: tabular-nums; }
.oc-version-state-pill.is-draining { color: var(--amber); border-color: color-mix(in srgb, var(--amber) 40%, transparent); }
.oc-version-state-pill.is-draining, .oc-version-state-pill.is-delayed { color: var(--amber); border-color: color-mix(in srgb, var(--amber) 40%, transparent); }
.oc-version-state-pill.is-missed { color: var(--red, var(--danger)); border-color: color-mix(in srgb, var(--red, var(--danger)) 40%, transparent); }
.oc-version-state-pill.is-done { color: var(--green, var(--success)); }
.oc-version-history { list-style: none; margin: 0; padding: 0; font-size: var(--fs-xs); color: var(--muted); display: flex; flex-direction: column; gap: 2px; }
Expand Down Expand Up @@ -45721,7 +45721,7 @@ <h3 style="color:var(--amber);text-align:center">Configure LiteLLM</h3>
state as a pill — with the drain countdown and the agents holding the
upgrade — and lists the history so a missed window is never silent. */
function versionStateLabels() {
return { scheduled: 'Scheduled', draining: 'Draining', pulling: 'Pulling', rolling: 'Rolling', verifying: 'Verifying', done: 'Upgraded', missed: 'Missed window' };
return { scheduled: 'Scheduled', delayed: 'Delayed', draining: 'Draining', pulling: 'Pulling', rolling: 'Rolling', verifying: 'Verifying', done: 'Upgraded', missed: 'Missed window' };
}

function versionStateLabel(state) {
Expand Down Expand Up @@ -45754,7 +45754,7 @@ <h3 style="color:var(--amber);text-align:center">Configure LiteLLM</h3>
const agents = versionAgentsInTurnText(st);
const left = typeof st.drainRemainingS === 'number' && st.drainRemainingS > 0 ? versionFormatDuration(st.drainRemainingS) + ' left' : 'deadline reached';
text += (agents ? ' · ' + agents : '') + ' · ' + left;
} else if (st.state === 'missed' && st.reason) {
} else if ((st.state === 'missed' || st.state === 'delayed') && st.reason) {
text += ' · ' + st.reason;
} else if (st.state === 'scheduled' && st.nextWindowAt) {
const at = new Date(st.nextWindowAt);
Expand Down Expand Up @@ -46920,7 +46920,6 @@ <h3 style="color:var(--amber);text-align:center">Configure LiteLLM</h3>
}, VERSION_CHECK_INTERVAL_MS);
}
// startVersionCheck() — deferred to _deferredInit

var _eventSource = null;
var _eventSourceReconnectTimer = null;
var _sseLastFrameAtMs = 0;
Expand Down
13 changes: 11 additions & 2 deletions src/pkg/dashboard/upgrade_progress.go
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,11 @@ func buildUpgradeProgress(in upgradeProgressInput) *spoke.UpgradeProgress {
nextWindow = in.Policy.NextWindowAt
history = in.Policy.UpgradeHistory
}
var upgradedAt time.Time
if in.Outcome != nil {
upgradedAt = in.Outcome.CompletedAt
}
history = spoke.UpgradeHistoryForDisplay(history, upgradedAt)

if d := in.Drain; d != nil {
turns := d.Turns
Expand Down Expand Up @@ -114,7 +119,9 @@ func buildUpgradeProgress(in upgradeProgressInput) *spoke.UpgradeProgress {
return p
}

if o := in.Outcome; o != nil && !o.CompletedAt.IsZero() && !in.StartedAt.IsZero() && !o.CompletedAt.Before(in.StartedAt.Add(-time.Minute)) {
last, hasLast := latestClosedAttempt(history)
if o := in.Outcome; o != nil && !o.CompletedAt.IsZero() && !in.StartedAt.IsZero() &&
!o.CompletedAt.Before(in.StartedAt.Add(-time.Minute)) && (!hasLast || !last.EndedAt.After(o.CompletedAt)) {
return &spoke.UpgradeProgress{
State: spoke.UpgradeStateDone,
Target: shortSHADashboard(o.TargetSHA),
Expand All @@ -126,7 +133,7 @@ func buildUpgradeProgress(in upgradeProgressInput) *spoke.UpgradeProgress {
}
}

if last, ok := latestClosedAttempt(history); ok && now.Sub(last.EndedAt) <= upgradeHistoryRecentFor {
if hasLast && now.Sub(last.EndedAt) <= upgradeHistoryRecentFor {
p := &spoke.UpgradeProgress{
State: last.Outcome,
Target: last.Target,
Expand All @@ -140,6 +147,8 @@ func buildUpgradeProgress(in upgradeProgressInput) *spoke.UpgradeProgress {
switch last.Outcome {
case spoke.UpgradeOutcomeDone:
p.Detail = "upgraded to " + orDashDashboard(last.Target) + " " + ago
case spoke.UpgradeOutcomeDelayed:
p.Detail = "delayed: " + last.Reason
default:
if last.Trigger == spoke.UpgradeTriggerWindow {
p.Detail = "missed window " + ago + ": " + last.Reason
Expand Down
34 changes: 32 additions & 2 deletions src/pkg/dashboard/upgrade_progress_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -54,13 +54,43 @@ func TestBuildUpgradeProgressStates(t *testing.T) {
state: spoke.UpgradeStateDone, detail: "upgraded to abc1234 9m ago",
},
{
name: "old outcome ignored, hub history missed window shown",
name: "old outcome ignored, legacy hub window shown as delayed",
in: upgradeProgressInput{StartedAt: now.Add(-10 * time.Minute),
Outcome: &upgradeOutcome{TargetSHA: "abc1234", CompletedAt: now.Add(-48 * time.Hour)},
Policy: &spoke.HeartbeatUpgradePolicy{HubManaged: true, NextWindowAt: next, UpgradeHistory: []spoke.UpgradeAttempt{
{Trigger: spoke.UpgradeTriggerWindow, Outcome: spoke.UpgradeOutcomeMissed, Reason: "spoke still draining",
StartedAt: now.Add(-3 * time.Hour), EndedAt: now.Add(-3 * time.Hour)}}}},
state: spoke.UpgradeStateMissed, detail: "missed window 3h0m ago: spoke still draining",
state: spoke.UpgradeStateDelayed, detail: "delayed: spoke still draining",
},
{
name: "landed outcome clears stale hub delay before next heartbeat",
in: upgradeProgressInput{StartedAt: now.Add(-10 * time.Minute),
Outcome: &upgradeOutcome{TargetSHA: "abc1234", CompletedAt: now.Add(-9 * time.Minute)},
Policy: &spoke.HeartbeatUpgradePolicy{HubManaged: true, UpgradeHistory: []spoke.UpgradeAttempt{
{Trigger: spoke.UpgradeTriggerWindow, Outcome: spoke.UpgradeOutcomeMissed, Reason: "merge debounce",
StartedAt: now.Add(-15 * time.Minute), EndedAt: now.Add(-12 * time.Minute)},
}}},
state: spoke.UpgradeStateDone, detail: "upgraded to abc1234",
},
{
name: "new delay is not hidden by this boot's earlier success",
in: upgradeProgressInput{StartedAt: now.Add(-10 * time.Minute),
Outcome: &upgradeOutcome{TargetSHA: "abc1234", CompletedAt: now.Add(-9 * time.Minute)},
Policy: &spoke.HeartbeatUpgradePolicy{HubManaged: true, UpgradeHistory: []spoke.UpgradeAttempt{
{Trigger: spoke.UpgradeTriggerWindow, Outcome: spoke.UpgradeOutcomeDelayed, Reason: "merge debounce",
StartedAt: now.Add(-2 * time.Minute), EndedAt: now.Add(-time.Minute)},
}}},
state: spoke.UpgradeStateDelayed, detail: "merge debounce",
},
{
name: "genuine hub failure remains missed",
in: upgradeProgressInput{
StartedAt: now.Add(-10 * time.Minute),
Outcome: &upgradeOutcome{TargetSHA: "abc1234", CompletedAt: now.Add(-9 * time.Minute)},
Policy: &spoke.HeartbeatUpgradePolicy{HubManaged: true, UpgradeHistory: []spoke.UpgradeAttempt{
{Trigger: spoke.UpgradeTriggerScheduled, Outcome: spoke.UpgradeOutcomeMissed, Reason: "ErrImagePull", EndedAt: now.Add(-time.Minute)},
}}},
state: spoke.UpgradeStateMissed, detail: "ErrImagePull",
},
{
name: "scheduled with window",
Expand Down
28 changes: 26 additions & 2 deletions src/pkg/dashboard/upgrade_progress_ui_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"encoding/json"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
Expand Down Expand Up @@ -55,10 +56,32 @@ func TestVersionCarriesUpgradeProgressAndHistory(t *testing.T) {
if len(p.AgentsInTurn) != 1 || p.AgentsInTurn[0].Agent != "reviewer" || p.AgentsInTurn[0].ElapsedS != 2280 {
t.Fatalf("agentsInTurn = %+v", p.AgentsInTurn)
}
if len(body.UpgradeHistory) != 1 || body.UpgradeHistory[0].Outcome != spoke.UpgradeOutcomeMissed || body.UpgradeHistory[0].Reason != "drain lapsed" {
if len(body.UpgradeHistory) != 1 || body.UpgradeHistory[0].Outcome != spoke.UpgradeOutcomeDelayed || body.UpgradeHistory[0].Reason != "drain lapsed" {
t.Fatalf("upgradeHistory = %+v", body.UpgradeHistory)
}

// The spoke knows the rollout landed before the hub's next heartbeat.
s.SetUpgradeDrainSource(nil)
outcome, err := json.Marshal(upgradeOutcome{TargetSHA: "abc1234", CompletedAt: time.Now()})
if err != nil {
t.Fatal(err)
}
if err := os.WriteFile(upgradeOutcomePath, outcome, 0o600); err != nil {
t.Fatal(err)
}
rec = httptest.NewRecorder()
s.handleVersion(rec, httptest.NewRequest(http.MethodGet, "/api/version", nil))
body.UpgradeHistory = nil
if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil {
t.Fatal(err)
}
if body.UpgradeProgress == nil || body.UpgradeProgress.State != spoke.UpgradeStateDone || len(body.UpgradeHistory) != 0 {
t.Fatalf("successful upgrade still shows a delay: %+v", body)
}
if err := os.Remove(upgradeOutcomePath); err != nil {
t.Fatal(err)
}

// No policy, no drain, nothing on disk: the keys are absent rather than
// null so older SPA builds see exactly the payload they did before.
s2 := &Server{status: &StatusPayload{}, startedAt: time.Now()}
Expand Down Expand Up @@ -88,6 +111,7 @@ func TestIndexRendersUpgradeProgressPillAndHistory(t *testing.T) {
"st.agentsInTurn",
".oc-version-state-pill {",
".oc-version-state-pill.is-draining",
".oc-version-state-pill.is-delayed",
".oc-version-state-pill.is-missed",
".oc-version-history {",
"row('Progress', statePill",
Expand All @@ -98,7 +122,7 @@ func TestIndexRendersUpgradeProgressPillAndHistory(t *testing.T) {
}
}
labels := jsFunc(t, html, "versionStateLabels")
for _, state := range []string{"scheduled", "draining", "pulling", "rolling", "verifying", "done", "missed"} {
for _, state := range []string{"scheduled", "delayed", "draining", "pulling", "rolling", "verifying", "done", "missed"} {
if !strings.Contains(labels, state+":") {
t.Errorf("versionStateLabels lacks %q", state)
}
Expand Down
18 changes: 12 additions & 6 deletions src/pkg/hub/assets/dashboard.html
Original file line number Diff line number Diff line change
Expand Up @@ -693,7 +693,7 @@
/* Scheduled-upgrade progress pill, history badge and interrupt action
on the hive card's Version cell (#14950). */
.upgrade-progress-pill { display: inline-block; padding: var(--sp-1) var(--sp-3); border-radius: var(--r-pill); font-size: var(--fs-2xs); background: var(--surface); color: var(--muted); border: 1px solid var(--border); white-space: nowrap; font-variant-numeric: tabular-nums; }
.upgrade-progress-pill.is-draining { color: var(--amber); }
.upgrade-progress-pill.is-draining, .upgrade-progress-pill.is-delayed { color: var(--amber); }
.upgrade-progress-pill.is-missed, .upgrade-progress-pill.has-missed { color: var(--red); }
.upgrade-progress-pill.is-done { color: var(--green); }
.upgrade-progress-history { margin-left: var(--sp-2); cursor: help; }
Expand Down Expand Up @@ -9940,8 +9940,8 @@ <h2 style="font-size:1.3rem;color:var(--hub-dashboard-accent);margin:var(--sp-0)
window is never silent. Only the states that mean "something is
happening or went wrong" get a pill; a plain "scheduled" row looks as
it does today (the auto-upgrade line already says when). */
var UPGRADE_PROGRESS_PILL_STATES = {draining: 1, pulling: 1, rolling: 1, verifying: 1, missed: 1};
var UPGRADE_PROGRESS_LABELS = {scheduled: 'Scheduled', draining: 'Draining', pulling: 'Pulling', rolling: 'Rolling', verifying: 'Verifying', done: 'Upgraded', missed: 'Missed window'};
var UPGRADE_PROGRESS_PILL_STATES = {delayed: 1, draining: 1, pulling: 1, rolling: 1, verifying: 1, missed: 1};
var UPGRADE_PROGRESS_LABELS = {scheduled: 'Scheduled', delayed: 'Delayed', draining: 'Draining', pulling: 'Pulling', rolling: 'Rolling', verifying: 'Verifying', done: 'Upgraded', missed: 'Missed window'};
var UPGRADE_HISTORY_MAX_ROWS = 14;

function upgradeProgressLabel(state) {
Expand All @@ -9964,9 +9964,15 @@ <h2 style="font-size:1.3rem;color:var(--hub-dashboard-accent);margin:var(--sp-0)
function renderUpgradeHistory(h) {
var rows = (h && h.upgradeHistory) || [];
if (!rows.length) return '';
var missed = 0;
var succeededAt = rows.reduce(function(latest, a) {
return a.outcome === 'done' ? Math.max(latest, Date.parse(a.endedAt) || 0) : latest;
}, 0);
var missed = rows.filter(function(a) {
return a.outcome === 'missed' && a.trigger !== 'window' &&
(Date.parse(a.endedAt || a.startedAt) || 0) > succeededAt;
}).length;
if (!missed) return '';
var lines = rows.slice(0, UPGRADE_HISTORY_MAX_ROWS).map(function(a) {
if (a.outcome === 'missed') missed++;
var when = a.endedAt && a.endedAt !== '0001-01-01T00:00:00Z' ? a.endedAt : a.startedAt;
var d = when ? new Date(when) : null;
var ts = d && !isNaN(d.getTime()) ? d.toLocaleString([], {month: 'short', day: 'numeric', hour: 'numeric', minute: '2-digit'}) : '?';
Expand Down Expand Up @@ -9998,7 +10004,7 @@ <h2 style="font-size:1.3rem;color:var(--hub-dashboard-accent);margin:var(--sp-0)
var left = typeof st.drainRemainingS === 'number' && st.drainRemainingS > 0 ? formatUpgradeElapsed(st.drainRemainingS * 1000) + ' left' : 'deadline reached';
extra = ' · ' + left;
if (agents) tipExtra = '\n' + agents;
} else if (st.state === 'missed' && st.reason) {
} else if ((st.state === 'missed' || st.state === 'delayed') && st.reason) {
tipExtra = '\n' + st.reason;
}
var tip = (st.detail || label) + tipExtra;
Expand Down
30 changes: 30 additions & 0 deletions src/pkg/hub/spoke/upgrade_history.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
package spoke

import "time"

// UpgradeHistoryForDisplay keeps the audit history intact while removing
// resolved schedule holds. Older hubs recorded those holds as missed windows.
// upgradedAt lets a spoke clear a hold before its next successful heartbeat.
func UpgradeHistoryForDisplay(history []UpgradeAttempt, upgradedAt time.Time) []UpgradeAttempt {
for _, a := range history {
if a.Outcome == UpgradeOutcomeDone && a.EndedAt.After(upgradedAt) {
upgradedAt = a.EndedAt
}
}
var rows []UpgradeAttempt
for _, a := range history {
if a.Outcome == UpgradeOutcomeDelayed ||
(a.Trigger == UpgradeTriggerWindow && a.Outcome == UpgradeOutcomeMissed) {
at := a.EndedAt
if at.IsZero() {
at = a.StartedAt
}
if !upgradedAt.IsZero() && !at.After(upgradedAt) {
continue
}
a.Outcome = UpgradeOutcomeDelayed
}
rows = append(rows, a)
}
return rows
}
Loading
Loading