From 25264a4cc2dfd1bcead9af76f1807e7f680847a1 Mon Sep 17 00:00:00 2001 From: Andrew Anderson Date: Wed, 28 Oct 2026 10:26:57 -0400 Subject: [PATCH 1/3] =?UTF-8?q?=E2=9C=A8=20dashboards:=20upgrade=20progres?= =?UTF-8?q?s=20pill,=20history=20and=20Upgrade=20now=20(interrupt=20turns)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Renders the scheduled-upgrade state machine from the previous change on both surfaces and adds the owner action that resolves a stalled drain. Hub hive card (dashboard.html): - A pill for draining/pulling/rolling/verifying/missed with the drain countdown and the agents holding the upgrade ("Draining · 2/5 agents in turn: reviewer 38m, scanner 12m · 14m left"); a quiet scheduled row renders exactly as before. - A "history" badge whose tooltip lists the hub's recent attempts; red with a miss count when any window was missed. - "Upgrade now (interrupt turns)" while draining or missed. Same POST as Upgrade now with ?interrupt=1, same owner/admin check, hiveConfirm (no native dialog), audit-logged with interrupt_turns=true and recorded on the hive timeline. Hub → spoke: the interrupt is held in memory per hive (2h TTL, cleared when the upgrade closes, dropped on hub restart so the spoke falls back to draining) and rides on the heartbeat beside upgrade_to as upgrade_interrupt. The spoke's drain decision reads it and proceeds immediately instead of waiting out the drain window. Spoke operator dashboard: /api/version carries upgradeProgress and upgradeHistory; the version menu gains a Progress row with the pill and an Upgrade history list, and the header chip shows Draining/Missed window while either state holds. Styled with .oc-version-* classes. Tests: interrupt store arm/pending/TTL/clear and target matching, request parsing, heartbeat response carries the flag only with upgrade_to, spoke dispatch exposes it to the upgrade callback, /api/version payload fields, and markup tests for both dashboards. Refs #14950 Signed-off-by: Andrew Anderson --- changelog.d/added-upgrade-progress-ui.md | 1 + src/cmd/hive/main.go | 7 + src/docs/api-reference.md | 52 ++++---- src/docs/operator-reference.md | 4 + src/pkg/dashboard/api_version.go | 13 ++ .../dashboard/sidebar_version_panel_test.go | 7 + src/pkg/dashboard/static/index.html | 92 +++++++++++++- src/pkg/dashboard/upgrade_progress_ui_test.go | 111 ++++++++++++++++ .../dashboard/version_provenance_ui_test.go | 1 + src/pkg/hub/assets/dashboard.html | 120 ++++++++++++++++++ src/pkg/hub/heartbeat.go | 4 + src/pkg/hub/saas_hives_api.go | 14 +- src/pkg/hub/saas_upgrade.go | 22 +++- src/pkg/hub/server.go | 11 ++ src/pkg/hub/spoke/heartbeat.go | 27 +++- ...artbeat_reconnect_upgrade_coverage_test.go | 28 ++-- src/pkg/hub/timeline.go | 1 + src/pkg/hub/upgrade_interrupt.go | 93 ++++++++++++++ src/pkg/hub/upgrade_interrupt_test.go | 116 +++++++++++++++++ src/pkg/hub/upgrade_progress_markup_test.go | 59 +++++++++ 20 files changed, 735 insertions(+), 48 deletions(-) create mode 100644 changelog.d/added-upgrade-progress-ui.md create mode 100644 src/pkg/dashboard/upgrade_progress_ui_test.go create mode 100644 src/pkg/hub/upgrade_interrupt.go create mode 100644 src/pkg/hub/upgrade_interrupt_test.go create mode 100644 src/pkg/hub/upgrade_progress_markup_test.go diff --git a/changelog.d/added-upgrade-progress-ui.md b/changelog.d/added-upgrade-progress-ui.md new file mode 100644 index 0000000000..1c0923f0af --- /dev/null +++ b/changelog.d/added-upgrade-progress-ui.md @@ -0,0 +1 @@ +Both dashboards now show scheduled-upgrade progress. The hub hive card and the spoke's version menu render the upgrade state as a pill — `Draining · 2/5 agents in turn: reviewer 38m, scanner 12m · 14m left`, `Missed window · drain lapsed` — and list the hub's recent upgrade attempts so a missed window is visible on both sides. `/api/version` carries `upgradeProgress` and `upgradeHistory`. Owners get an **Upgrade now (interrupt turns)** action on the hub card while a hive is draining or missed; it is the normal upgrade request with `?interrupt=1`, audit-logged, delivered on the next heartbeat as `upgrade_interrupt`, and makes the spoke skip its drain and restart immediately. diff --git a/src/cmd/hive/main.go b/src/cmd/hive/main.go index aa52394fee..dd04607225 100644 --- a/src/cmd/hive/main.go +++ b/src/cmd/hive/main.go @@ -5178,6 +5178,13 @@ func (b *boot) bootHeartbeatWith(deps bootHeartbeatDeps) { } ignoredWedged := wedgedKickAgents(wedged) drainTimeout := b.cfg.Hub.EffectiveUpgradeDrainTimeout() + if spoke.UpgradeInterruptRequested() && drainTimeout > 0 && len(inFlight) > 0 { + // The owner chose "Upgrade now (interrupt turns)" on the + // hub: cut the drain rather than wait for these turns. + b.logger.Warn("self-upgrade: interrupting in-flight agent turns on the owner's request", + "target", targetSHA, "in_flight", inFlight) + drainTimeout = 0 + } dec := selfUpgradeDrain.decide(targetSHA, inFlight, drainTimeout, selfUpgradeDrainNow()) selfUpgradeDrain.setKickTimes(agentKickTimes(statuses, inFlight)) switch { diff --git a/src/docs/api-reference.md b/src/docs/api-reference.md index 1d54983c56..7db055579d 100644 --- a/src/docs/api-reference.md +++ b/src/docs/api-reference.md @@ -724,31 +724,31 @@ always resolved server-side from the validated token. | `GET` | `/api/hub/clusters` | Hub auth | List Clusters | `pkg/hub/saas.go:528` | | `GET` | `/api/hub/image-pulls` | Hub auth | Per-Release Image Pull Series | `pkg/hub/saas.go:377` | | `GET` | `/api/reach` | Hub admin | PR Reach Report (?pr=NNN or ?recent=K) | `pkg/hub/saas.go:519` | -| `GET` | `/fleet` | Hub handler-specific | My-Hives Fleet Page (static; data via `/api/saas/my-hives`) | `pkg/hub/server.go:1756` | -| `GET` | `/my-hives` | Hub handler-specific | 301 redirect to `/fleet` (query preserved) | `pkg/hub/server.go:1757` | -| `POST` | `/api/heartbeat` | Hub handler-specific | Heartbeat | `pkg/hub/server.go:1697` | -| `POST` | `/api/task-status` | Hub handler-specific | Task Status | `pkg/hub/server.go:1698` | -| `GET` | `/api/registry` | Hub handler-specific | Registry | `pkg/hub/server.go:1706` | -| `GET` | `/api/hub/leaderboard` | Hub handler-specific | Leaderboard | `pkg/hub/server.go:1707` | -| `GET` | `/api/hub/stats` | Hub handler-specific | Stats | `pkg/hub/server.go:1708` | -| `GET` | `/api/fleet-stats` | Hub handler-specific | Fleet Stats | `pkg/hub/server.go:1709` | +| `GET` | `/fleet` | Hub handler-specific | My-Hives Fleet Page (static; data via `/api/saas/my-hives`) | `pkg/hub/server.go:1761` | +| `GET` | `/my-hives` | Hub handler-specific | 301 redirect to `/fleet` (query preserved) | `pkg/hub/server.go:1762` | +| `POST` | `/api/heartbeat` | Hub handler-specific | Heartbeat | `pkg/hub/server.go:1702` | +| `POST` | `/api/task-status` | Hub handler-specific | Task Status | `pkg/hub/server.go:1703` | +| `GET` | `/api/registry` | Hub handler-specific | Registry | `pkg/hub/server.go:1711` | +| `GET` | `/api/hub/leaderboard` | Hub handler-specific | Leaderboard | `pkg/hub/server.go:1712` | +| `GET` | `/api/hub/stats` | Hub handler-specific | Stats | `pkg/hub/server.go:1713` | +| `GET` | `/api/fleet-stats` | Hub handler-specific | Fleet Stats | `pkg/hub/server.go:1714` | | `GET` | `/api/hub/evidence` | Hub admin | Read-only cross-hive evidence drill-down from each spoke's stored heartbeat `evidence` summary ([#12116](https://github.com/hivecommons/hive/issues/12116)); non-admins get `403`. Returns `spokes`, `reporting`, pooled `calibration` (bucket 0–5 → `{n, bad, rate, lo, hi}`; the hub view folds the six buckets into four reviewer-confidence ranges), `findings[]` (the plain-language *What this says* sentences, generated from the same fields; [#12456](https://github.com/hivecommons/hive/issues/12456)), `by_acmm_level` (`level`, `hives`, `merged_90d`, `revert_rate`, and `time_to_merge` — `median_of_hive_medians_s`, `hives`, `n` — the median of the level's per-hive median time-to-merge, omitted when no hive at the level reports one; [#12464](https://github.com/hivecommons/hive/issues/12464)), `family_false_approve_rate` (`same`/`cross`/`mixed`, hives classed by their reviewer vs author family sets), fleet `record_coverage` (merged PRs with an evidence record over `prs_merged_90d`), fleet panels ([#12370](https://github.com/hivecommons/hive/issues/12370)) `outcomes_by_author_kind[]` (`kind`, `hives`, `n`, `merged`, `reverted`, `merge_rate`, `revert_rate`), `write_counters[]` (`tier`, `hives`, `attempted`, `blocked`, `block_rate`) and `shadow_gate` (`review_gate` / `intent` → `held` / `passed` outcomes with `revert_rate`), each omitted while no spoke reports it and repeated per `by_acmm_level[]` row, and per hive `hives[]` with `acmm_level`, `prs_merged_90d`, `record_coverage`, `family_mix`, false-approve / false-block / revert rates, families, a calibration series the raw `counts` behind them (including `by_author_kind`) so the drill-down can re-pool a hive / ACMM-level selection client-side, and `criteria[]`: the per-axis criteria table derived hub-side from the quadrant criteria (`axis`, `label`, `scored`, `criteria[]` with `name`, `label`, `status` `met`/`unmet`/`not_reported`, `display`/`score` when measured, `reason` otherwise; offline, evidence-less or unscored hives are `not_reported`, never `unmet`; [#12463](https://github.com/hivecommons/hive/issues/12463)). Every rate is `{n, rate, lo, hi}` with a 95% Wilson interval; `rate`/`lo`/`hi` are null when `n` < 10. Window selection ([RFC #12062](https://github.com/hivecommons/hive/issues/12062)): `?hours=24|168|720` pools each hive's matching heartbeat `windows` block instead of its 90-day block (any other or absent `hours` → 90 days); `?from=YYYY-MM-DD|RFC3339[&to=…]` (date-only `to` is inclusive, `to` defaults to now, `to` ≥ `from`, span ≤ 366 days, else `400`) is served from the smallest reported window that reaches back to `from` and flagged `window.approximate`. The response carries `window` (`hours`, `key`, `label`, `from`, `to`, `approximate`, `coverage_available` — `record_coverage` is only computed for the 90-day window because `prs_merged_90d` is a 90-day count), `window_unreported` and per-hive `window_unreported` for hives whose spoke predates per-window reporting. `?hive=[,…]` (repeatable) and `?level=` restrict the registry server-side to those hives / that ACMM level before pooling. Hub admins open it from the Fleet quadrant card or its **🔬 Evidence** button ([#12117](https://github.com/hivecommons/hive/issues/12117)), pick a window (24h / 7d / 30d / 90d / custom), filter it by hive and ACMM level and read each section's ⓘ explainer; the per-hive author-kind revert counts behind the *Revert rate by author kind* panel are `hives[].counts.by_author_kind` ([#12326](https://github.com/hivecommons/hive/issues/12326)). Never consulted by merge decisions | `pkg/hub/evidence.go` | | `GET` | `/api/hub/evidence/report` | Hub admin | Plain-language report of the hub evidence view ([#12456](https://github.com/hivecommons/hive/issues/12456)), rendered server-side from the same `EvidenceView` as `/api/hub/evidence` so every figure reconciles with the modal; non-admins get `403`. Same `hours=` / `from=` / `to=` window and `hive=` / `level=` filter parameters. `format=md` (default, `text/markdown`) or `format=html` (standalone page, inline CSS, no scripts or external assets, hive names escaped); any other value → `400`. `download=1` sets `Content-Disposition: attachment; filename="hive-evidence---.md|html"`, otherwise inline. Sections: Summary (the generated findings), What we measured and why, The numbers, How to read these, What is not reported yet and why, Grade per section, footer link to `review-evidence.md`. `Cache-Control: no-store`. Read-only; never consulted by merge decisions | `pkg/hub/evidence_report.go` | -| `GET` | `/api/hub/version` | Hub handler-specific | Hub Version | `pkg/hub/server.go:1714` | -| `DELETE` | `/api/hub/registry/{id}` | Hub handler-specific | Registry Delete | `pkg/hub/server.go:1724` | -| `POST` | `/api/contribute/register` | Hub handler-specific | Contribute Proxy | `pkg/hub/server.go:1725` | -| `GET` | `/api/contribute/status` | Hub handler-specific | Contribute Status | `pkg/hub/server.go:1726` | -| `GET` | `/api/contribute/ws` | Hub handler-specific | Contribute WSProxy | `pkg/hub/server.go:1727` | -| `POST` | `/api/github/webhook` | Hub handler-specific | GitHub Webhook | `pkg/hub/server.go:1728` | -| `GET` | `/gh-setup` | Hub handler-specific | GitHub App Setup Router | `pkg/hub/server.go:1729` | -| `GET` | `/learn` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1730` | -| `GET` | `/get-started` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1731` | -| `GET` | `/api/docs` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1734` | -| `GET` | `/api/reading-list` | Hub handler-specific | Reading List | `pkg/hub/server.go:1737` | -| `GET` | `/reading` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1738` | -| `GET` | `/cncf-reference-architecture` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1745` | -| `GET` | `/cncf-reference-architecture/console` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1746` | -| `GET` | `/cncf-reference-architecture/bluefin` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1747` | -| `GET` | `/{$}` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1764` | -| `GET` | `/og-card.png` | Hub handler-specific | OGCard | `pkg/hub/server.go:1769` | -| `GET` | `/` | Public | Static asset fallback (`http.FileServerFS` over the embedded `static/` tree) for any path no other route claims | `pkg/hub/server.go:1771` | +| `GET` | `/api/hub/version` | Hub handler-specific | Hub Version | `pkg/hub/server.go:1719` | +| `DELETE` | `/api/hub/registry/{id}` | Hub handler-specific | Registry Delete | `pkg/hub/server.go:1729` | +| `POST` | `/api/contribute/register` | Hub handler-specific | Contribute Proxy | `pkg/hub/server.go:1730` | +| `GET` | `/api/contribute/status` | Hub handler-specific | Contribute Status | `pkg/hub/server.go:1731` | +| `GET` | `/api/contribute/ws` | Hub handler-specific | Contribute WSProxy | `pkg/hub/server.go:1732` | +| `POST` | `/api/github/webhook` | Hub handler-specific | GitHub Webhook | `pkg/hub/server.go:1733` | +| `GET` | `/gh-setup` | Hub handler-specific | GitHub App Setup Router | `pkg/hub/server.go:1734` | +| `GET` | `/learn` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1735` | +| `GET` | `/get-started` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1736` | +| `GET` | `/api/docs` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1739` | +| `GET` | `/api/reading-list` | Hub handler-specific | Reading List | `pkg/hub/server.go:1742` | +| `GET` | `/reading` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1743` | +| `GET` | `/cncf-reference-architecture` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1750` | +| `GET` | `/cncf-reference-architecture/console` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1751` | +| `GET` | `/cncf-reference-architecture/bluefin` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1752` | +| `GET` | `/{$}` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1769` | +| `GET` | `/og-card.png` | Hub handler-specific | OGCard | `pkg/hub/server.go:1774` | +| `GET` | `/` | Public | Static asset fallback (`http.FileServerFS` over the embedded `static/` tree) for any path no other route claims | `pkg/hub/server.go:1776` | diff --git a/src/docs/operator-reference.md b/src/docs/operator-reference.md index ba16a36e46..3bf1315a42 100644 --- a/src/docs/operator-reference.md +++ b/src/docs/operator-reference.md @@ -957,6 +957,10 @@ A hub-instructed upgrade is a short state machine, and both dashboards expose it **History.** The hub persists the last `HIVE_HUB_UPGRADE_HISTORY_MAX` (default 14) attempts and held windows per hive under `/data/saas/upgrade-history/.json`, so they survive a hub restart. Each row carries `target`, `from`, `trigger` (`scheduled` / `instant` / `manual` / `observed` / `window`), `startedAt`, `endedAt`, `outcome` (`done` / `missed`) and `reason`. A daily window the gate held records exactly one `window` row per calendar day (ET), updated in place if the hold reason changes; a window that fired records the attempt instead. Deleting a hive drops its history. +**Where it shows.** On the hub, the hive card's Version cell gains a pill for the `draining`, `pulling`, `rolling`, `verifying` and `missed` states — `Draining · 2/5 agents in turn: reviewer 38m, scanner 12m · 14m left` — and a `history` badge whose tooltip lists the recent attempts (it turns red and counts the misses when any are present); a quiet `scheduled` row renders as before. On the spoke, `/api/version` carries the same `upgradeProgress` and `upgradeHistory`, and the operator dashboard's version menu shows a **Progress** row with the pill and an **Upgrade history** list; the header chip adds a `Draining 14m` / `Missed window` pill while either state holds. + +**Upgrade now (interrupt turns).** While a hive is `draining` or `missed`, its owner sees an *Upgrade now (interrupt turns)* action on the hub card. It is the ordinary upgrade request — `POST /api/saas/hives/{id}/upgrade?interrupt=1`, same owner/admin check as every other hub mutation — with an interrupt flag the hub audit-logs (`audit: hosted hive upgrade requested … interrupt_turns=true`), records on the hive timeline, and hands to the spoke on the next heartbeat (`upgrade_interrupt: true` beside `upgrade_to`). The spoke then skips the drain and restarts at once; the agents listed as in-turn are cut and their owners must re-run those turns. The request is held in hub memory for 2 hours and cleared when the upgrade closes, so a stale interrupt never cuts a later upgrade's turns; a hub restart drops it (the spoke falls back to draining). While it is armed the card shows `interrupt requested` instead of the action. + ## Governor cadence and budget - Agent cadences are evaluated from persisted state: the last-kick map lives in `/data/hive-state.json` and is honored across pod restarts — a Deployment roll does **not** re-kick every cadenced agent at boot ([#3817](https://github.com/hivecommons/hive/pull/3817)). A fresh install (no persisted state) still kicks every cadenced agent on the first eval. There is no global default interval; a zero/absent interval means the agent is never cadence-kicked. diff --git a/src/pkg/dashboard/api_version.go b/src/pkg/dashboard/api_version.go index 04aea8d9e2..bffdd673db 100644 --- a/src/pkg/dashboard/api_version.go +++ b/src/pkg/dashboard/api_version.go @@ -84,6 +84,19 @@ func (s *Server) handleVersion(w http.ResponseWriter, r *http.Request) { resp["channel"] = channel } } + // Scheduled-upgrade progress (#14950): the same derived state the hub + // card shows, so the version menu can say "draining — 2 agents in turn, + // 14m left" instead of just "upgrading", plus the hub-kept history so a + // missed window is visible from the spoke too. + s.statusMu.RLock() + curStatus := s.status + s.statusMu.RUnlock() + if progress := s.upgradeProgress(curStatus, time.Now()); progress != nil { + resp["upgradeProgress"] = progress + } + if policy != nil && len(policy.UpgradeHistory) > 0 { + resp["upgradeHistory"] = policy.UpgradeHistory + } if cacheAge > dashboardVersionTipCacheTTL || cached == "" { if latest, err := s.fetchLatestRemoteHash(); err == nil && latest != "" { diff --git a/src/pkg/dashboard/sidebar_version_panel_test.go b/src/pkg/dashboard/sidebar_version_panel_test.go index 9dd14ceec8..6ac0ccab43 100644 --- a/src/pkg/dashboard/sidebar_version_panel_test.go +++ b/src/pkg/dashboard/sidebar_version_panel_test.go @@ -117,6 +117,13 @@ const window = {}; jsFunc(t, html, "versionCompareURL") + "\n" + jsFunc(t, html, "versionStatusText") + "\n" + jsFunc(t, html, "versionLastUpgradeText") + "\n" + + jsFunc(t, html, "versionStateLabels") + "\n" + + jsFunc(t, html, "versionStateLabel") + "\n" + + jsFunc(t, html, "versionFormatDuration") + "\n" + + jsFunc(t, html, "versionAgentsInTurnText") + "\n" + + jsFunc(t, html, "versionUpgradeStateText") + "\n" + + jsFunc(t, html, "versionUpgradeStatePillHTML") + "\n" + + jsFunc(t, html, "versionUpgradeHistoryHTML") + "\n" + jsFunc(t, html, "versionShortSHA") + "\n" + jsFunc(t, html, "versionSameCommit") + "\n" + jsFunc(t, html, "versionDashHTML") + "\n" + diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html index fbcf95a6c7..7b7137a169 100644 --- a/src/pkg/dashboard/static/index.html +++ b/src/pkg/dashboard/static/index.html @@ -403,6 +403,15 @@ .oc-version-channel { flex: 0 1 auto; min-width: 0; max-width: 150px; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; padding: 1px 7px; border-radius: var(--r-pill); color: var(--accent); background: color-mix(in srgb, var(--accent) 14%, transparent); border: 1px solid color-mix(in srgb, var(--accent) 36%, transparent); font-weight: 700; } .oc-version-manual-pill { white-space: nowrap; padding: 1px 7px; border-radius: var(--r-pill); border: 1px solid var(--border); color: var(--muted); font-size: var(--fs-xs); } .oc-version-upgrade-note { color: var(--muted); font-size: var(--fs-xs); overflow-wrap: anywhere; } + .oc-version-state-pill { white-space: nowrap; padding: 1px 7px; border-radius: var(--r-pill); border: 1px solid var(--border); color: var(--muted); font-size: var(--fs-xs); font-variant-numeric: tabular-nums; } + .oc-version-state-pill.is-draining { color: var(--amber); border-color: color-mix(in srgb, var(--amber) 40%, transparent); } + .oc-version-state-pill.is-missed { color: var(--red, var(--danger)); border-color: color-mix(in srgb, var(--red, var(--danger)) 40%, transparent); } + .oc-version-state-pill.is-done { color: var(--green, var(--success)); } + .oc-version-history { list-style: none; margin: 0; padding: 0; font-size: var(--fs-xs); color: var(--muted); display: flex; flex-direction: column; gap: 2px; } + .oc-version-history li { display: flex; gap: var(--sp-2); min-width: 0; } + .oc-version-history li.is-missed { color: var(--red, var(--danger)); } + .oc-version-history time { flex: 0 0 auto; font-variant-numeric: tabular-nums; } + .oc-version-history span { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } .oc-version-chip-chevron { flex: 0 0 auto; width: 0.8rem; font-size: var(--fs-sm); font-weight: 700; color: var(--muted); line-height: 1; text-align: center; margin-left: auto; transition: transform 0.15s ease; } .oc-version-chip[aria-expanded="false"] .oc-version-chip-chevron { transform: rotate(-90deg); } .oc-version-update-dot { flex: 0 0 auto; color: var(--amber); font-weight: 900; } @@ -39378,6 +39387,79 @@

Configure LiteLLM

return 'Version current'; } + /* Scheduled-upgrade progress (#14950). /api/version carries the state + the hub and spoke derive together (upgradeProgress: scheduled → + draining → pulling → rolling → verifying → done | missed) and the + hub's history of recent attempts (upgradeHistory). The menu shows the + state as a pill — with the drain countdown and the agents holding the + upgrade — and lists the history so a missed window is never silent. */ + function versionStateLabels() { + return { scheduled: 'Scheduled', draining: 'Draining', pulling: 'Pulling', rolling: 'Rolling', verifying: 'Verifying', done: 'Upgraded', missed: 'Missed window' }; + } + + function versionStateLabel(state) { + return versionStateLabels()[state] || (state ? String(state).charAt(0).toUpperCase() + String(state).slice(1) : ''); + } + + function versionFormatDuration(sec) { + sec = Math.floor(Number(sec) || 0); + if (sec < 0) return ''; + if (sec < 60) return sec + 's'; + const min = Math.floor(sec / 60); + if (min < 60) return min + 'm'; + return Math.floor(min / 60) + 'h' + (min % 60) + 'm'; + } + + function versionAgentsInTurnText(st) { + const turns = (st && st.agentsInTurn) || []; + if (!turns.length) return ''; + const parts = turns.map(function(t) { + const el = versionFormatDuration(t.elapsedS || 0); + return (t.agent || '?') + (el ? ' ' + el : ''); + }); + return turns.length + (st.agentsTotal ? '/' + st.agentsTotal : '') + ' agent' + (turns.length === 1 && !st.agentsTotal ? '' : 's') + ' in turn: ' + parts.join(', '); + } + + function versionUpgradeStateText(st) { + if (!st || !st.state) return ''; + let text = versionStateLabel(st.state); + if (st.state === 'draining') { + const agents = versionAgentsInTurnText(st); + const left = typeof st.drainRemainingS === 'number' && st.drainRemainingS > 0 ? versionFormatDuration(st.drainRemainingS) + ' left' : 'deadline reached'; + text += (agents ? ' · ' + agents : '') + ' · ' + left; + } else if (st.state === 'missed' && st.reason) { + text += ' · ' + st.reason; + } else if (st.state === 'scheduled' && st.nextWindowAt) { + const at = new Date(st.nextWindowAt); + if (!isNaN(at.getTime())) text += ' · next window ' + formatDashboardDateTime(at); + } else if (st.state === 'done' && st.target) { + text += ' → ' + versionShortSHA(st.target); + } + return text; + } + + function versionUpgradeStatePillHTML(st) { + const text = versionUpgradeStateText(st); + if (!text) return ''; + return '' + escapeHtml(text) + ''; + } + + function versionUpgradeHistoryHTML(v) { + const rows = (v && v.upgradeHistory) || []; + if (!rows.length) return ''; + const items = rows.slice(0, 14).map(function(a) { + const when = a.endedAt && a.endedAt !== '0001-01-01T00:00:00Z' ? a.endedAt : a.startedAt; + const d = when ? new Date(when) : null; + const ts = d && !isNaN(d.getTime()) ? formatDashboardDateTime(d) : '?'; + const what = a.outcome ? versionStateLabel(a.outcome) : 'In flight'; + const to = a.target ? ' → ' + versionShortSHA(a.target) : ''; + const why = a.reason ? ' — ' + a.reason : ''; + const text = what + to + (a.trigger ? ' (' + a.trigger + ')' : '') + why; + return '
  • ' + escapeHtml(text) + '
  • '; + }); + return '
      ' + items.join('') + '
    '; + } + function versionLastUpgradeText(v) { const rs = v && v.releaseStatus; const at = rs && rs.attempt ? rs.attempt : null; @@ -39687,7 +39769,11 @@

    Configure LiteLLM

    } } if (au.heldReason) html += row('Held', value(au.heldReason, 'The scheduled update window opened but the hub held the upgrade' + (au.heldAt ? ' at ' + au.heldAt : '') + ': ' + au.heldReason), au.heldReason); + const statePill = versionUpgradeStatePillHTML(v && v.upgradeProgress); + if (statePill) html += row('Progress', statePill, (v.upgradeProgress && v.upgradeProgress.detail) || ''); if (lastUpgrade) html += row('History', value(lastUpgrade.text, lastUpgrade.title || lastUpgrade.text), lastUpgrade.title || lastUpgrade.text); + const historyHTML = versionUpgradeHistoryHTML(v); + if (historyHTML) html += '
    Upgrade history
    ' + historyHTML; if (compareURL && (v && (v.behind || (v.commitsBehind != null && v.commitsBehind > 0)))) { html += 'Compare with target ↗'; } @@ -39722,7 +39808,11 @@

    Configure LiteLLM

    const progress = ctx.upgradeProgress || null; const updateDot = ctx.hasUpgrade ? '↑' : ''; const upgrading = progress && !progress.completedAt; - const progressGlyph = progress && progress.completedAt ? '✓' : ''; + const serverState = v.upgradeProgress || null; + const stateChip = !upgrading && serverState && (serverState.state === 'draining' || serverState.state === 'missed') + ? '' + escapeHtml(versionStateLabel(serverState.state)) + (serverState.state === 'draining' && serverState.drainRemainingS > 0 ? ' ' + escapeHtml(versionFormatDuration(serverState.drainRemainingS)) : '') + '' + : ''; + const progressGlyph = (progress && progress.completedAt ? '✓' : '') + stateChip; if (chip.classList && chip.classList.toggle) chip.classList.toggle('is-upgrading', !!upgrading); const html = upgrading ? versionNavbarUpgradeHTML(progress, v) + '' : (v.release ? 'v' + escapeHtml(String(v.release).replace(/^v/i, '')) + '' : '') + diff --git a/src/pkg/dashboard/upgrade_progress_ui_test.go b/src/pkg/dashboard/upgrade_progress_ui_test.go new file mode 100644 index 0000000000..40250cc65a --- /dev/null +++ b/src/pkg/dashboard/upgrade_progress_ui_test.go @@ -0,0 +1,111 @@ +package dashboard + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/hub/spoke" +) + +// /api/version — what the SPA's version menu actually reads — carries the +// same derived upgrade state as /api/status plus the hub's attempt history, +// so the spoke dashboard can show "draining · 1 agent in turn · 1m left" and +// the missed-window list without a second fetch (#14950). +func TestVersionCarriesUpgradeProgressAndHistory(t *testing.T) { + origMarker, origOutcome := upgradeMarkerPath, upgradeOutcomePath + dir := t.TempDir() + upgradeMarkerPath = filepath.Join(dir, "upgrade-requested") + upgradeOutcomePath = filepath.Join(dir, "last-upgrade-outcome") + t.Cleanup(func() { upgradeMarkerPath, upgradeOutcomePath = origMarker, origOutcome }) + + s := &Server{status: &StatusPayload{Agents: []FrontendAgent{{Name: "reviewer"}, {Name: "scanner"}}}, startedAt: time.Now()} + started := time.Date(2026, 10, 26, 17, 0, 0, 0, time.UTC) + s.SetHubUpgradePolicy(&spoke.HeartbeatUpgradePolicy{ + HubManaged: true, + NextWindowAt: "2026-10-29T17:00:00Z", + UpgradeHistory: []spoke.UpgradeAttempt{ + {Target: "abc1234", Trigger: spoke.UpgradeTriggerWindow, StartedAt: started, EndedAt: started, Outcome: spoke.UpgradeOutcomeMissed, Reason: "drain lapsed"}, + }, + }) + s.SetUpgradeDrainSource(func(now time.Time) *spoke.UpgradeDrainReport { + return &spoke.UpgradeDrainReport{Target: "abc1234", RemainingS: 60, InFlight: []string{"reviewer"}, Turns: []spoke.UpgradeTurn{{Agent: "reviewer", ElapsedS: 2280}}} + }) + + rec := httptest.NewRecorder() + s.handleVersion(rec, httptest.NewRequest(http.MethodGet, "/api/version", nil)) + if rec.Code != http.StatusOK { + t.Fatalf("status %d: %s", rec.Code, rec.Body.String()) + } + var body struct { + UpgradeProgress *spoke.UpgradeProgress `json:"upgradeProgress"` + UpgradeHistory []spoke.UpgradeAttempt `json:"upgradeHistory"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil { + t.Fatal(err) + } + p := body.UpgradeProgress + if p == nil || p.State != spoke.UpgradeStateDraining || p.DrainRemainingS != 60 || p.AgentsTotal != 2 { + t.Fatalf("upgradeProgress = %+v, want draining/60s/2 agents", p) + } + if len(p.AgentsInTurn) != 1 || p.AgentsInTurn[0].Agent != "reviewer" || p.AgentsInTurn[0].ElapsedS != 2280 { + t.Fatalf("agentsInTurn = %+v", p.AgentsInTurn) + } + if len(body.UpgradeHistory) != 1 || body.UpgradeHistory[0].Outcome != spoke.UpgradeOutcomeMissed || body.UpgradeHistory[0].Reason != "drain lapsed" { + t.Fatalf("upgradeHistory = %+v", body.UpgradeHistory) + } + + // No policy, no drain, nothing on disk: the keys are absent rather than + // null so older SPA builds see exactly the payload they did before. + s2 := &Server{status: &StatusPayload{}, startedAt: time.Now()} + rec = httptest.NewRecorder() + s2.handleVersion(rec, httptest.NewRequest(http.MethodGet, "/api/version", nil)) + var raw map[string]json.RawMessage + if err := json.Unmarshal(rec.Body.Bytes(), &raw); err != nil { + t.Fatal(err) + } + if _, ok := raw["upgradeHistory"]; ok { + t.Errorf("upgradeHistory present without a policy: %s", raw["upgradeHistory"]) + } +} + +// The SPA renders the state pill and history list from those fields, with +// a label for every state the server can emit, and never through a native +// dialog or an inline style. +func TestIndexRendersUpgradeProgressPillAndHistory(t *testing.T) { + html := indexHTML(t) + for _, want := range []string{ + "function versionUpgradeStatePillHTML(", + "function versionUpgradeHistoryHTML(", + "function versionAgentsInTurnText(", + "v.upgradeProgress", + "v.upgradeHistory", + "st.drainRemainingS", + "st.agentsInTurn", + ".oc-version-state-pill {", + ".oc-version-state-pill.is-draining", + ".oc-version-state-pill.is-missed", + ".oc-version-history {", + "row('Progress', statePill", + ">Upgrade history", + } { + if !strings.Contains(html, want) { + t.Errorf("index.html missing %q", want) + } + } + labels := jsFunc(t, html, "versionStateLabels") + for _, state := range []string{"scheduled", "draining", "pulling", "rolling", "verifying", "done", "missed"} { + if !strings.Contains(labels, state+":") { + t.Errorf("versionStateLabels lacks %q", state) + } + } + fnStart := strings.Index(html, "function versionUpgradeHistoryHTML(") + fn := html[fnStart : fnStart+strings.Index(html[fnStart:], "\n }\n")] + if strings.Contains(fn, "style=") { + t.Errorf("versionUpgradeHistoryHTML must use the .oc-version-history classes, not inline styles:\n%s", fn) + } +} diff --git a/src/pkg/dashboard/version_provenance_ui_test.go b/src/pkg/dashboard/version_provenance_ui_test.go index 6bcc1cf1a2..7d0f79bd60 100644 --- a/src/pkg/dashboard/version_provenance_ui_test.go +++ b/src/pkg/dashboard/version_provenance_ui_test.go @@ -18,6 +18,7 @@ func TestVersionProvenanceRendering(t *testing.T) { var source strings.Builder for _, name := range []string{ "escapeHtml", "versionDeliveryLabel", "versionTrackingLabel", "versionTrackingTooltip", "upgradeTargetLabel", "versionCompareURL", "versionStatusText", "versionLastUpgradeText", + "versionStateLabels", "versionStateLabel", "versionFormatDuration", "versionAgentsInTurnText", "versionUpgradeStateText", "versionUpgradeStatePillHTML", "versionUpgradeHistoryHTML", "versionNowMs", "versionReadUpgradeProgress", "versionWriteUpgradeProgress", "versionClearUpgradeProgress", "versionMarkUpgradeComplete", "versionReconcileUpgradeProgress", "versionHubUpgradeAttemptKey", "versionAdoptHubUpgradeProgress", "versionElapsedText", "versionScheduleUpgradePoll", "versionShortSHA", "versionSameCommit", "versionDashHTML", "versionPolicy", "versionManagedSuffix", "versionTrackingSummary", "versionCadenceLabel", "versionStatusSummary", "versionUpgradeProgressStatus", "versionUpgradeKey", "versionBeeGlyphHTML", "versionUpgradeHiveHTML", "versionBeeProgressHTML", "versionSetHTMLPreservingUpgradeBee", "versionButtonHTML", "renderVersionUpgradeAction", "renderVersionDetails", "versionManualUpgradeActive", diff --git a/src/pkg/hub/assets/dashboard.html b/src/pkg/hub/assets/dashboard.html index d1c7a88228..3cd0592e50 100644 --- a/src/pkg/hub/assets/dashboard.html +++ b/src/pkg/hub/assets/dashboard.html @@ -9095,10 +9095,16 @@

    ' + esc(sha) + '' + status + behindBadge + (driftDot ? ' ' + driftDot : ''); + /* The interrupt action is offered on the same terms as Upgrade + now: the hub enforces owner/admin on the POST, the card only + hides it from readers who could not click it anyway. */ + var upgradeProgressLine = renderUpgradeProgress(h, isHosted && h.role === 'owner' && !h.upgradeInterruptPending); + if (h.upgradeInterruptPending) upgradeProgressLine += 'interrupt requested'; versionCell = '
    ' + '
    ' + branch + '
    ' + '
    ' + shaLine + '
    ' + (upgradeIcon ? '
    ' + upgradeIcon + '
    ' : '') + + (upgradeProgressLine ? '
    ' + upgradeProgressLine + '
    ' : '') + (autoUpgradeCheck ? '
    ' + autoUpgradeCheck + '
    ' : '') + '
    '; /* No version reported, but drift can still exist (e.g. heartbeat-stale on a @@ -9885,6 +9891,89 @@

    history' + (missed ? ' · ' + missed + ' missed' : '') + ''; + } + + /* renderUpgradeProgress builds the state pill line for the Version cell. + Returns '' when there is nothing worth a line (no state, or a quiet + scheduled/done state), so healthy rows render exactly as today. */ + function renderUpgradeProgress(h, canAct) { + var st = h && h.upgradeState; + var history = renderUpgradeHistory(h); + if (!st || !UPGRADE_PROGRESS_PILL_STATES[st.state]) return history; + var label = upgradeProgressLabel(st.state); + var extra = ''; + if (st.state === 'draining') { + var agents = upgradeAgentsInTurnText(st); + var left = typeof st.drainRemainingS === 'number' && st.drainRemainingS > 0 ? formatUpgradeElapsed(st.drainRemainingS * 1000) + ' left' : 'deadline reached'; + extra = (agents ? ' · ' + agents : '') + ' · ' + left; + } else if (st.state === 'missed' && st.reason) { + extra = ' · ' + st.reason; + } + var tip = st.detail || label; + var pill = '' + esc(label + extra) + ''; + var action = ''; + if (canAct && (st.state === 'draining' || st.state === 'missed')) { + var actTitle = st.state === 'draining' + ? 'Stop waiting for the agents and roll the upgrade now. The turns listed are interrupted and will be retried by their owners.' + : 'The scheduled window was missed; roll the upgrade now, interrupting any agent turn in progress.'; + action = ' Upgrade now (interrupt turns)'; + } + return pill + action + history; + } + function formatUpgradeElapsed(ms) { if (typeof ms !== 'number' || isNaN(ms) || ms < 0) return ''; var totalSec = Math.floor(ms / 1000); @@ -10118,6 +10207,37 @@


    ' + + '' + esc(fromSHA) + ' → ' + esc(toSHA) + '

    ' + + 'Agents still in a turn are stopped mid-work; their owners will have to re-run those turns. Use this when a drain has stalled or a window was missed.', true); + if (!ok) return; + try { + hiveToast('Upgrading ' + id + ' (interrupting turns)...', 'info'); + var resp = await fetch('/api/saas/hives/' + encodeURIComponent(id) + '/upgrade?interrupt=1', {method: 'POST'}); + var data = await resp.json(); + if (!resp.ok) { hiveToast(data.error || 'Upgrade failed', 'error'); delete _upgradingHives[id]; loadHives(); return; } + if (data.status === 'current') { + hiveToast(data.message || ('Already up to date at ' + (data.current || toSHA)), 'info'); + loadHives(); + return; + } + _upgradingHives[id] = currentSHA; + hiveToast('Upgrade armed for ' + id + ' — the spoke interrupts its turns on the next heartbeat', 'success'); + loadHives(); + setTimeout(loadHives, 10000); + setTimeout(loadHives, 30000); + setTimeout(loadHives, 60000); + } catch(e) { hiveToast('Error: ' + e.message, 'error'); delete _upgradingHives[id]; loadHives(); } + } + function toggleBranchMenu(hiveId) { var menu = document.getElementById('branch-menu-' + hiveId); if (!menu) return; diff --git a/src/pkg/hub/heartbeat.go b/src/pkg/hub/heartbeat.go index ddfe268f81..7b30381de1 100644 --- a/src/pkg/hub/heartbeat.go +++ b/src/pkg/hub/heartbeat.go @@ -434,6 +434,10 @@ type HeartbeatGatewayConfig struct { type HeartbeatResponse struct { OK bool `json:"ok"` UpgradeTo string `json:"upgrade_to,omitempty"` + // UpgradeInterrupt rides with UpgradeTo when the owner asked the upgrade + // to proceed without draining in-flight agent turns (#14950). The spoke + // skips its drain for this instruction only. + UpgradeInterrupt bool `json:"upgrade_interrupt,omitempty"` HubGitHash string `json:"hub_git_hash,omitempty"` LatestSHA string `json:"latest_sha,omitempty"` LatestTag string `json:"latest_tag,omitempty"` diff --git a/src/pkg/hub/saas_hives_api.go b/src/pkg/hub/saas_hives_api.go index 013a0d2c89..9942e95fbe 100644 --- a/src/pkg/hub/saas_hives_api.go +++ b/src/pkg/hub/saas_hives_api.go @@ -57,10 +57,13 @@ type MyHiveEntry struct { // the persisted record of recent attempts and held windows. Both are // computed on read by upgradeProgressFor so the card and the spoke // dashboard speak the same words (#14856). - UpgradeState *spoke.UpgradeProgress `json:"upgradeState,omitempty"` - UpgradeHistory []spoke.UpgradeAttempt `json:"upgradeHistory,omitempty"` - PendingRequestCount int `json:"pendingRequestCount,omitempty"` - PendingRequests []PendingAccessRequest `json:"pending_requests,omitempty"` + UpgradeState *spoke.UpgradeProgress `json:"upgradeState,omitempty"` + UpgradeHistory []spoke.UpgradeAttempt `json:"upgradeHistory,omitempty"` + // UpgradeInterruptPending is true while an "Upgrade now (interrupt + // turns)" request is armed and not yet collected by the spoke. + UpgradeInterruptPending bool `json:"upgradeInterruptPending,omitempty"` + PendingRequestCount int `json:"pendingRequestCount,omitempty"` + PendingRequests []PendingAccessRequest `json:"pending_requests,omitempty"` // Access lists who can sign in to this hive and with what role, so My Hives // can show it on hover without a per-row API call. Same data as @@ -1017,6 +1020,9 @@ func (s *HubServer) buildMyHivesResponse(username string, queryValues url.Values } result[i].UpgradeState, result[i].UpgradeHistory = s.upgradeProgressForRow(&result[i].RegistryEntry, saasByID[result[i].ID], result[i].TrackedChannel, journeyNow) + if _, ok := s.upgradeInterrupts.pending(result[i].ID, journeyNow); ok && result[i].Upgrading { + result[i].UpgradeInterruptPending = true + } st := s.journey.get(result[i].ID) status := JourneyStatusFor(&result[i].RegistryEntry, st, journeyNow) diff --git a/src/pkg/hub/saas_upgrade.go b/src/pkg/hub/saas_upgrade.go index 1fae271dda..afddb4391d 100644 --- a/src/pkg/hub/saas_upgrade.go +++ b/src/pkg/hub/saas_upgrade.go @@ -612,11 +612,29 @@ func (s *HubServer) handleUpgradeHive(w http.ResponseWriter, r *http.Request) { // arm) so a LATER refusal for this same target is reported afresh rather than // suppressed by a stale entry. s.forgetUncollectibleUpgrade(id) + // "Upgrade now (interrupt turns)": the owner accepts cutting whatever the + // agents are mid-way through. Recorded separately from the arm so the + // audit trail says who chose to interrupt, and delivered on the next + // heartbeat alongside UpgradeTo (#14950). + interrupt := upgradeInterruptRequested(r) + if interrupt { + s.upgradeInterrupts.arm(id, latestSHA, username, time.Now()) + } else { + s.upgradeInterrupts.clear(id) + } s.logger.Info("audit: hosted hive upgrade requested", - "hive_id", id, "by", username, "cluster", cluster.ID, "mode", mode) - s.recordTimeline(id, TimelineUpgradeStarted, "upgrade requested from the hub dashboard ("+mode+")", username) + "hive_id", id, "by", username, "cluster", cluster.ID, "mode", mode, "interrupt_turns", interrupt) + detail := "upgrade requested from the hub dashboard (" + mode + ")" + if interrupt { + detail = "upgrade requested from the hub dashboard (" + mode + ", interrupting in-flight agent turns)" + } + s.recordTimeline(id, TimelineUpgradeStarted, detail, username) s.noteUpgradeStarted(id, currentSHA, latestSHA, spoke.UpgradeTriggerManual) w.Header().Set("Content-Type", "application/json") + if interrupt { + _, _ = w.Write([]byte(`{"status":"upgrading","mode":"` + mode + `","interrupt":true}`)) + return + } _, _ = w.Write([]byte(`{"status":"upgrading","mode":"` + mode + `"}`)) } diff --git a/src/pkg/hub/server.go b/src/pkg/hub/server.go index 418196d0f7..43296e89de 100644 --- a/src/pkg/hub/server.go +++ b/src/pkg/hub/server.go @@ -1368,6 +1368,10 @@ type HubServer struct { // windows with their outcomes (#14856). Leaf mutex; see upgrade_history.go. upgradeHistory *upgradeHistoryStore + // upgradeInterrupts holds "Upgrade now (interrupt turns)" requests until + // the spoke collects them on its heartbeat (#14950). Leaf mutex. + upgradeInterrupts *upgradeInterrupts + // journey holds per-hive user-journey nudge state: which banner each owner // was last sent and when, plus admin snoozes. Persisted to the PVC. journey *journeyStore @@ -1575,6 +1579,7 @@ func NewHubServer(port int, logger *slog.Logger, gitHash, gitBranch string) *Hub authRolloutSeen: make(map[string]authRolloutEntry), timeline: newTimelineStore(), upgradeHistory: newUpgradeHistoryStore(), + upgradeInterrupts: newUpgradeInterrupts(), journey: newJourneyStore(), alerts: newAlertState(), revokedSessions: newRevokedSessions(), @@ -3226,6 +3231,9 @@ func (s *HubServer) handleHeartbeat(w http.ResponseWriter, r *http.Request) { } } else if spokeManagedTarget != "" { resp.UpgradeTo = spokeManagedTarget + if req, ok := s.upgradeInterrupts.pending(payload.HiveID, time.Now()); ok && req.appliesTo(spokeManagedTarget) { + resp.UpgradeInterrupt = true + } s.logger.Info("heartbeat: instructing spoke-managed hive to upgrade", "hive_id", payload.HiveID, "from", payload.GitHash, @@ -3234,6 +3242,9 @@ func (s *HubServer) handleHeartbeat(w http.ResponseWriter, r *http.Request) { ) } else if hbTarget != "" && !sameCommit(payload.GitHash, hbTarget) { resp.UpgradeTo = hbTarget + if req, ok := s.upgradeInterrupts.pending(payload.HiveID, time.Now()); ok && req.appliesTo(hbTarget) { + resp.UpgradeInterrupt = true + } s.logger.Info("heartbeat: instructing hub-managed hive to upgrade (kubectl fallback)", "hive_id", payload.HiveID, "from", payload.GitHash, diff --git a/src/pkg/hub/spoke/heartbeat.go b/src/pkg/hub/spoke/heartbeat.go index c989143ae1..c70d35d906 100644 --- a/src/pkg/hub/spoke/heartbeat.go +++ b/src/pkg/hub/spoke/heartbeat.go @@ -1413,6 +1413,7 @@ func StartHeartbeat(ctx context.Context, hubURL string, collect StatusCollector, // changes the image tag, which a same-tag re-pull can't do. onSwitchBranch(resp.SwitchToTag) } else if resp.UpgradeTo != "" && onUpgrade != nil { + setUpgradeInterrupt(resp.UpgradeInterrupt) onUpgrade(resp.UpgradeTo) } if resp.GitHubAppConfig != nil && onGitHubAppConfig != nil { @@ -2739,11 +2740,14 @@ type GatewayConfigCallback func(cfg *HeartbeatGatewayConfig) // It includes version info so the spoke can display hub version on its dashboard // and self-upgrade when behind. type HeartbeatResponse struct { - OK bool `json:"ok"` - UpgradeTo string `json:"upgrade_to,omitempty"` - HubGitHash string `json:"hub_git_hash,omitempty"` - LatestSHA string `json:"latest_sha,omitempty"` - LatestTag string `json:"latest_tag,omitempty"` + OK bool `json:"ok"` + UpgradeTo string `json:"upgrade_to,omitempty"` + // UpgradeInterrupt rides with UpgradeTo when the owner asked the upgrade + // to proceed without draining in-flight agent turns (#14950). + UpgradeInterrupt bool `json:"upgrade_interrupt,omitempty"` + HubGitHash string `json:"hub_git_hash,omitempty"` + LatestSHA string `json:"latest_sha,omitempty"` + LatestTag string `json:"latest_tag,omitempty"` // SwitchToTag instructs the spoke to change its own deployment image to // ghcr.io/hivecommons/hive: and restart. Used for branch // switches on clusters the hub can't reach over kubectl — the spoke has @@ -2992,3 +2996,16 @@ func hubHeartbeatStatusIsDefect(status int) bool { } return status >= http.StatusInternalServerError } + +// upgradeInterrupt is the latest heartbeat's UpgradeInterrupt flag, read by +// the UpgradeCallback in cmd/hive to skip its drain (#14950). A package +// atomic rather than a callback parameter so the long-standing +// UpgradeCallback signature stays put. +var upgradeInterrupt atomic.Bool + +func setUpgradeInterrupt(v bool) { upgradeInterrupt.Store(v) } + +// UpgradeInterruptRequested reports whether the hub's latest upgrade +// instruction asked the spoke to interrupt in-flight agent turns rather +// than drain them. +func UpgradeInterruptRequested() bool { return upgradeInterrupt.Load() } diff --git a/src/pkg/hub/spoke/heartbeat_reconnect_upgrade_coverage_test.go b/src/pkg/hub/spoke/heartbeat_reconnect_upgrade_coverage_test.go index df33007878..4203b53b6a 100644 --- a/src/pkg/hub/spoke/heartbeat_reconnect_upgrade_coverage_test.go +++ b/src/pkg/hub/spoke/heartbeat_reconnect_upgrade_coverage_test.go @@ -45,16 +45,17 @@ func TestStartHeartbeatProcessesFullResponse(t *testing.T) { var beats atomic.Int32 isPublic := true resp := HeartbeatResponse{ - OK: true, - UpgradeTo: "deadbeef", - GitHubAppConfig: &HeartbeatGitHubAppConfig{AppID: 7, InstallationID: 99}, - HubBanner: &HubBanner{ID: "b1", Message: "hi"}, - IsPublic: &isPublic, - AuthorizedUsers: []string{"alice:owner"}, - ProjectConfig: &HeartbeatProjectConfig{Org: "acme", Repos: []string{"r"}}, - PendingGateway: &HeartbeatGatewayConfig{Name: "gw", Key: "k"}, - ClusterID: "vllm-d", - RestartSpoke: true, + OK: true, + UpgradeTo: "deadbeef", + UpgradeInterrupt: true, + GitHubAppConfig: &HeartbeatGitHubAppConfig{AppID: 7, InstallationID: 99}, + HubBanner: &HubBanner{ID: "b1", Message: "hi"}, + IsPublic: &isPublic, + AuthorizedUsers: []string{"alice:owner"}, + ProjectConfig: &HeartbeatProjectConfig{Org: "acme", Repos: []string{"r"}}, + PendingGateway: &HeartbeatGatewayConfig{Name: "gw", Key: "k"}, + ClusterID: "vllm-d", + RestartSpoke: true, } srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if beats.Add(1) == 1 { @@ -69,6 +70,7 @@ func TestStartHeartbeatProcessesFullResponse(t *testing.T) { var ( mu sync.Mutex gotUpgrade string + gotInterrupt bool gotApp *HeartbeatGitHubAppConfig gotBanner *HubBanner gotVisibility *bool @@ -83,6 +85,9 @@ func TestStartHeartbeatProcessesFullResponse(t *testing.T) { onUpgrade := UpgradeCallback(func(tag string) { mu.Lock() gotUpgrade = tag + // The interrupt flag must already be visible when the upgrade + // callback runs — that is when the drain decision reads it. + gotInterrupt = UpgradeInterruptRequested() mu.Unlock() }) onApp := GitHubAppConfigCallback(func(c *HeartbeatGitHubAppConfig) { @@ -165,6 +170,9 @@ func TestStartHeartbeatProcessesFullResponse(t *testing.T) { if gotUpgrade != "deadbeef" { t.Errorf("upgrade callback got %q, want deadbeef", gotUpgrade) } + if !gotInterrupt { + t.Error("upgrade_interrupt=true was not visible to the upgrade callback") + } if gotSwitchBranch != "" { t.Errorf("switch-branch callback fired (%q) though response carried only upgrade_to", gotSwitchBranch) } diff --git a/src/pkg/hub/timeline.go b/src/pkg/hub/timeline.go index 940f21593f..9c247acff4 100644 --- a/src/pkg/hub/timeline.go +++ b/src/pkg/hub/timeline.go @@ -547,6 +547,7 @@ func (s *HubServer) recordUpgradeHistoryTransition(old, new *RegistryEntry) { case !old.Upgrading && new.Upgrading: s.noteUpgradeStarted(new.ID, old.GitHash, new.UpgradeTarget, spoke.UpgradeTriggerObserved) case old.Upgrading && !new.Upgrading: + s.upgradeInterrupts.clear(new.ID) switch { case new.UpgradeFailed && !old.UpgradeFailed: reason := new.UpgradeError diff --git a/src/pkg/hub/upgrade_interrupt.go b/src/pkg/hub/upgrade_interrupt.go new file mode 100644 index 0000000000..1faf029197 --- /dev/null +++ b/src/pkg/hub/upgrade_interrupt.go @@ -0,0 +1,93 @@ +package hub + +import ( + "net/http" + "strings" + "sync" + "time" +) + +// upgradeInterruptTTL bounds how long an "Upgrade now (interrupt turns)" +// request stays armed. The spoke collects it on its next heartbeat; if the +// upgrade has not even been delivered after this long the drain it was meant +// to cut is long gone and a stale interrupt must not cut a FUTURE upgrade's +// turns. +const upgradeInterruptTTL = 2 * time.Hour + +// upgradeInterruptReq is one operator request to skip the self-upgrade drain. +type upgradeInterruptReq struct { + Target string + By string + At time.Time +} + +// upgradeInterrupts remembers, per hive, that the owner asked the pending +// upgrade to interrupt in-flight agent turns instead of draining (#14950). +// Leaf mutex; in-memory only — a hub restart drops the request, which is +// the safe failure (the spoke falls back to draining). +type upgradeInterrupts struct { + mu sync.Mutex + byID map[string]upgradeInterruptReq +} + +func newUpgradeInterrupts() *upgradeInterrupts { + return &upgradeInterrupts{byID: make(map[string]upgradeInterruptReq)} +} + +func (u *upgradeInterrupts) arm(hiveID, target, by string, now time.Time) { + if u == nil || hiveID == "" { + return + } + u.mu.Lock() + u.byID[hiveID] = upgradeInterruptReq{Target: target, By: by, At: now} + u.mu.Unlock() +} + +// pending reports whether an unexpired interrupt is armed for hiveID and +// the upgrade target it was requested for. +func (u *upgradeInterrupts) pending(hiveID string, now time.Time) (upgradeInterruptReq, bool) { + if u == nil { + return upgradeInterruptReq{}, false + } + u.mu.Lock() + defer u.mu.Unlock() + req, ok := u.byID[hiveID] + if !ok { + return upgradeInterruptReq{}, false + } + if now.Sub(req.At) > upgradeInterruptTTL { + delete(u.byID, hiveID) + return upgradeInterruptReq{}, false + } + return req, true +} + +// clear drops the request once the upgrade it belonged to is over. +func (u *upgradeInterrupts) clear(hiveID string) { + if u == nil { + return + } + u.mu.Lock() + delete(u.byID, hiveID) + u.mu.Unlock() +} + +// appliesTo reports whether an armed interrupt should ride along with an +// upgrade instruction for target: the request was for this target, or was +// made before the target was known (empty) and is still fresh. +func (r upgradeInterruptReq) appliesTo(target string) bool { + return r.Target == "" || target == "" || sameCommit(r.Target, target) || r.Target == target +} + +// upgradeInterruptRequested reads the interrupt flag off an upgrade request: +// `?interrupt=1` or a JSON body `{"interrupt":true}`. +func upgradeInterruptRequested(r *http.Request) bool { + if r == nil { + return false + } + switch strings.ToLower(strings.TrimSpace(r.URL.Query().Get("interrupt"))) { + case "1", "true", "yes": + return true + } + return false +} diff --git a/src/pkg/hub/upgrade_interrupt_test.go b/src/pkg/hub/upgrade_interrupt_test.go new file mode 100644 index 0000000000..c298d459ea --- /dev/null +++ b/src/pkg/hub/upgrade_interrupt_test.go @@ -0,0 +1,116 @@ +package hub + +import ( + "net/http/httptest" + "testing" + "time" +) + +func TestUpgradeInterrupts_ArmPendingTTLClear(t *testing.T) { + t.Parallel() + u := newUpgradeInterrupts() + now := time.Date(2026, 10, 28, 17, 0, 0, 0, time.UTC) + + if _, ok := u.pending("h1", now); ok { + t.Fatal("nothing armed yet") + } + u.arm("h1", "abc1234", "alice", now) + req, ok := u.pending("h1", now.Add(time.Minute)) + if !ok || req.By != "alice" || req.Target != "abc1234" { + t.Fatalf("pending = %+v, %v", req, ok) + } + if _, ok := u.pending("h1", now.Add(upgradeInterruptTTL+time.Second)); ok { + t.Fatal("expired interrupt must not be pending") + } + if _, ok := u.pending("h1", now); ok { + t.Fatal("expired interrupt must be forgotten, not resurrected") + } + u.arm("h1", "abc1234", "alice", now) + u.clear("h1") + if _, ok := u.pending("h1", now); ok { + t.Fatal("clear must drop the request") + } + u.arm("", "x", "y", now) + if len(u.byID) != 0 { + t.Fatal("empty hive id must be ignored") + } + var nilStore *upgradeInterrupts + nilStore.arm("h", "t", "b", now) + nilStore.clear("h") + if _, ok := nilStore.pending("h", now); ok { + t.Fatal("nil store is inert") + } +} + +func TestUpgradeInterruptReq_AppliesTo(t *testing.T) { + t.Parallel() + full := "551ffc34f0123456789abcdef0123456789abcde" + cases := []struct { + name, reqTarget, target string + want bool + }{ + {"same sha", full, full, true}, + {"short vs full", "551ffc34f", full, true}, + {"request before target known", "", full, true}, + {"instruction without target", full, "", true}, + {"different target", "deadbeef0", full, false}, + } + for _, tc := range cases { + if got := (upgradeInterruptReq{Target: tc.reqTarget}).appliesTo(tc.target); got != tc.want { + t.Errorf("%s: appliesTo = %v, want %v", tc.name, got, tc.want) + } + } +} + +func TestUpgradeInterruptRequested(t *testing.T) { + t.Parallel() + for q, want := range map[string]bool{ + "": false, "?interrupt=1": true, "?interrupt=true": true, "?interrupt=YES": true, + "?interrupt=0": false, "?interrupt=no": false, "?other=1": false, + } { + r := httptest.NewRequest("POST", "/api/saas/hives/h/upgrade"+q, nil) + if got := upgradeInterruptRequested(r); got != want { + t.Errorf("%q: got %v, want %v", q, got, want) + } + } + if upgradeInterruptRequested(nil) { + t.Fatal("nil request") + } +} + +// An armed "Upgrade now (interrupt turns)" rides on the heartbeat that +// carries the upgrade instruction, and only then — a hive that is not being +// told to upgrade never sees the flag, and a different hive never does. +func TestHeartbeatCarriesUpgradeInterruptOnlyWithUpgradeTo(t *testing.T) { + resetCommitOrderState(t) + stubBranchHead(t, "v2", "526ef71") + stubChannelRevisions(t, map[string]string{"stable": "77ba848"}) + srv := newHubServerForTest(t) + srv.setHubSecret("") + now := time.Now() + srv.upgradeInterrupts.arm("chan-spoke", "77ba848", "owner", now) + srv.upgradeInterrupts.arm("chan-other", "77ba848", "owner", now) + + resp := postChannelHeartbeat(t, srv, + `{"hive_id":"chan-spoke","git_hash":"old1111","git_branch":"v2","auto_upgrade":true,`+ + `"image_ref":"ghcr.io/hivecommons/hive:stable"}`) + if resp.UpgradeTo != "77ba848" || !resp.UpgradeInterrupt { + t.Fatalf("got upgrade_to=%q interrupt=%v, want 77ba848 with interrupt", resp.UpgradeTo, resp.UpgradeInterrupt) + } + + // Already at the channel: no instruction, so no interrupt either. + resp = postChannelHeartbeat(t, srv, + `{"hive_id":"chan-other","git_hash":"77ba848","git_branch":"v2","auto_upgrade":true,`+ + `"image_ref":"ghcr.io/hivecommons/hive:stable"}`) + if resp.UpgradeTo != "" || resp.UpgradeInterrupt { + t.Fatalf("got upgrade_to=%q interrupt=%v, want neither", resp.UpgradeTo, resp.UpgradeInterrupt) + } + + // Plain hive with nothing armed. + resp = postChannelHeartbeat(t, srv, + `{"hive_id":"chan-third","git_hash":"old1111","git_branch":"v2","auto_upgrade":true,`+ + `"image_ref":"ghcr.io/hivecommons/hive:stable"}`) + if resp.UpgradeTo != "77ba848" || resp.UpgradeInterrupt { + t.Fatalf("got upgrade_to=%q interrupt=%v, want instruction without interrupt", resp.UpgradeTo, resp.UpgradeInterrupt) + } +} diff --git a/src/pkg/hub/upgrade_progress_markup_test.go b/src/pkg/hub/upgrade_progress_markup_test.go new file mode 100644 index 0000000000..cbddff3f7e --- /dev/null +++ b/src/pkg/hub/upgrade_progress_markup_test.go @@ -0,0 +1,59 @@ +package hub + +import ( + "strings" + "testing" +) + +// The hub card renders the derived upgrade state (#14950): a pill for the +// in-flight/missed states with the drain countdown and the agents holding +// the upgrade, an "Upgrade history" tooltip, and an "Upgrade now (interrupt +// turns)" action that goes through the existing upgrade POST with +// ?interrupt=1 — never a parallel endpoint, never a native dialog. +func TestDashboardRendersUpgradeProgressPillAndHistory(t *testing.T) { + for _, want := range []string{ + "function renderUpgradeProgress(", + "function renderUpgradeHistory(", + "function upgradeAgentsInTurnText(", + "h.upgradeState", + "h.upgradeHistory", + "st.drainRemainingS", + "st.agentsInTurn", + "Upgrade now (interrupt turns)", + "async function upgradeHiveInterrupt(", + "/upgrade?interrupt=1'", + "h.upgradeInterruptPending", + "upgradeProgressLine", + } { + if !strings.Contains(dashboardHTML, want) { + t.Errorf("dashboard.html missing %q", want) + } + } + // Every state the hub can emit has a human label. + labels := extractHubJSVar(t, dashboardHTML, "UPGRADE_PROGRESS_LABELS") + for _, state := range []string{"scheduled", "draining", "pulling", "rolling", "verifying", "done", "missed"} { + if !strings.Contains(labels, state+":") { + t.Errorf("UPGRADE_PROGRESS_LABELS lacks %q: %s", state, labels) + } + } + fn := extractHubJSFunction(t, dashboardHTML, "upgradeHiveInterrupt") + if !strings.Contains(fn, "await hiveConfirm(") || strings.Contains(fn, "window.confirm(") || strings.Contains(fn, " confirm(") { + t.Errorf("upgradeHiveInterrupt must confirm through hiveConfirm, not a native dialog:\n%s", fn) + } + if !strings.Contains(fn, "interrupt in-flight agent turns") { + t.Errorf("upgradeHiveInterrupt confirm wording must say turns are interrupted:\n%s", fn) + } +} + +func extractHubJSVar(t *testing.T, html, name string) string { + t.Helper() + start := strings.Index(html, "var "+name+" = ") + if start < 0 { + t.Fatalf("%s not found in dashboard.html", name) + } + end := strings.Index(html[start:], ";\n") + if end < 0 { + t.Fatalf("%s has no terminating semicolon", name) + } + return html[start : start+end] +} From 868ba0494218fd437c8481cd73bb2cfc67846ca2 Mon Sep 17 00:00:00 2001 From: Andrew Anderson Date: Wed, 28 Oct 2026 10:46:43 -0400 Subject: [PATCH 2/3] =?UTF-8?q?=E2=9C=A8=20dashboards:=20tokenised=20class?= =?UTF-8?q?es=20for=20the=20upgrade=20progress=20pill;=20fragment=20bullet?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The style ratchet counts inline style= attributes and raw px/rem values; the new pill, history badge and interrupt action now use .upgrade-progress-* (hub) and .oc-version-* (spoke) classes built on tokens.css, and the progress line shares the existing stacked Version line. Operator raw-colour baseline lowered to the measured 135. Changelog fragment starts with the required '- ' bullet. Signed-off-by: Andrew Anderson --- changelog.d/added-upgrade-progress-ui.md | 2 +- src/pkg/dashboard/static/index.html | 2 +- .../dashboard/webstatic/style_ratchet_test.go | 2 +- src/pkg/hub/assets/dashboard.html | 33 ++++++++++--------- 4 files changed, 21 insertions(+), 18 deletions(-) diff --git a/changelog.d/added-upgrade-progress-ui.md b/changelog.d/added-upgrade-progress-ui.md index 1c0923f0af..db55bf5aad 100644 --- a/changelog.d/added-upgrade-progress-ui.md +++ b/changelog.d/added-upgrade-progress-ui.md @@ -1 +1 @@ -Both dashboards now show scheduled-upgrade progress. The hub hive card and the spoke's version menu render the upgrade state as a pill — `Draining · 2/5 agents in turn: reviewer 38m, scanner 12m · 14m left`, `Missed window · drain lapsed` — and list the hub's recent upgrade attempts so a missed window is visible on both sides. `/api/version` carries `upgradeProgress` and `upgradeHistory`. Owners get an **Upgrade now (interrupt turns)** action on the hub card while a hive is draining or missed; it is the normal upgrade request with `?interrupt=1`, audit-logged, delivered on the next heartbeat as `upgrade_interrupt`, and makes the spoke skip its drain and restart immediately. +- Both dashboards now show scheduled-upgrade progress ([#14950](https://github.com/hivecommons/hive/issues/14950)). The hub hive card and the spoke's version menu render the upgrade state as a pill — `Draining · 2/5 agents in turn: reviewer 38m, scanner 12m · 14m left`, `Missed window · drain lapsed` — and list the hub's recent upgrade attempts so a missed window is visible on both sides. `/api/version` carries `upgradeProgress` and `upgradeHistory`. Owners get an **Upgrade now (interrupt turns)** action on the hub card while a hive is draining or missed; it is the normal upgrade request with `?interrupt=1`, audit-logged, delivered on the next heartbeat as `upgrade_interrupt`, and makes the spoke skip its drain and restart immediately. diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html index 7b7137a169..1b8bea59cb 100644 --- a/src/pkg/dashboard/static/index.html +++ b/src/pkg/dashboard/static/index.html @@ -403,7 +403,7 @@ .oc-version-channel { flex: 0 1 auto; min-width: 0; max-width: 150px; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; padding: 1px 7px; border-radius: var(--r-pill); color: var(--accent); background: color-mix(in srgb, var(--accent) 14%, transparent); border: 1px solid color-mix(in srgb, var(--accent) 36%, transparent); font-weight: 700; } .oc-version-manual-pill { white-space: nowrap; padding: 1px 7px; border-radius: var(--r-pill); border: 1px solid var(--border); color: var(--muted); font-size: var(--fs-xs); } .oc-version-upgrade-note { color: var(--muted); font-size: var(--fs-xs); overflow-wrap: anywhere; } - .oc-version-state-pill { white-space: nowrap; padding: 1px 7px; border-radius: var(--r-pill); border: 1px solid var(--border); color: var(--muted); font-size: var(--fs-xs); font-variant-numeric: tabular-nums; } + .oc-version-state-pill { white-space: nowrap; padding: var(--sp-1) var(--sp-3); border-radius: var(--r-pill); border: 1px solid var(--border); color: var(--muted); font-size: var(--fs-xs); font-variant-numeric: tabular-nums; } .oc-version-state-pill.is-draining { color: var(--amber); border-color: color-mix(in srgb, var(--amber) 40%, transparent); } .oc-version-state-pill.is-missed { color: var(--red, var(--danger)); border-color: color-mix(in srgb, var(--red, var(--danger)) 40%, transparent); } .oc-version-state-pill.is-done { color: var(--green, var(--success)); } diff --git a/src/pkg/dashboard/webstatic/style_ratchet_test.go b/src/pkg/dashboard/webstatic/style_ratchet_test.go index d77f9ea649..52b411a35f 100644 --- a/src/pkg/dashboard/webstatic/style_ratchet_test.go +++ b/src/pkg/dashboard/webstatic/style_ratchet_test.go @@ -27,7 +27,7 @@ type styleRatchetCounts struct { var styleRatchetBaselines = map[string]styleRatchetCounts{ "operator static/index.html": { inlineStyles: 1427, - rawColors: 137, + rawColors: 135, rawFontSizes: 601, rawPadding: 204, rawBorderRadii: 174}, diff --git a/src/pkg/hub/assets/dashboard.html b/src/pkg/hub/assets/dashboard.html index 3cd0592e50..058fb7d889 100644 --- a/src/pkg/hub/assets/dashboard.html +++ b/src/pkg/hub/assets/dashboard.html @@ -674,6 +674,17 @@ .nps-admin-response-text { font-size: var(--fs-sm); color: var(--text); white-space: pre-wrap; overflow-wrap: anywhere; margin-top: var(--sp-2); } .upgrade-ctl[aria-disabled="true"] { cursor: not-allowed !important; opacity: 0.5; pointer-events: auto; } .version-missing { font-size: var(--fs-2xs); white-space: nowrap; color: var(--muted); } + /* Scheduled-upgrade progress pill, history badge and interrupt action + on the hive card's Version cell (#14950). */ + .upgrade-progress-pill { display: inline-block; padding: var(--sp-1) var(--sp-3); border-radius: var(--r-pill); font-size: var(--fs-2xs); background: var(--surface); color: var(--muted); border: 1px solid var(--border); white-space: nowrap; font-variant-numeric: tabular-nums; } + .upgrade-progress-pill.is-draining { color: var(--amber); } + .upgrade-progress-pill.is-missed, .upgrade-progress-pill.has-missed { color: var(--red); } + .upgrade-progress-pill.is-done { color: var(--green); } + .upgrade-progress-history { margin-left: var(--sp-2); cursor: help; } + .upgrade-progress-interrupt { cursor: pointer; text-decoration: underline; text-underline-offset: 2px; white-space: nowrap; color: var(--red); margin-left: var(--sp-2); font-size: var(--fs-2xs); } + .upgrade-progress-mono { font-family: monospace; font-size: var(--fs-base); color: var(--muted); } + .upgrade-progress-mono.is-target { color: var(--green); } + .upgrade-progress-warn { color: var(--red); } @@ -9099,12 +9110,11 @@

    interrupt requested'; + if (h.upgradeInterruptPending) upgradeProgressLine += 'interrupt requested'; versionCell = '
    ' + '
    ' + branch + '
    ' + '
    ' + shaLine + '
    ' + - (upgradeIcon ? '
    ' + upgradeIcon + '
    ' : '') + - (upgradeProgressLine ? '
    ' + upgradeProgressLine + '
    ' : '') + + (upgradeIcon || upgradeProgressLine ? '
    ' + upgradeIcon + (upgradeIcon && upgradeProgressLine ? ' ' : '') + upgradeProgressLine + '
    ' : '') + (autoUpgradeCheck ? '
    ' + autoUpgradeCheck + '
    ' : '') + '
    '; /* No version reported, but drift can still exist (e.g. heartbeat-stale on a @@ -9906,13 +9916,6 @@

    history' + (missed ? ' · ' + missed + ' missed' : '') + ''; + return 'history' + (missed ? ' · ' + missed + ' missed' : '') + ''; } /* renderUpgradeProgress builds the state pill line for the Version cell. @@ -9961,13 +9964,13 @@

    ' + esc(label + extra) + ''; + var pill = '' + esc(label + extra) + ''; var action = ''; if (canAct && (st.state === 'draining' || st.state === 'missed')) { var actTitle = st.state === 'draining' ? 'Stop waiting for the agents and roll the upgrade now. The turns listed are interrupted and will be retried by their owners.' : 'The scheduled window was missed; roll the upgrade now, interrupting any agent turn in progress.'; - action = ' Upgrade now (interrupt turns)'; } @@ -10216,8 +10219,8 @@

    ' + esc(fromSHA) + ' → ' + esc(toSHA) + '

    ' + - 'Agents still in a turn are stopped mid-work; their owners will have to re-run those turns. Use this when a drain has stalled or a window was missed.', true); + '' + esc(fromSHA) + ' → ' + esc(toSHA) + '

    ' + + 'Agents still in a turn are stopped mid-work; their owners will have to re-run those turns. Use this when a drain has stalled or a window was missed.', true); if (!ok) return; try { hiveToast('Upgrading ' + id + ' (interrupting turns)...', 'info'); From 814639d25e6b67a6f477091590ef32f367483a0d Mon Sep 17 00:00:00 2001 From: Andrew Anderson Date: Wed, 28 Oct 2026 11:07:23 -0400 Subject: [PATCH 3/3] hub card: render the progress pill on the existing upgrade line Keeps the Version cell's one-element-per-line assembly intact for TestVersionCellPutsEachElementOnItsOwnLine. Signed-off-by: Andrew Anderson --- src/pkg/hub/assets/dashboard.html | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/pkg/hub/assets/dashboard.html b/src/pkg/hub/assets/dashboard.html index 058fb7d889..6bef464aaf 100644 --- a/src/pkg/hub/assets/dashboard.html +++ b/src/pkg/hub/assets/dashboard.html @@ -9111,10 +9111,13 @@

    interrupt requested'; + /* The progress pill shares the upgrade line so the cell keeps its + one-element-per-line shape (TestVersionCellPutsEachElementOnItsOwnLine). */ + if (upgradeProgressLine) upgradeIcon = upgradeIcon ? upgradeIcon + ' ' + upgradeProgressLine : upgradeProgressLine; versionCell = '
    ' + '
    ' + branch + '
    ' + '
    ' + shaLine + '
    ' + - (upgradeIcon || upgradeProgressLine ? '
    ' + upgradeIcon + (upgradeIcon && upgradeProgressLine ? ' ' : '') + upgradeProgressLine + '
    ' : '') + + (upgradeIcon ? '
    ' + upgradeIcon + '
    ' : '') + (autoUpgradeCheck ? '
    ' + autoUpgradeCheck + '
    ' : '') + '
    '; /* No version reported, but drift can still exist (e.g. heartbeat-stale on a