diff --git a/changelog.d/added-upgrade-progress-ui.md b/changelog.d/added-upgrade-progress-ui.md new file mode 100644 index 0000000000..db55bf5aad --- /dev/null +++ b/changelog.d/added-upgrade-progress-ui.md @@ -0,0 +1 @@ +- Both dashboards now show scheduled-upgrade progress ([#14950](https://github.com/hivecommons/hive/issues/14950)). The hub hive card and the spoke's version menu render the upgrade state as a pill — `Draining · 2/5 agents in turn: reviewer 38m, scanner 12m · 14m left`, `Missed window · drain lapsed` — and list the hub's recent upgrade attempts so a missed window is visible on both sides. `/api/version` carries `upgradeProgress` and `upgradeHistory`. Owners get an **Upgrade now (interrupt turns)** action on the hub card while a hive is draining or missed; it is the normal upgrade request with `?interrupt=1`, audit-logged, delivered on the next heartbeat as `upgrade_interrupt`, and makes the spoke skip its drain and restart immediately. diff --git a/src/cmd/hive/main.go b/src/cmd/hive/main.go index aa52394fee..dd04607225 100644 --- a/src/cmd/hive/main.go +++ b/src/cmd/hive/main.go @@ -5178,6 +5178,13 @@ func (b *boot) bootHeartbeatWith(deps bootHeartbeatDeps) { } ignoredWedged := wedgedKickAgents(wedged) drainTimeout := b.cfg.Hub.EffectiveUpgradeDrainTimeout() + if spoke.UpgradeInterruptRequested() && drainTimeout > 0 && len(inFlight) > 0 { + // The owner chose "Upgrade now (interrupt turns)" on the + // hub: cut the drain rather than wait for these turns. + b.logger.Warn("self-upgrade: interrupting in-flight agent turns on the owner's request", + "target", targetSHA, "in_flight", inFlight) + drainTimeout = 0 + } dec := selfUpgradeDrain.decide(targetSHA, inFlight, drainTimeout, selfUpgradeDrainNow()) selfUpgradeDrain.setKickTimes(agentKickTimes(statuses, inFlight)) switch { diff --git a/src/docs/api-reference.md b/src/docs/api-reference.md index 1d54983c56..7db055579d 100644 --- a/src/docs/api-reference.md +++ b/src/docs/api-reference.md @@ -724,31 +724,31 @@ always resolved server-side from the validated token. | `GET` | `/api/hub/clusters` | Hub auth | List Clusters | `pkg/hub/saas.go:528` | | `GET` | `/api/hub/image-pulls` | Hub auth | Per-Release Image Pull Series | `pkg/hub/saas.go:377` | | `GET` | `/api/reach` | Hub admin | PR Reach Report (?pr=NNN or ?recent=K) | `pkg/hub/saas.go:519` | -| `GET` | `/fleet` | Hub handler-specific | My-Hives Fleet Page (static; data via `/api/saas/my-hives`) | `pkg/hub/server.go:1756` | -| `GET` | `/my-hives` | Hub handler-specific | 301 redirect to `/fleet` (query preserved) | `pkg/hub/server.go:1757` | -| `POST` | `/api/heartbeat` | Hub handler-specific | Heartbeat | `pkg/hub/server.go:1697` | -| `POST` | `/api/task-status` | Hub handler-specific | Task Status | `pkg/hub/server.go:1698` | -| `GET` | `/api/registry` | Hub handler-specific | Registry | `pkg/hub/server.go:1706` | -| `GET` | `/api/hub/leaderboard` | Hub handler-specific | Leaderboard | `pkg/hub/server.go:1707` | -| `GET` | `/api/hub/stats` | Hub handler-specific | Stats | `pkg/hub/server.go:1708` | -| `GET` | `/api/fleet-stats` | Hub handler-specific | Fleet Stats | `pkg/hub/server.go:1709` | +| `GET` | `/fleet` | Hub handler-specific | My-Hives Fleet Page (static; data via `/api/saas/my-hives`) | `pkg/hub/server.go:1761` | +| `GET` | `/my-hives` | Hub handler-specific | 301 redirect to `/fleet` (query preserved) | `pkg/hub/server.go:1762` | +| `POST` | `/api/heartbeat` | Hub handler-specific | Heartbeat | `pkg/hub/server.go:1702` | +| `POST` | `/api/task-status` | Hub handler-specific | Task Status | `pkg/hub/server.go:1703` | +| `GET` | `/api/registry` | Hub handler-specific | Registry | `pkg/hub/server.go:1711` | +| `GET` | `/api/hub/leaderboard` | Hub handler-specific | Leaderboard | `pkg/hub/server.go:1712` | +| `GET` | `/api/hub/stats` | Hub handler-specific | Stats | `pkg/hub/server.go:1713` | +| `GET` | `/api/fleet-stats` | Hub handler-specific | Fleet Stats | `pkg/hub/server.go:1714` | | `GET` | `/api/hub/evidence` | Hub admin | Read-only cross-hive evidence drill-down from each spoke's stored heartbeat `evidence` summary ([#12116](https://github.com/hivecommons/hive/issues/12116)); non-admins get `403`. Returns `spokes`, `reporting`, pooled `calibration` (bucket 0–5 → `{n, bad, rate, lo, hi}`; the hub view folds the six buckets into four reviewer-confidence ranges), `findings[]` (the plain-language *What this says* sentences, generated from the same fields; [#12456](https://github.com/hivecommons/hive/issues/12456)), `by_acmm_level` (`level`, `hives`, `merged_90d`, `revert_rate`, and `time_to_merge` — `median_of_hive_medians_s`, `hives`, `n` — the median of the level's per-hive median time-to-merge, omitted when no hive at the level reports one; [#12464](https://github.com/hivecommons/hive/issues/12464)), `family_false_approve_rate` (`same`/`cross`/`mixed`, hives classed by their reviewer vs author family sets), fleet `record_coverage` (merged PRs with an evidence record over `prs_merged_90d`), fleet panels ([#12370](https://github.com/hivecommons/hive/issues/12370)) `outcomes_by_author_kind[]` (`kind`, `hives`, `n`, `merged`, `reverted`, `merge_rate`, `revert_rate`), `write_counters[]` (`tier`, `hives`, `attempted`, `blocked`, `block_rate`) and `shadow_gate` (`review_gate` / `intent` → `held` / `passed` outcomes with `revert_rate`), each omitted while no spoke reports it and repeated per `by_acmm_level[]` row, and per hive `hives[]` with `acmm_level`, `prs_merged_90d`, `record_coverage`, `family_mix`, false-approve / false-block / revert rates, families, a calibration series the raw `counts` behind them (including `by_author_kind`) so the drill-down can re-pool a hive / ACMM-level selection client-side, and `criteria[]`: the per-axis criteria table derived hub-side from the quadrant criteria (`axis`, `label`, `scored`, `criteria[]` with `name`, `label`, `status` `met`/`unmet`/`not_reported`, `display`/`score` when measured, `reason` otherwise; offline, evidence-less or unscored hives are `not_reported`, never `unmet`; [#12463](https://github.com/hivecommons/hive/issues/12463)). Every rate is `{n, rate, lo, hi}` with a 95% Wilson interval; `rate`/`lo`/`hi` are null when `n` < 10. Window selection ([RFC #12062](https://github.com/hivecommons/hive/issues/12062)): `?hours=24|168|720` pools each hive's matching heartbeat `windows` block instead of its 90-day block (any other or absent `hours` → 90 days); `?from=YYYY-MM-DD|RFC3339[&to=…]` (date-only `to` is inclusive, `to` defaults to now, `to` ≥ `from`, span ≤ 366 days, else `400`) is served from the smallest reported window that reaches back to `from` and flagged `window.approximate`. The response carries `window` (`hours`, `key`, `label`, `from`, `to`, `approximate`, `coverage_available` — `record_coverage` is only computed for the 90-day window because `prs_merged_90d` is a 90-day count), `window_unreported` and per-hive `window_unreported` for hives whose spoke predates per-window reporting. `?hive=[,…]` (repeatable) and `?level=` restrict the registry server-side to those hives / that ACMM level before pooling. Hub admins open it from the Fleet quadrant card or its **🔬 Evidence** button ([#12117](https://github.com/hivecommons/hive/issues/12117)), pick a window (24h / 7d / 30d / 90d / custom), filter it by hive and ACMM level and read each section's ⓘ explainer; the per-hive author-kind revert counts behind the *Revert rate by author kind* panel are `hives[].counts.by_author_kind` ([#12326](https://github.com/hivecommons/hive/issues/12326)). Never consulted by merge decisions | `pkg/hub/evidence.go` | | `GET` | `/api/hub/evidence/report` | Hub admin | Plain-language report of the hub evidence view ([#12456](https://github.com/hivecommons/hive/issues/12456)), rendered server-side from the same `EvidenceView` as `/api/hub/evidence` so every figure reconciles with the modal; non-admins get `403`. Same `hours=` / `from=` / `to=` window and `hive=` / `level=` filter parameters. `format=md` (default, `text/markdown`) or `format=html` (standalone page, inline CSS, no scripts or external assets, hive names escaped); any other value → `400`. `download=1` sets `Content-Disposition: attachment; filename="hive-evidence---.md|html"`, otherwise inline. Sections: Summary (the generated findings), What we measured and why, The numbers, How to read these, What is not reported yet and why, Grade per section, footer link to `review-evidence.md`. `Cache-Control: no-store`. Read-only; never consulted by merge decisions | `pkg/hub/evidence_report.go` | -| `GET` | `/api/hub/version` | Hub handler-specific | Hub Version | `pkg/hub/server.go:1714` | -| `DELETE` | `/api/hub/registry/{id}` | Hub handler-specific | Registry Delete | `pkg/hub/server.go:1724` | -| `POST` | `/api/contribute/register` | Hub handler-specific | Contribute Proxy | `pkg/hub/server.go:1725` | -| `GET` | `/api/contribute/status` | Hub handler-specific | Contribute Status | `pkg/hub/server.go:1726` | -| `GET` | `/api/contribute/ws` | Hub handler-specific | Contribute WSProxy | `pkg/hub/server.go:1727` | -| `POST` | `/api/github/webhook` | Hub handler-specific | GitHub Webhook | `pkg/hub/server.go:1728` | -| `GET` | `/gh-setup` | Hub handler-specific | GitHub App Setup Router | `pkg/hub/server.go:1729` | -| `GET` | `/learn` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1730` | -| `GET` | `/get-started` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1731` | -| `GET` | `/api/docs` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1734` | -| `GET` | `/api/reading-list` | Hub handler-specific | Reading List | `pkg/hub/server.go:1737` | -| `GET` | `/reading` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1738` | -| `GET` | `/cncf-reference-architecture` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1745` | -| `GET` | `/cncf-reference-architecture/console` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1746` | -| `GET` | `/cncf-reference-architecture/bluefin` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1747` | -| `GET` | `/{$}` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1764` | -| `GET` | `/og-card.png` | Hub handler-specific | OGCard | `pkg/hub/server.go:1769` | -| `GET` | `/` | Public | Static asset fallback (`http.FileServerFS` over the embedded `static/` tree) for any path no other route claims | `pkg/hub/server.go:1771` | +| `GET` | `/api/hub/version` | Hub handler-specific | Hub Version | `pkg/hub/server.go:1719` | +| `DELETE` | `/api/hub/registry/{id}` | Hub handler-specific | Registry Delete | `pkg/hub/server.go:1729` | +| `POST` | `/api/contribute/register` | Hub handler-specific | Contribute Proxy | `pkg/hub/server.go:1730` | +| `GET` | `/api/contribute/status` | Hub handler-specific | Contribute Status | `pkg/hub/server.go:1731` | +| `GET` | `/api/contribute/ws` | Hub handler-specific | Contribute WSProxy | `pkg/hub/server.go:1732` | +| `POST` | `/api/github/webhook` | Hub handler-specific | GitHub Webhook | `pkg/hub/server.go:1733` | +| `GET` | `/gh-setup` | Hub handler-specific | GitHub App Setup Router | `pkg/hub/server.go:1734` | +| `GET` | `/learn` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1735` | +| `GET` | `/get-started` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1736` | +| `GET` | `/api/docs` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1739` | +| `GET` | `/api/reading-list` | Hub handler-specific | Reading List | `pkg/hub/server.go:1742` | +| `GET` | `/reading` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1743` | +| `GET` | `/cncf-reference-architecture` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1750` | +| `GET` | `/cncf-reference-architecture/console` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1751` | +| `GET` | `/cncf-reference-architecture/bluefin` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1752` | +| `GET` | `/{$}` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1769` | +| `GET` | `/og-card.png` | Hub handler-specific | OGCard | `pkg/hub/server.go:1774` | +| `GET` | `/` | Public | Static asset fallback (`http.FileServerFS` over the embedded `static/` tree) for any path no other route claims | `pkg/hub/server.go:1776` | diff --git a/src/docs/operator-reference.md b/src/docs/operator-reference.md index ba16a36e46..3bf1315a42 100644 --- a/src/docs/operator-reference.md +++ b/src/docs/operator-reference.md @@ -957,6 +957,10 @@ A hub-instructed upgrade is a short state machine, and both dashboards expose it **History.** The hub persists the last `HIVE_HUB_UPGRADE_HISTORY_MAX` (default 14) attempts and held windows per hive under `/data/saas/upgrade-history/.json`, so they survive a hub restart. Each row carries `target`, `from`, `trigger` (`scheduled` / `instant` / `manual` / `observed` / `window`), `startedAt`, `endedAt`, `outcome` (`done` / `missed`) and `reason`. A daily window the gate held records exactly one `window` row per calendar day (ET), updated in place if the hold reason changes; a window that fired records the attempt instead. Deleting a hive drops its history. +**Where it shows.** On the hub, the hive card's Version cell gains a pill for the `draining`, `pulling`, `rolling`, `verifying` and `missed` states — `Draining · 2/5 agents in turn: reviewer 38m, scanner 12m · 14m left` — and a `history` badge whose tooltip lists the recent attempts (it turns red and counts the misses when any are present); a quiet `scheduled` row renders as before. On the spoke, `/api/version` carries the same `upgradeProgress` and `upgradeHistory`, and the operator dashboard's version menu shows a **Progress** row with the pill and an **Upgrade history** list; the header chip adds a `Draining 14m` / `Missed window` pill while either state holds. + +**Upgrade now (interrupt turns).** While a hive is `draining` or `missed`, its owner sees an *Upgrade now (interrupt turns)* action on the hub card. It is the ordinary upgrade request — `POST /api/saas/hives/{id}/upgrade?interrupt=1`, same owner/admin check as every other hub mutation — with an interrupt flag the hub audit-logs (`audit: hosted hive upgrade requested … interrupt_turns=true`), records on the hive timeline, and hands to the spoke on the next heartbeat (`upgrade_interrupt: true` beside `upgrade_to`). The spoke then skips the drain and restarts at once; the agents listed as in-turn are cut and their owners must re-run those turns. The request is held in hub memory for 2 hours and cleared when the upgrade closes, so a stale interrupt never cuts a later upgrade's turns; a hub restart drops it (the spoke falls back to draining). While it is armed the card shows `interrupt requested` instead of the action. + ## Governor cadence and budget - Agent cadences are evaluated from persisted state: the last-kick map lives in `/data/hive-state.json` and is honored across pod restarts — a Deployment roll does **not** re-kick every cadenced agent at boot ([#3817](https://github.com/hivecommons/hive/pull/3817)). A fresh install (no persisted state) still kicks every cadenced agent on the first eval. There is no global default interval; a zero/absent interval means the agent is never cadence-kicked. diff --git a/src/pkg/dashboard/api_version.go b/src/pkg/dashboard/api_version.go index 04aea8d9e2..bffdd673db 100644 --- a/src/pkg/dashboard/api_version.go +++ b/src/pkg/dashboard/api_version.go @@ -84,6 +84,19 @@ func (s *Server) handleVersion(w http.ResponseWriter, r *http.Request) { resp["channel"] = channel } } + // Scheduled-upgrade progress (#14950): the same derived state the hub + // card shows, so the version menu can say "draining — 2 agents in turn, + // 14m left" instead of just "upgrading", plus the hub-kept history so a + // missed window is visible from the spoke too. + s.statusMu.RLock() + curStatus := s.status + s.statusMu.RUnlock() + if progress := s.upgradeProgress(curStatus, time.Now()); progress != nil { + resp["upgradeProgress"] = progress + } + if policy != nil && len(policy.UpgradeHistory) > 0 { + resp["upgradeHistory"] = policy.UpgradeHistory + } if cacheAge > dashboardVersionTipCacheTTL || cached == "" { if latest, err := s.fetchLatestRemoteHash(); err == nil && latest != "" { diff --git a/src/pkg/dashboard/sidebar_version_panel_test.go b/src/pkg/dashboard/sidebar_version_panel_test.go index 9dd14ceec8..6ac0ccab43 100644 --- a/src/pkg/dashboard/sidebar_version_panel_test.go +++ b/src/pkg/dashboard/sidebar_version_panel_test.go @@ -117,6 +117,13 @@ const window = {}; jsFunc(t, html, "versionCompareURL") + "\n" + jsFunc(t, html, "versionStatusText") + "\n" + jsFunc(t, html, "versionLastUpgradeText") + "\n" + + jsFunc(t, html, "versionStateLabels") + "\n" + + jsFunc(t, html, "versionStateLabel") + "\n" + + jsFunc(t, html, "versionFormatDuration") + "\n" + + jsFunc(t, html, "versionAgentsInTurnText") + "\n" + + jsFunc(t, html, "versionUpgradeStateText") + "\n" + + jsFunc(t, html, "versionUpgradeStatePillHTML") + "\n" + + jsFunc(t, html, "versionUpgradeHistoryHTML") + "\n" + jsFunc(t, html, "versionShortSHA") + "\n" + jsFunc(t, html, "versionSameCommit") + "\n" + jsFunc(t, html, "versionDashHTML") + "\n" + diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html index fbcf95a6c7..1b8bea59cb 100644 --- a/src/pkg/dashboard/static/index.html +++ b/src/pkg/dashboard/static/index.html @@ -403,6 +403,15 @@ .oc-version-channel { flex: 0 1 auto; min-width: 0; max-width: 150px; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; padding: 1px 7px; border-radius: var(--r-pill); color: var(--accent); background: color-mix(in srgb, var(--accent) 14%, transparent); border: 1px solid color-mix(in srgb, var(--accent) 36%, transparent); font-weight: 700; } .oc-version-manual-pill { white-space: nowrap; padding: 1px 7px; border-radius: var(--r-pill); border: 1px solid var(--border); color: var(--muted); font-size: var(--fs-xs); } .oc-version-upgrade-note { color: var(--muted); font-size: var(--fs-xs); overflow-wrap: anywhere; } + .oc-version-state-pill { white-space: nowrap; padding: var(--sp-1) var(--sp-3); border-radius: var(--r-pill); border: 1px solid var(--border); color: var(--muted); font-size: var(--fs-xs); font-variant-numeric: tabular-nums; } + .oc-version-state-pill.is-draining { color: var(--amber); border-color: color-mix(in srgb, var(--amber) 40%, transparent); } + .oc-version-state-pill.is-missed { color: var(--red, var(--danger)); border-color: color-mix(in srgb, var(--red, var(--danger)) 40%, transparent); } + .oc-version-state-pill.is-done { color: var(--green, var(--success)); } + .oc-version-history { list-style: none; margin: 0; padding: 0; font-size: var(--fs-xs); color: var(--muted); display: flex; flex-direction: column; gap: 2px; } + .oc-version-history li { display: flex; gap: var(--sp-2); min-width: 0; } + .oc-version-history li.is-missed { color: var(--red, var(--danger)); } + .oc-version-history time { flex: 0 0 auto; font-variant-numeric: tabular-nums; } + .oc-version-history span { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } .oc-version-chip-chevron { flex: 0 0 auto; width: 0.8rem; font-size: var(--fs-sm); font-weight: 700; color: var(--muted); line-height: 1; text-align: center; margin-left: auto; transition: transform 0.15s ease; } .oc-version-chip[aria-expanded="false"] .oc-version-chip-chevron { transform: rotate(-90deg); } .oc-version-update-dot { flex: 0 0 auto; color: var(--amber); font-weight: 900; } @@ -39378,6 +39387,79 @@

Configure LiteLLM

return 'Version current'; } + /* Scheduled-upgrade progress (#14950). /api/version carries the state + the hub and spoke derive together (upgradeProgress: scheduled → + draining → pulling → rolling → verifying → done | missed) and the + hub's history of recent attempts (upgradeHistory). The menu shows the + state as a pill — with the drain countdown and the agents holding the + upgrade — and lists the history so a missed window is never silent. */ + function versionStateLabels() { + return { scheduled: 'Scheduled', draining: 'Draining', pulling: 'Pulling', rolling: 'Rolling', verifying: 'Verifying', done: 'Upgraded', missed: 'Missed window' }; + } + + function versionStateLabel(state) { + return versionStateLabels()[state] || (state ? String(state).charAt(0).toUpperCase() + String(state).slice(1) : ''); + } + + function versionFormatDuration(sec) { + sec = Math.floor(Number(sec) || 0); + if (sec < 0) return ''; + if (sec < 60) return sec + 's'; + const min = Math.floor(sec / 60); + if (min < 60) return min + 'm'; + return Math.floor(min / 60) + 'h' + (min % 60) + 'm'; + } + + function versionAgentsInTurnText(st) { + const turns = (st && st.agentsInTurn) || []; + if (!turns.length) return ''; + const parts = turns.map(function(t) { + const el = versionFormatDuration(t.elapsedS || 0); + return (t.agent || '?') + (el ? ' ' + el : ''); + }); + return turns.length + (st.agentsTotal ? '/' + st.agentsTotal : '') + ' agent' + (turns.length === 1 && !st.agentsTotal ? '' : 's') + ' in turn: ' + parts.join(', '); + } + + function versionUpgradeStateText(st) { + if (!st || !st.state) return ''; + let text = versionStateLabel(st.state); + if (st.state === 'draining') { + const agents = versionAgentsInTurnText(st); + const left = typeof st.drainRemainingS === 'number' && st.drainRemainingS > 0 ? versionFormatDuration(st.drainRemainingS) + ' left' : 'deadline reached'; + text += (agents ? ' · ' + agents : '') + ' · ' + left; + } else if (st.state === 'missed' && st.reason) { + text += ' · ' + st.reason; + } else if (st.state === 'scheduled' && st.nextWindowAt) { + const at = new Date(st.nextWindowAt); + if (!isNaN(at.getTime())) text += ' · next window ' + formatDashboardDateTime(at); + } else if (st.state === 'done' && st.target) { + text += ' → ' + versionShortSHA(st.target); + } + return text; + } + + function versionUpgradeStatePillHTML(st) { + const text = versionUpgradeStateText(st); + if (!text) return ''; + return '' + escapeHtml(text) + ''; + } + + function versionUpgradeHistoryHTML(v) { + const rows = (v && v.upgradeHistory) || []; + if (!rows.length) return ''; + const items = rows.slice(0, 14).map(function(a) { + const when = a.endedAt && a.endedAt !== '0001-01-01T00:00:00Z' ? a.endedAt : a.startedAt; + const d = when ? new Date(when) : null; + const ts = d && !isNaN(d.getTime()) ? formatDashboardDateTime(d) : '?'; + const what = a.outcome ? versionStateLabel(a.outcome) : 'In flight'; + const to = a.target ? ' → ' + versionShortSHA(a.target) : ''; + const why = a.reason ? ' — ' + a.reason : ''; + const text = what + to + (a.trigger ? ' (' + a.trigger + ')' : '') + why; + return '
  • ' + escapeHtml(text) + '
  • '; + }); + return '
      ' + items.join('') + '
    '; + } + function versionLastUpgradeText(v) { const rs = v && v.releaseStatus; const at = rs && rs.attempt ? rs.attempt : null; @@ -39687,7 +39769,11 @@

    Configure LiteLLM

    } } if (au.heldReason) html += row('Held', value(au.heldReason, 'The scheduled update window opened but the hub held the upgrade' + (au.heldAt ? ' at ' + au.heldAt : '') + ': ' + au.heldReason), au.heldReason); + const statePill = versionUpgradeStatePillHTML(v && v.upgradeProgress); + if (statePill) html += row('Progress', statePill, (v.upgradeProgress && v.upgradeProgress.detail) || ''); if (lastUpgrade) html += row('History', value(lastUpgrade.text, lastUpgrade.title || lastUpgrade.text), lastUpgrade.title || lastUpgrade.text); + const historyHTML = versionUpgradeHistoryHTML(v); + if (historyHTML) html += '
    Upgrade history
    ' + historyHTML; if (compareURL && (v && (v.behind || (v.commitsBehind != null && v.commitsBehind > 0)))) { html += 'Compare with target ↗'; } @@ -39722,7 +39808,11 @@

    Configure LiteLLM

    const progress = ctx.upgradeProgress || null; const updateDot = ctx.hasUpgrade ? '↑' : ''; const upgrading = progress && !progress.completedAt; - const progressGlyph = progress && progress.completedAt ? '✓' : ''; + const serverState = v.upgradeProgress || null; + const stateChip = !upgrading && serverState && (serverState.state === 'draining' || serverState.state === 'missed') + ? '' + escapeHtml(versionStateLabel(serverState.state)) + (serverState.state === 'draining' && serverState.drainRemainingS > 0 ? ' ' + escapeHtml(versionFormatDuration(serverState.drainRemainingS)) : '') + '' + : ''; + const progressGlyph = (progress && progress.completedAt ? '✓' : '') + stateChip; if (chip.classList && chip.classList.toggle) chip.classList.toggle('is-upgrading', !!upgrading); const html = upgrading ? versionNavbarUpgradeHTML(progress, v) + '' : (v.release ? 'v' + escapeHtml(String(v.release).replace(/^v/i, '')) + '' : '') + diff --git a/src/pkg/dashboard/upgrade_progress_ui_test.go b/src/pkg/dashboard/upgrade_progress_ui_test.go new file mode 100644 index 0000000000..40250cc65a --- /dev/null +++ b/src/pkg/dashboard/upgrade_progress_ui_test.go @@ -0,0 +1,111 @@ +package dashboard + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/hub/spoke" +) + +// /api/version — what the SPA's version menu actually reads — carries the +// same derived upgrade state as /api/status plus the hub's attempt history, +// so the spoke dashboard can show "draining · 1 agent in turn · 1m left" and +// the missed-window list without a second fetch (#14950). +func TestVersionCarriesUpgradeProgressAndHistory(t *testing.T) { + origMarker, origOutcome := upgradeMarkerPath, upgradeOutcomePath + dir := t.TempDir() + upgradeMarkerPath = filepath.Join(dir, "upgrade-requested") + upgradeOutcomePath = filepath.Join(dir, "last-upgrade-outcome") + t.Cleanup(func() { upgradeMarkerPath, upgradeOutcomePath = origMarker, origOutcome }) + + s := &Server{status: &StatusPayload{Agents: []FrontendAgent{{Name: "reviewer"}, {Name: "scanner"}}}, startedAt: time.Now()} + started := time.Date(2026, 10, 26, 17, 0, 0, 0, time.UTC) + s.SetHubUpgradePolicy(&spoke.HeartbeatUpgradePolicy{ + HubManaged: true, + NextWindowAt: "2026-10-29T17:00:00Z", + UpgradeHistory: []spoke.UpgradeAttempt{ + {Target: "abc1234", Trigger: spoke.UpgradeTriggerWindow, StartedAt: started, EndedAt: started, Outcome: spoke.UpgradeOutcomeMissed, Reason: "drain lapsed"}, + }, + }) + s.SetUpgradeDrainSource(func(now time.Time) *spoke.UpgradeDrainReport { + return &spoke.UpgradeDrainReport{Target: "abc1234", RemainingS: 60, InFlight: []string{"reviewer"}, Turns: []spoke.UpgradeTurn{{Agent: "reviewer", ElapsedS: 2280}}} + }) + + rec := httptest.NewRecorder() + s.handleVersion(rec, httptest.NewRequest(http.MethodGet, "/api/version", nil)) + if rec.Code != http.StatusOK { + t.Fatalf("status %d: %s", rec.Code, rec.Body.String()) + } + var body struct { + UpgradeProgress *spoke.UpgradeProgress `json:"upgradeProgress"` + UpgradeHistory []spoke.UpgradeAttempt `json:"upgradeHistory"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil { + t.Fatal(err) + } + p := body.UpgradeProgress + if p == nil || p.State != spoke.UpgradeStateDraining || p.DrainRemainingS != 60 || p.AgentsTotal != 2 { + t.Fatalf("upgradeProgress = %+v, want draining/60s/2 agents", p) + } + if len(p.AgentsInTurn) != 1 || p.AgentsInTurn[0].Agent != "reviewer" || p.AgentsInTurn[0].ElapsedS != 2280 { + t.Fatalf("agentsInTurn = %+v", p.AgentsInTurn) + } + if len(body.UpgradeHistory) != 1 || body.UpgradeHistory[0].Outcome != spoke.UpgradeOutcomeMissed || body.UpgradeHistory[0].Reason != "drain lapsed" { + t.Fatalf("upgradeHistory = %+v", body.UpgradeHistory) + } + + // No policy, no drain, nothing on disk: the keys are absent rather than + // null so older SPA builds see exactly the payload they did before. + s2 := &Server{status: &StatusPayload{}, startedAt: time.Now()} + rec = httptest.NewRecorder() + s2.handleVersion(rec, httptest.NewRequest(http.MethodGet, "/api/version", nil)) + var raw map[string]json.RawMessage + if err := json.Unmarshal(rec.Body.Bytes(), &raw); err != nil { + t.Fatal(err) + } + if _, ok := raw["upgradeHistory"]; ok { + t.Errorf("upgradeHistory present without a policy: %s", raw["upgradeHistory"]) + } +} + +// The SPA renders the state pill and history list from those fields, with +// a label for every state the server can emit, and never through a native +// dialog or an inline style. +func TestIndexRendersUpgradeProgressPillAndHistory(t *testing.T) { + html := indexHTML(t) + for _, want := range []string{ + "function versionUpgradeStatePillHTML(", + "function versionUpgradeHistoryHTML(", + "function versionAgentsInTurnText(", + "v.upgradeProgress", + "v.upgradeHistory", + "st.drainRemainingS", + "st.agentsInTurn", + ".oc-version-state-pill {", + ".oc-version-state-pill.is-draining", + ".oc-version-state-pill.is-missed", + ".oc-version-history {", + "row('Progress', statePill", + ">Upgrade history", + } { + if !strings.Contains(html, want) { + t.Errorf("index.html missing %q", want) + } + } + labels := jsFunc(t, html, "versionStateLabels") + for _, state := range []string{"scheduled", "draining", "pulling", "rolling", "verifying", "done", "missed"} { + if !strings.Contains(labels, state+":") { + t.Errorf("versionStateLabels lacks %q", state) + } + } + fnStart := strings.Index(html, "function versionUpgradeHistoryHTML(") + fn := html[fnStart : fnStart+strings.Index(html[fnStart:], "\n }\n")] + if strings.Contains(fn, "style=") { + t.Errorf("versionUpgradeHistoryHTML must use the .oc-version-history classes, not inline styles:\n%s", fn) + } +} diff --git a/src/pkg/dashboard/version_provenance_ui_test.go b/src/pkg/dashboard/version_provenance_ui_test.go index 6bcc1cf1a2..7d0f79bd60 100644 --- a/src/pkg/dashboard/version_provenance_ui_test.go +++ b/src/pkg/dashboard/version_provenance_ui_test.go @@ -18,6 +18,7 @@ func TestVersionProvenanceRendering(t *testing.T) { var source strings.Builder for _, name := range []string{ "escapeHtml", "versionDeliveryLabel", "versionTrackingLabel", "versionTrackingTooltip", "upgradeTargetLabel", "versionCompareURL", "versionStatusText", "versionLastUpgradeText", + "versionStateLabels", "versionStateLabel", "versionFormatDuration", "versionAgentsInTurnText", "versionUpgradeStateText", "versionUpgradeStatePillHTML", "versionUpgradeHistoryHTML", "versionNowMs", "versionReadUpgradeProgress", "versionWriteUpgradeProgress", "versionClearUpgradeProgress", "versionMarkUpgradeComplete", "versionReconcileUpgradeProgress", "versionHubUpgradeAttemptKey", "versionAdoptHubUpgradeProgress", "versionElapsedText", "versionScheduleUpgradePoll", "versionShortSHA", "versionSameCommit", "versionDashHTML", "versionPolicy", "versionManagedSuffix", "versionTrackingSummary", "versionCadenceLabel", "versionStatusSummary", "versionUpgradeProgressStatus", "versionUpgradeKey", "versionBeeGlyphHTML", "versionUpgradeHiveHTML", "versionBeeProgressHTML", "versionSetHTMLPreservingUpgradeBee", "versionButtonHTML", "renderVersionUpgradeAction", "renderVersionDetails", "versionManualUpgradeActive", diff --git a/src/pkg/dashboard/webstatic/style_ratchet_test.go b/src/pkg/dashboard/webstatic/style_ratchet_test.go index d77f9ea649..52b411a35f 100644 --- a/src/pkg/dashboard/webstatic/style_ratchet_test.go +++ b/src/pkg/dashboard/webstatic/style_ratchet_test.go @@ -27,7 +27,7 @@ type styleRatchetCounts struct { var styleRatchetBaselines = map[string]styleRatchetCounts{ "operator static/index.html": { inlineStyles: 1427, - rawColors: 137, + rawColors: 135, rawFontSizes: 601, rawPadding: 204, rawBorderRadii: 174}, diff --git a/src/pkg/hub/assets/dashboard.html b/src/pkg/hub/assets/dashboard.html index d1c7a88228..6bef464aaf 100644 --- a/src/pkg/hub/assets/dashboard.html +++ b/src/pkg/hub/assets/dashboard.html @@ -674,6 +674,17 @@ .nps-admin-response-text { font-size: var(--fs-sm); color: var(--text); white-space: pre-wrap; overflow-wrap: anywhere; margin-top: var(--sp-2); } .upgrade-ctl[aria-disabled="true"] { cursor: not-allowed !important; opacity: 0.5; pointer-events: auto; } .version-missing { font-size: var(--fs-2xs); white-space: nowrap; color: var(--muted); } + /* Scheduled-upgrade progress pill, history badge and interrupt action + on the hive card's Version cell (#14950). */ + .upgrade-progress-pill { display: inline-block; padding: var(--sp-1) var(--sp-3); border-radius: var(--r-pill); font-size: var(--fs-2xs); background: var(--surface); color: var(--muted); border: 1px solid var(--border); white-space: nowrap; font-variant-numeric: tabular-nums; } + .upgrade-progress-pill.is-draining { color: var(--amber); } + .upgrade-progress-pill.is-missed, .upgrade-progress-pill.has-missed { color: var(--red); } + .upgrade-progress-pill.is-done { color: var(--green); } + .upgrade-progress-history { margin-left: var(--sp-2); cursor: help; } + .upgrade-progress-interrupt { cursor: pointer; text-decoration: underline; text-underline-offset: 2px; white-space: nowrap; color: var(--red); margin-left: var(--sp-2); font-size: var(--fs-2xs); } + .upgrade-progress-mono { font-family: monospace; font-size: var(--fs-base); color: var(--muted); } + .upgrade-progress-mono.is-target { color: var(--green); } + .upgrade-progress-warn { color: var(--red); } @@ -9095,6 +9106,14 @@

    ' + esc(sha) + '' + status + behindBadge + (driftDot ? ' ' + driftDot : ''); + /* The interrupt action is offered on the same terms as Upgrade + now: the hub enforces owner/admin on the POST, the card only + hides it from readers who could not click it anyway. */ + var upgradeProgressLine = renderUpgradeProgress(h, isHosted && h.role === 'owner' && !h.upgradeInterruptPending); + if (h.upgradeInterruptPending) upgradeProgressLine += 'interrupt requested'; + /* The progress pill shares the upgrade line so the cell keeps its + one-element-per-line shape (TestVersionCellPutsEachElementOnItsOwnLine). */ + if (upgradeProgressLine) upgradeIcon = upgradeIcon ? upgradeIcon + ' ' + upgradeProgressLine : upgradeProgressLine; versionCell = '
    ' + '
    ' + branch + '
    ' + '
    ' + shaLine + '
    ' + @@ -9885,6 +9904,82 @@

    history' + (missed ? ' · ' + missed + ' missed' : '') + ''; + } + + /* renderUpgradeProgress builds the state pill line for the Version cell. + Returns '' when there is nothing worth a line (no state, or a quiet + scheduled/done state), so healthy rows render exactly as today. */ + function renderUpgradeProgress(h, canAct) { + var st = h && h.upgradeState; + var history = renderUpgradeHistory(h); + if (!st || !UPGRADE_PROGRESS_PILL_STATES[st.state]) return history; + var label = upgradeProgressLabel(st.state); + var extra = ''; + if (st.state === 'draining') { + var agents = upgradeAgentsInTurnText(st); + var left = typeof st.drainRemainingS === 'number' && st.drainRemainingS > 0 ? formatUpgradeElapsed(st.drainRemainingS * 1000) + ' left' : 'deadline reached'; + extra = (agents ? ' · ' + agents : '') + ' · ' + left; + } else if (st.state === 'missed' && st.reason) { + extra = ' · ' + st.reason; + } + var tip = st.detail || label; + var pill = '' + esc(label + extra) + ''; + var action = ''; + if (canAct && (st.state === 'draining' || st.state === 'missed')) { + var actTitle = st.state === 'draining' + ? 'Stop waiting for the agents and roll the upgrade now. The turns listed are interrupted and will be retried by their owners.' + : 'The scheduled window was missed; roll the upgrade now, interrupting any agent turn in progress.'; + action = ' Upgrade now (interrupt turns)'; + } + return pill + action + history; + } + function formatUpgradeElapsed(ms) { if (typeof ms !== 'number' || isNaN(ms) || ms < 0) return ''; var totalSec = Math.floor(ms / 1000); @@ -10118,6 +10213,37 @@


    ' + + '' + esc(fromSHA) + ' → ' + esc(toSHA) + '

    ' + + 'Agents still in a turn are stopped mid-work; their owners will have to re-run those turns. Use this when a drain has stalled or a window was missed.', true); + if (!ok) return; + try { + hiveToast('Upgrading ' + id + ' (interrupting turns)...', 'info'); + var resp = await fetch('/api/saas/hives/' + encodeURIComponent(id) + '/upgrade?interrupt=1', {method: 'POST'}); + var data = await resp.json(); + if (!resp.ok) { hiveToast(data.error || 'Upgrade failed', 'error'); delete _upgradingHives[id]; loadHives(); return; } + if (data.status === 'current') { + hiveToast(data.message || ('Already up to date at ' + (data.current || toSHA)), 'info'); + loadHives(); + return; + } + _upgradingHives[id] = currentSHA; + hiveToast('Upgrade armed for ' + id + ' — the spoke interrupts its turns on the next heartbeat', 'success'); + loadHives(); + setTimeout(loadHives, 10000); + setTimeout(loadHives, 30000); + setTimeout(loadHives, 60000); + } catch(e) { hiveToast('Error: ' + e.message, 'error'); delete _upgradingHives[id]; loadHives(); } + } + function toggleBranchMenu(hiveId) { var menu = document.getElementById('branch-menu-' + hiveId); if (!menu) return; diff --git a/src/pkg/hub/heartbeat.go b/src/pkg/hub/heartbeat.go index ddfe268f81..7b30381de1 100644 --- a/src/pkg/hub/heartbeat.go +++ b/src/pkg/hub/heartbeat.go @@ -434,6 +434,10 @@ type HeartbeatGatewayConfig struct { type HeartbeatResponse struct { OK bool `json:"ok"` UpgradeTo string `json:"upgrade_to,omitempty"` + // UpgradeInterrupt rides with UpgradeTo when the owner asked the upgrade + // to proceed without draining in-flight agent turns (#14950). The spoke + // skips its drain for this instruction only. + UpgradeInterrupt bool `json:"upgrade_interrupt,omitempty"` HubGitHash string `json:"hub_git_hash,omitempty"` LatestSHA string `json:"latest_sha,omitempty"` LatestTag string `json:"latest_tag,omitempty"` diff --git a/src/pkg/hub/saas_hives_api.go b/src/pkg/hub/saas_hives_api.go index 013a0d2c89..9942e95fbe 100644 --- a/src/pkg/hub/saas_hives_api.go +++ b/src/pkg/hub/saas_hives_api.go @@ -57,10 +57,13 @@ type MyHiveEntry struct { // the persisted record of recent attempts and held windows. Both are // computed on read by upgradeProgressFor so the card and the spoke // dashboard speak the same words (#14856). - UpgradeState *spoke.UpgradeProgress `json:"upgradeState,omitempty"` - UpgradeHistory []spoke.UpgradeAttempt `json:"upgradeHistory,omitempty"` - PendingRequestCount int `json:"pendingRequestCount,omitempty"` - PendingRequests []PendingAccessRequest `json:"pending_requests,omitempty"` + UpgradeState *spoke.UpgradeProgress `json:"upgradeState,omitempty"` + UpgradeHistory []spoke.UpgradeAttempt `json:"upgradeHistory,omitempty"` + // UpgradeInterruptPending is true while an "Upgrade now (interrupt + // turns)" request is armed and not yet collected by the spoke. + UpgradeInterruptPending bool `json:"upgradeInterruptPending,omitempty"` + PendingRequestCount int `json:"pendingRequestCount,omitempty"` + PendingRequests []PendingAccessRequest `json:"pending_requests,omitempty"` // Access lists who can sign in to this hive and with what role, so My Hives // can show it on hover without a per-row API call. Same data as @@ -1017,6 +1020,9 @@ func (s *HubServer) buildMyHivesResponse(username string, queryValues url.Values } result[i].UpgradeState, result[i].UpgradeHistory = s.upgradeProgressForRow(&result[i].RegistryEntry, saasByID[result[i].ID], result[i].TrackedChannel, journeyNow) + if _, ok := s.upgradeInterrupts.pending(result[i].ID, journeyNow); ok && result[i].Upgrading { + result[i].UpgradeInterruptPending = true + } st := s.journey.get(result[i].ID) status := JourneyStatusFor(&result[i].RegistryEntry, st, journeyNow) diff --git a/src/pkg/hub/saas_upgrade.go b/src/pkg/hub/saas_upgrade.go index 1fae271dda..afddb4391d 100644 --- a/src/pkg/hub/saas_upgrade.go +++ b/src/pkg/hub/saas_upgrade.go @@ -612,11 +612,29 @@ func (s *HubServer) handleUpgradeHive(w http.ResponseWriter, r *http.Request) { // arm) so a LATER refusal for this same target is reported afresh rather than // suppressed by a stale entry. s.forgetUncollectibleUpgrade(id) + // "Upgrade now (interrupt turns)": the owner accepts cutting whatever the + // agents are mid-way through. Recorded separately from the arm so the + // audit trail says who chose to interrupt, and delivered on the next + // heartbeat alongside UpgradeTo (#14950). + interrupt := upgradeInterruptRequested(r) + if interrupt { + s.upgradeInterrupts.arm(id, latestSHA, username, time.Now()) + } else { + s.upgradeInterrupts.clear(id) + } s.logger.Info("audit: hosted hive upgrade requested", - "hive_id", id, "by", username, "cluster", cluster.ID, "mode", mode) - s.recordTimeline(id, TimelineUpgradeStarted, "upgrade requested from the hub dashboard ("+mode+")", username) + "hive_id", id, "by", username, "cluster", cluster.ID, "mode", mode, "interrupt_turns", interrupt) + detail := "upgrade requested from the hub dashboard (" + mode + ")" + if interrupt { + detail = "upgrade requested from the hub dashboard (" + mode + ", interrupting in-flight agent turns)" + } + s.recordTimeline(id, TimelineUpgradeStarted, detail, username) s.noteUpgradeStarted(id, currentSHA, latestSHA, spoke.UpgradeTriggerManual) w.Header().Set("Content-Type", "application/json") + if interrupt { + _, _ = w.Write([]byte(`{"status":"upgrading","mode":"` + mode + `","interrupt":true}`)) + return + } _, _ = w.Write([]byte(`{"status":"upgrading","mode":"` + mode + `"}`)) } diff --git a/src/pkg/hub/server.go b/src/pkg/hub/server.go index 418196d0f7..43296e89de 100644 --- a/src/pkg/hub/server.go +++ b/src/pkg/hub/server.go @@ -1368,6 +1368,10 @@ type HubServer struct { // windows with their outcomes (#14856). Leaf mutex; see upgrade_history.go. upgradeHistory *upgradeHistoryStore + // upgradeInterrupts holds "Upgrade now (interrupt turns)" requests until + // the spoke collects them on its heartbeat (#14950). Leaf mutex. + upgradeInterrupts *upgradeInterrupts + // journey holds per-hive user-journey nudge state: which banner each owner // was last sent and when, plus admin snoozes. Persisted to the PVC. journey *journeyStore @@ -1575,6 +1579,7 @@ func NewHubServer(port int, logger *slog.Logger, gitHash, gitBranch string) *Hub authRolloutSeen: make(map[string]authRolloutEntry), timeline: newTimelineStore(), upgradeHistory: newUpgradeHistoryStore(), + upgradeInterrupts: newUpgradeInterrupts(), journey: newJourneyStore(), alerts: newAlertState(), revokedSessions: newRevokedSessions(), @@ -3226,6 +3231,9 @@ func (s *HubServer) handleHeartbeat(w http.ResponseWriter, r *http.Request) { } } else if spokeManagedTarget != "" { resp.UpgradeTo = spokeManagedTarget + if req, ok := s.upgradeInterrupts.pending(payload.HiveID, time.Now()); ok && req.appliesTo(spokeManagedTarget) { + resp.UpgradeInterrupt = true + } s.logger.Info("heartbeat: instructing spoke-managed hive to upgrade", "hive_id", payload.HiveID, "from", payload.GitHash, @@ -3234,6 +3242,9 @@ func (s *HubServer) handleHeartbeat(w http.ResponseWriter, r *http.Request) { ) } else if hbTarget != "" && !sameCommit(payload.GitHash, hbTarget) { resp.UpgradeTo = hbTarget + if req, ok := s.upgradeInterrupts.pending(payload.HiveID, time.Now()); ok && req.appliesTo(hbTarget) { + resp.UpgradeInterrupt = true + } s.logger.Info("heartbeat: instructing hub-managed hive to upgrade (kubectl fallback)", "hive_id", payload.HiveID, "from", payload.GitHash, diff --git a/src/pkg/hub/spoke/heartbeat.go b/src/pkg/hub/spoke/heartbeat.go index c989143ae1..c70d35d906 100644 --- a/src/pkg/hub/spoke/heartbeat.go +++ b/src/pkg/hub/spoke/heartbeat.go @@ -1413,6 +1413,7 @@ func StartHeartbeat(ctx context.Context, hubURL string, collect StatusCollector, // changes the image tag, which a same-tag re-pull can't do. onSwitchBranch(resp.SwitchToTag) } else if resp.UpgradeTo != "" && onUpgrade != nil { + setUpgradeInterrupt(resp.UpgradeInterrupt) onUpgrade(resp.UpgradeTo) } if resp.GitHubAppConfig != nil && onGitHubAppConfig != nil { @@ -2739,11 +2740,14 @@ type GatewayConfigCallback func(cfg *HeartbeatGatewayConfig) // It includes version info so the spoke can display hub version on its dashboard // and self-upgrade when behind. type HeartbeatResponse struct { - OK bool `json:"ok"` - UpgradeTo string `json:"upgrade_to,omitempty"` - HubGitHash string `json:"hub_git_hash,omitempty"` - LatestSHA string `json:"latest_sha,omitempty"` - LatestTag string `json:"latest_tag,omitempty"` + OK bool `json:"ok"` + UpgradeTo string `json:"upgrade_to,omitempty"` + // UpgradeInterrupt rides with UpgradeTo when the owner asked the upgrade + // to proceed without draining in-flight agent turns (#14950). + UpgradeInterrupt bool `json:"upgrade_interrupt,omitempty"` + HubGitHash string `json:"hub_git_hash,omitempty"` + LatestSHA string `json:"latest_sha,omitempty"` + LatestTag string `json:"latest_tag,omitempty"` // SwitchToTag instructs the spoke to change its own deployment image to // ghcr.io/hivecommons/hive: and restart. Used for branch // switches on clusters the hub can't reach over kubectl — the spoke has @@ -2992,3 +2996,16 @@ func hubHeartbeatStatusIsDefect(status int) bool { } return status >= http.StatusInternalServerError } + +// upgradeInterrupt is the latest heartbeat's UpgradeInterrupt flag, read by +// the UpgradeCallback in cmd/hive to skip its drain (#14950). A package +// atomic rather than a callback parameter so the long-standing +// UpgradeCallback signature stays put. +var upgradeInterrupt atomic.Bool + +func setUpgradeInterrupt(v bool) { upgradeInterrupt.Store(v) } + +// UpgradeInterruptRequested reports whether the hub's latest upgrade +// instruction asked the spoke to interrupt in-flight agent turns rather +// than drain them. +func UpgradeInterruptRequested() bool { return upgradeInterrupt.Load() } diff --git a/src/pkg/hub/spoke/heartbeat_reconnect_upgrade_coverage_test.go b/src/pkg/hub/spoke/heartbeat_reconnect_upgrade_coverage_test.go index df33007878..4203b53b6a 100644 --- a/src/pkg/hub/spoke/heartbeat_reconnect_upgrade_coverage_test.go +++ b/src/pkg/hub/spoke/heartbeat_reconnect_upgrade_coverage_test.go @@ -45,16 +45,17 @@ func TestStartHeartbeatProcessesFullResponse(t *testing.T) { var beats atomic.Int32 isPublic := true resp := HeartbeatResponse{ - OK: true, - UpgradeTo: "deadbeef", - GitHubAppConfig: &HeartbeatGitHubAppConfig{AppID: 7, InstallationID: 99}, - HubBanner: &HubBanner{ID: "b1", Message: "hi"}, - IsPublic: &isPublic, - AuthorizedUsers: []string{"alice:owner"}, - ProjectConfig: &HeartbeatProjectConfig{Org: "acme", Repos: []string{"r"}}, - PendingGateway: &HeartbeatGatewayConfig{Name: "gw", Key: "k"}, - ClusterID: "vllm-d", - RestartSpoke: true, + OK: true, + UpgradeTo: "deadbeef", + UpgradeInterrupt: true, + GitHubAppConfig: &HeartbeatGitHubAppConfig{AppID: 7, InstallationID: 99}, + HubBanner: &HubBanner{ID: "b1", Message: "hi"}, + IsPublic: &isPublic, + AuthorizedUsers: []string{"alice:owner"}, + ProjectConfig: &HeartbeatProjectConfig{Org: "acme", Repos: []string{"r"}}, + PendingGateway: &HeartbeatGatewayConfig{Name: "gw", Key: "k"}, + ClusterID: "vllm-d", + RestartSpoke: true, } srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if beats.Add(1) == 1 { @@ -69,6 +70,7 @@ func TestStartHeartbeatProcessesFullResponse(t *testing.T) { var ( mu sync.Mutex gotUpgrade string + gotInterrupt bool gotApp *HeartbeatGitHubAppConfig gotBanner *HubBanner gotVisibility *bool @@ -83,6 +85,9 @@ func TestStartHeartbeatProcessesFullResponse(t *testing.T) { onUpgrade := UpgradeCallback(func(tag string) { mu.Lock() gotUpgrade = tag + // The interrupt flag must already be visible when the upgrade + // callback runs — that is when the drain decision reads it. + gotInterrupt = UpgradeInterruptRequested() mu.Unlock() }) onApp := GitHubAppConfigCallback(func(c *HeartbeatGitHubAppConfig) { @@ -165,6 +170,9 @@ func TestStartHeartbeatProcessesFullResponse(t *testing.T) { if gotUpgrade != "deadbeef" { t.Errorf("upgrade callback got %q, want deadbeef", gotUpgrade) } + if !gotInterrupt { + t.Error("upgrade_interrupt=true was not visible to the upgrade callback") + } if gotSwitchBranch != "" { t.Errorf("switch-branch callback fired (%q) though response carried only upgrade_to", gotSwitchBranch) } diff --git a/src/pkg/hub/timeline.go b/src/pkg/hub/timeline.go index 940f21593f..9c247acff4 100644 --- a/src/pkg/hub/timeline.go +++ b/src/pkg/hub/timeline.go @@ -547,6 +547,7 @@ func (s *HubServer) recordUpgradeHistoryTransition(old, new *RegistryEntry) { case !old.Upgrading && new.Upgrading: s.noteUpgradeStarted(new.ID, old.GitHash, new.UpgradeTarget, spoke.UpgradeTriggerObserved) case old.Upgrading && !new.Upgrading: + s.upgradeInterrupts.clear(new.ID) switch { case new.UpgradeFailed && !old.UpgradeFailed: reason := new.UpgradeError diff --git a/src/pkg/hub/upgrade_interrupt.go b/src/pkg/hub/upgrade_interrupt.go new file mode 100644 index 0000000000..1faf029197 --- /dev/null +++ b/src/pkg/hub/upgrade_interrupt.go @@ -0,0 +1,93 @@ +package hub + +import ( + "net/http" + "strings" + "sync" + "time" +) + +// upgradeInterruptTTL bounds how long an "Upgrade now (interrupt turns)" +// request stays armed. The spoke collects it on its next heartbeat; if the +// upgrade has not even been delivered after this long the drain it was meant +// to cut is long gone and a stale interrupt must not cut a FUTURE upgrade's +// turns. +const upgradeInterruptTTL = 2 * time.Hour + +// upgradeInterruptReq is one operator request to skip the self-upgrade drain. +type upgradeInterruptReq struct { + Target string + By string + At time.Time +} + +// upgradeInterrupts remembers, per hive, that the owner asked the pending +// upgrade to interrupt in-flight agent turns instead of draining (#14950). +// Leaf mutex; in-memory only — a hub restart drops the request, which is +// the safe failure (the spoke falls back to draining). +type upgradeInterrupts struct { + mu sync.Mutex + byID map[string]upgradeInterruptReq +} + +func newUpgradeInterrupts() *upgradeInterrupts { + return &upgradeInterrupts{byID: make(map[string]upgradeInterruptReq)} +} + +func (u *upgradeInterrupts) arm(hiveID, target, by string, now time.Time) { + if u == nil || hiveID == "" { + return + } + u.mu.Lock() + u.byID[hiveID] = upgradeInterruptReq{Target: target, By: by, At: now} + u.mu.Unlock() +} + +// pending reports whether an unexpired interrupt is armed for hiveID and +// the upgrade target it was requested for. +func (u *upgradeInterrupts) pending(hiveID string, now time.Time) (upgradeInterruptReq, bool) { + if u == nil { + return upgradeInterruptReq{}, false + } + u.mu.Lock() + defer u.mu.Unlock() + req, ok := u.byID[hiveID] + if !ok { + return upgradeInterruptReq{}, false + } + if now.Sub(req.At) > upgradeInterruptTTL { + delete(u.byID, hiveID) + return upgradeInterruptReq{}, false + } + return req, true +} + +// clear drops the request once the upgrade it belonged to is over. +func (u *upgradeInterrupts) clear(hiveID string) { + if u == nil { + return + } + u.mu.Lock() + delete(u.byID, hiveID) + u.mu.Unlock() +} + +// appliesTo reports whether an armed interrupt should ride along with an +// upgrade instruction for target: the request was for this target, or was +// made before the target was known (empty) and is still fresh. +func (r upgradeInterruptReq) appliesTo(target string) bool { + return r.Target == "" || target == "" || sameCommit(r.Target, target) || r.Target == target +} + +// upgradeInterruptRequested reads the interrupt flag off an upgrade request: +// `?interrupt=1` or a JSON body `{"interrupt":true}`. +func upgradeInterruptRequested(r *http.Request) bool { + if r == nil { + return false + } + switch strings.ToLower(strings.TrimSpace(r.URL.Query().Get("interrupt"))) { + case "1", "true", "yes": + return true + } + return false +} diff --git a/src/pkg/hub/upgrade_interrupt_test.go b/src/pkg/hub/upgrade_interrupt_test.go new file mode 100644 index 0000000000..c298d459ea --- /dev/null +++ b/src/pkg/hub/upgrade_interrupt_test.go @@ -0,0 +1,116 @@ +package hub + +import ( + "net/http/httptest" + "testing" + "time" +) + +func TestUpgradeInterrupts_ArmPendingTTLClear(t *testing.T) { + t.Parallel() + u := newUpgradeInterrupts() + now := time.Date(2026, 10, 28, 17, 0, 0, 0, time.UTC) + + if _, ok := u.pending("h1", now); ok { + t.Fatal("nothing armed yet") + } + u.arm("h1", "abc1234", "alice", now) + req, ok := u.pending("h1", now.Add(time.Minute)) + if !ok || req.By != "alice" || req.Target != "abc1234" { + t.Fatalf("pending = %+v, %v", req, ok) + } + if _, ok := u.pending("h1", now.Add(upgradeInterruptTTL+time.Second)); ok { + t.Fatal("expired interrupt must not be pending") + } + if _, ok := u.pending("h1", now); ok { + t.Fatal("expired interrupt must be forgotten, not resurrected") + } + u.arm("h1", "abc1234", "alice", now) + u.clear("h1") + if _, ok := u.pending("h1", now); ok { + t.Fatal("clear must drop the request") + } + u.arm("", "x", "y", now) + if len(u.byID) != 0 { + t.Fatal("empty hive id must be ignored") + } + var nilStore *upgradeInterrupts + nilStore.arm("h", "t", "b", now) + nilStore.clear("h") + if _, ok := nilStore.pending("h", now); ok { + t.Fatal("nil store is inert") + } +} + +func TestUpgradeInterruptReq_AppliesTo(t *testing.T) { + t.Parallel() + full := "551ffc34f0123456789abcdef0123456789abcde" + cases := []struct { + name, reqTarget, target string + want bool + }{ + {"same sha", full, full, true}, + {"short vs full", "551ffc34f", full, true}, + {"request before target known", "", full, true}, + {"instruction without target", full, "", true}, + {"different target", "deadbeef0", full, false}, + } + for _, tc := range cases { + if got := (upgradeInterruptReq{Target: tc.reqTarget}).appliesTo(tc.target); got != tc.want { + t.Errorf("%s: appliesTo = %v, want %v", tc.name, got, tc.want) + } + } +} + +func TestUpgradeInterruptRequested(t *testing.T) { + t.Parallel() + for q, want := range map[string]bool{ + "": false, "?interrupt=1": true, "?interrupt=true": true, "?interrupt=YES": true, + "?interrupt=0": false, "?interrupt=no": false, "?other=1": false, + } { + r := httptest.NewRequest("POST", "/api/saas/hives/h/upgrade"+q, nil) + if got := upgradeInterruptRequested(r); got != want { + t.Errorf("%q: got %v, want %v", q, got, want) + } + } + if upgradeInterruptRequested(nil) { + t.Fatal("nil request") + } +} + +// An armed "Upgrade now (interrupt turns)" rides on the heartbeat that +// carries the upgrade instruction, and only then — a hive that is not being +// told to upgrade never sees the flag, and a different hive never does. +func TestHeartbeatCarriesUpgradeInterruptOnlyWithUpgradeTo(t *testing.T) { + resetCommitOrderState(t) + stubBranchHead(t, "v2", "526ef71") + stubChannelRevisions(t, map[string]string{"stable": "77ba848"}) + srv := newHubServerForTest(t) + srv.setHubSecret("") + now := time.Now() + srv.upgradeInterrupts.arm("chan-spoke", "77ba848", "owner", now) + srv.upgradeInterrupts.arm("chan-other", "77ba848", "owner", now) + + resp := postChannelHeartbeat(t, srv, + `{"hive_id":"chan-spoke","git_hash":"old1111","git_branch":"v2","auto_upgrade":true,`+ + `"image_ref":"ghcr.io/hivecommons/hive:stable"}`) + if resp.UpgradeTo != "77ba848" || !resp.UpgradeInterrupt { + t.Fatalf("got upgrade_to=%q interrupt=%v, want 77ba848 with interrupt", resp.UpgradeTo, resp.UpgradeInterrupt) + } + + // Already at the channel: no instruction, so no interrupt either. + resp = postChannelHeartbeat(t, srv, + `{"hive_id":"chan-other","git_hash":"77ba848","git_branch":"v2","auto_upgrade":true,`+ + `"image_ref":"ghcr.io/hivecommons/hive:stable"}`) + if resp.UpgradeTo != "" || resp.UpgradeInterrupt { + t.Fatalf("got upgrade_to=%q interrupt=%v, want neither", resp.UpgradeTo, resp.UpgradeInterrupt) + } + + // Plain hive with nothing armed. + resp = postChannelHeartbeat(t, srv, + `{"hive_id":"chan-third","git_hash":"old1111","git_branch":"v2","auto_upgrade":true,`+ + `"image_ref":"ghcr.io/hivecommons/hive:stable"}`) + if resp.UpgradeTo != "77ba848" || resp.UpgradeInterrupt { + t.Fatalf("got upgrade_to=%q interrupt=%v, want instruction without interrupt", resp.UpgradeTo, resp.UpgradeInterrupt) + } +} diff --git a/src/pkg/hub/upgrade_progress_markup_test.go b/src/pkg/hub/upgrade_progress_markup_test.go new file mode 100644 index 0000000000..cbddff3f7e --- /dev/null +++ b/src/pkg/hub/upgrade_progress_markup_test.go @@ -0,0 +1,59 @@ +package hub + +import ( + "strings" + "testing" +) + +// The hub card renders the derived upgrade state (#14950): a pill for the +// in-flight/missed states with the drain countdown and the agents holding +// the upgrade, an "Upgrade history" tooltip, and an "Upgrade now (interrupt +// turns)" action that goes through the existing upgrade POST with +// ?interrupt=1 — never a parallel endpoint, never a native dialog. +func TestDashboardRendersUpgradeProgressPillAndHistory(t *testing.T) { + for _, want := range []string{ + "function renderUpgradeProgress(", + "function renderUpgradeHistory(", + "function upgradeAgentsInTurnText(", + "h.upgradeState", + "h.upgradeHistory", + "st.drainRemainingS", + "st.agentsInTurn", + "Upgrade now (interrupt turns)", + "async function upgradeHiveInterrupt(", + "/upgrade?interrupt=1'", + "h.upgradeInterruptPending", + "upgradeProgressLine", + } { + if !strings.Contains(dashboardHTML, want) { + t.Errorf("dashboard.html missing %q", want) + } + } + // Every state the hub can emit has a human label. + labels := extractHubJSVar(t, dashboardHTML, "UPGRADE_PROGRESS_LABELS") + for _, state := range []string{"scheduled", "draining", "pulling", "rolling", "verifying", "done", "missed"} { + if !strings.Contains(labels, state+":") { + t.Errorf("UPGRADE_PROGRESS_LABELS lacks %q: %s", state, labels) + } + } + fn := extractHubJSFunction(t, dashboardHTML, "upgradeHiveInterrupt") + if !strings.Contains(fn, "await hiveConfirm(") || strings.Contains(fn, "window.confirm(") || strings.Contains(fn, " confirm(") { + t.Errorf("upgradeHiveInterrupt must confirm through hiveConfirm, not a native dialog:\n%s", fn) + } + if !strings.Contains(fn, "interrupt in-flight agent turns") { + t.Errorf("upgradeHiveInterrupt confirm wording must say turns are interrupted:\n%s", fn) + } +} + +func extractHubJSVar(t *testing.T, html, name string) string { + t.Helper() + start := strings.Index(html, "var "+name+" = ") + if start < 0 { + t.Fatalf("%s not found in dashboard.html", name) + } + end := strings.Index(html[start:], ";\n") + if end < 0 { + t.Fatalf("%s has no terminating semicolon", name) + } + return html[start : start+end] +}