From e0723a1b42bc0d4479e8f02983dca9388de79c34 Mon Sep 17 00:00:00 2001 From: Andrew Anderson Date: Wed, 28 Oct 2026 10:00:46 -0400 Subject: [PATCH] hub+spoke: expose scheduled-upgrade progress, drain countdown and upgrade history MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A hub-instructed upgrade is now a visible state machine on both surfaces (scheduled → draining → pulling → rolling → verifying → done | missed): the hub derives it per hive on /api/saas/my-hives (upgradeState, upgradeHistory) from its latch, the spoke's drain report and the schedule; the spoke derives the same vocabulary on /api/status.upgradeProgress from its drain latch, self-upgrade marker, landed outcome and the hub's policy. The drain report gains per-agent turn elapsed so the card can say "2/5 agents in turn: reviewer 38m, scanner 12m — proceeds in 40m". The heartbeat policy gains next_window_at (the hive's own window regardless of channel — the stable channel's nextUpdateAt is the promotion ETA, which is what made 'overdue since' unreadable) and upgrade_history. The hub persists the last 14 attempts and held windows per hive under /data/saas/upgrade-history/ (HIVE_HUB_UPGRADE_HISTORY_MAX), recorded from the manual and auto arm paths, the heartbeat transition detector, the orphan sweep and the schedule gate's hold recorder, so a missed window survives a restart and is never silent. Refs #14950 (PR A of two). Signed-off-by: Andrew Anderson --- .../added-upgrade-progress-visibility.md | 1 + src/cmd/hive/main.go | 5 +- src/cmd/hive/self_upgrade_drain.go | 43 ++- src/cmd/hive/self_upgrade_drain_test.go | 47 ++- src/docs/api-reference.md | 84 ++-- src/docs/operator-reference.md | 18 + src/docs/release-channels.md | 8 +- src/pkg/dashboard/server.go | 13 +- src/pkg/dashboard/upgrade_progress.go | 223 +++++++++++ src/pkg/dashboard/upgrade_progress_test.go | 166 ++++++++ src/pkg/hub/heartbeat.go | 11 + src/pkg/hub/heartbeat_upgrade_policy.go | 13 +- src/pkg/hub/saas_hives_api.go | 20 +- src/pkg/hub/saas_upgrade.go | 5 + src/pkg/hub/server.go | 6 + src/pkg/hub/spoke/heartbeat.go | 12 + src/pkg/hub/spoke/upgrade_progress.go | 183 +++++++++ src/pkg/hub/spoke/upgrade_progress_test.go | 44 +++ src/pkg/hub/stale_upgrade.go | 5 + src/pkg/hub/timeline.go | 32 ++ src/pkg/hub/upgrade_history.go | 365 ++++++++++++++++++ src/pkg/hub/upgrade_history_test.go | 177 +++++++++ src/pkg/hub/upgrade_progress.go | 218 +++++++++++ src/pkg/hub/upgrade_progress_test.go | 241 ++++++++++++ 24 files changed, 1888 insertions(+), 52 deletions(-) create mode 100644 changelog.d/added-upgrade-progress-visibility.md create mode 100644 src/pkg/dashboard/upgrade_progress.go create mode 100644 src/pkg/dashboard/upgrade_progress_test.go create mode 100644 src/pkg/hub/spoke/upgrade_progress.go create mode 100644 src/pkg/hub/spoke/upgrade_progress_test.go create mode 100644 src/pkg/hub/upgrade_history.go create mode 100644 src/pkg/hub/upgrade_history_test.go create mode 100644 src/pkg/hub/upgrade_progress.go create mode 100644 src/pkg/hub/upgrade_progress_test.go diff --git a/changelog.d/added-upgrade-progress-visibility.md b/changelog.d/added-upgrade-progress-visibility.md new file mode 100644 index 0000000000..54ab4cd1ad --- /dev/null +++ b/changelog.d/added-upgrade-progress-visibility.md @@ -0,0 +1 @@ +- Scheduled upgrades now expose their progress as data on both dashboards ([#14950](https://github.com/hivecommons/hive/issues/14950), motivated by [#14856](https://github.com/hivecommons/hive/issues/14856)): the hub's `/api/saas/my-hives` carries `upgradeState` (`scheduled` → `draining` → `pulling` → `rolling` → `verifying` → `done` | `missed`, with the drain deadline/countdown and the agents whose turns are holding it — "2/5 agents in turn: reviewer 38m, scanner 12m") and `upgradeHistory`; the spoke's `/api/status.upgradeProgress` derives the same state locally and the heartbeat policy gains `next_window_at` (the hive's own window, even on `stable`) plus `upgrade_history`. The hub persists the last 14 attempts and held windows per hive (`HIVE_HUB_UPGRADE_HISTORY_MAX`) under `/data/saas/upgrade-history/`, so a missed window survives a hub restart and is never silent. The spoke's drain report now includes per-agent turn elapsed (`upgrade_drain.turns`). diff --git a/src/cmd/hive/main.go b/src/cmd/hive/main.go index a7b1a13aa7..e7831e77ec 100644 --- a/src/cmd/hive/main.go +++ b/src/cmd/hive/main.go @@ -3721,6 +3721,7 @@ func (b *boot) bootDashboardAPIWith(deps bootDashboardAPIDeps) { }) b.dashSrv.SetGitHubAppRequired(b.githubAppRequired) + b.dashSrv.SetUpgradeDrainSource(selfUpgradeDrain.Report) // Order matters: SetGitHubAppRequired(false) clears both fields, so the // classified state is applied only after it, and only when a failure was // actually detected. @@ -5156,7 +5157,8 @@ func (b *boot) bootHeartbeatWith(deps bootHeartbeatDeps) { // heartbeat) and withhold new kicks; proceed once the turns end or // hub.upgrade_drain_timeout_s elapses. if b.agentMgr != nil { - inFlight, wedged := agentKicksInFlight(b.agentMgr.AllStatuses(), b.startTime, selfUpgradeDrainNow(), func(name string) time.Duration { + statuses := b.agentMgr.AllStatuses() + inFlight, wedged := agentKicksInFlight(statuses, b.startTime, selfUpgradeDrainNow(), func(name string) time.Duration { if ac, ok := b.cfg.Agents[name]; ok { return ac.EffectiveContinuousTurnCeiling() } @@ -5173,6 +5175,7 @@ func (b *boot) bootHeartbeatWith(deps bootHeartbeatDeps) { ignoredWedged := wedgedKickAgents(wedged) drainTimeout := b.cfg.Hub.EffectiveUpgradeDrainTimeout() dec := selfUpgradeDrain.decide(targetSHA, inFlight, drainTimeout, selfUpgradeDrainNow()) + selfUpgradeDrain.setKickTimes(agentKickTimes(statuses, inFlight)) switch { case !dec.Proceed: b.logger.Info("self-upgrade deferred: draining in-flight agent kicks", diff --git a/src/cmd/hive/self_upgrade_drain.go b/src/cmd/hive/self_upgrade_drain.go index eb0d1dd1ef..32abce31ca 100644 --- a/src/cmd/hive/self_upgrade_drain.go +++ b/src/cmd/hive/self_upgrade_drain.go @@ -36,6 +36,10 @@ type upgradeDrain struct { // tell the hub WHAT the drain is waiting on and for how much longer. inFlight []string timeout time.Duration + // kickAt is when each in-flight agent's current turn was kicked, so + // Report can say "reviewer 38m" (#14856). Names absent from the map are + // reported with an unknown elapsed time. + kickAt map[string]time.Time // released is set once decide lets the upgrade proceed. Every path after // that point exits the process or blocks until SIGTERM, so kicks stay // withheld for good: a kick started now would be cut by the restart. @@ -135,12 +139,48 @@ func (d *upgradeDrain) Report(now time.Time) *spoke.UpgradeDrainReport { if remaining < 0 { remaining = 0 } - return &spoke.UpgradeDrainReport{ + rep := &spoke.UpgradeDrainReport{ Target: d.target, InFlight: append([]string(nil), d.inFlight...), WaitedS: int(waited / time.Second), RemainingS: int(remaining / time.Second), } + for _, name := range d.inFlight { + turn := spoke.UpgradeTurn{Agent: name} + if at, ok := d.kickAt[name]; ok && !at.IsZero() && now.After(at) { + turn.ElapsedS = int(now.Sub(at) / time.Second) + } + rep.Turns = append(rep.Turns, turn) + } + return rep +} + +// setKickTimes records when the in-flight agents' turns started. Called next +// to decide with the same statuses it classified; a drain that is not open +// ignores it. +func (d *upgradeDrain) setKickTimes(kickAt map[string]time.Time) { + d.mu.Lock() + defer d.mu.Unlock() + if d.released || d.target == "" { + d.kickAt = nil + return + } + d.kickAt = kickAt +} + +// agentKickTimes returns the kick time of each named agent, for +// upgradeDrain.setKickTimes. +func agentKickTimes(statuses map[string]*crux.Process, names []string) map[string]time.Time { + if len(names) == 0 { + return nil + } + out := make(map[string]time.Time, len(names)) + for _, name := range names { + if proc := statuses[name]; proc != nil && proc.LastKick != nil { + out[name] = *proc.LastKick + } + } + return out } // releaseLocked closes the drain window and latches the kick hold until the @@ -150,6 +190,7 @@ func (d *upgradeDrain) releaseLocked(target string, now time.Time) { d.startedAt = time.Time{} d.checkedAt = now d.inFlight, d.timeout = nil, 0 + d.kickAt = nil d.released = true } diff --git a/src/cmd/hive/self_upgrade_drain_test.go b/src/cmd/hive/self_upgrade_drain_test.go index 6b41c81f20..8cbb12487c 100644 --- a/src/cmd/hive/self_upgrade_drain_test.go +++ b/src/cmd/hive/self_upgrade_drain_test.go @@ -245,7 +245,8 @@ func TestUpgradeDrain_ReportReflectsDeferralForHub(t *testing.T) { if r == nil { t.Fatal("deferring drain must report") } - want := &spoke.UpgradeDrainReport{Target: "4f8be0b", InFlight: []string{"brainstorm"}, WaitedS: 600, RemainingS: 2100} + want := &spoke.UpgradeDrainReport{Target: "4f8be0b", InFlight: []string{"brainstorm"}, WaitedS: 600, RemainingS: 2100, + Turns: []spoke.UpgradeTurn{{Agent: "brainstorm"}}} if !reflect.DeepEqual(r, want) { t.Fatalf("Report = %+v, want %+v", r, want) } @@ -259,3 +260,47 @@ func TestUpgradeDrain_ReportReflectsDeferralForHub(t *testing.T) { t.Fatalf("released drain must report nil, got %+v", r) } } + +// #14856: the drain report names how long each held turn has been running, +// so the hub card can say "reviewer 38m" instead of just "reviewer". +func TestUpgradeDrain_ReportCarriesTurnElapsed(t *testing.T) { + d := &upgradeDrain{} + now := time.Date(2026, 10, 26, 17, 10, 0, 0, time.UTC) + reviewerKick := now.Add(-38 * time.Minute) + scannerKick := now.Add(-12 * time.Minute) + statuses := map[string]*crux.Process{ + "reviewer": {State: crux.StateRunning, LastKick: &reviewerKick}, + "scanner": {State: crux.StateRunning, LastKick: &scannerKick}, + "idle": {State: crux.StateRunning}, + } + inFlight := []string{"reviewer", "scanner"} + + // Not open yet: kick times are ignored rather than retained. + d.setKickTimes(agentKickTimes(statuses, inFlight)) + if d.kickAt != nil { + t.Fatal("kick times must not stick to a closed drain") + } + + d.decide("abc1234", inFlight, 45*time.Minute, now) + d.setKickTimes(agentKickTimes(statuses, inFlight)) + r := d.Report(now.Add(time.Minute)) + if r == nil { + t.Fatal("open drain must report") + } + want := []spoke.UpgradeTurn{{Agent: "reviewer", ElapsedS: 39 * 60}, {Agent: "scanner", ElapsedS: 13 * 60}} + if !reflect.DeepEqual(r.Turns, want) { + t.Fatalf("Turns = %+v, want %+v", r.Turns, want) + } + if got := spoke.FormatTurns(r.Turns, 3); got != "2/3 agents in turn: reviewer 39m, scanner 13m" { + t.Fatalf("FormatTurns = %q", got) + } + + // Release clears the kick times with the rest of the drain inputs. + d.decide("abc1234", nil, 45*time.Minute, now.Add(2*time.Minute)) + if d.kickAt != nil || d.Report(now.Add(2*time.Minute)) != nil { + t.Fatal("released drain must drop kick times and report nil") + } + if agentKickTimes(statuses, nil) != nil { + t.Fatal("no names, no map") + } +} diff --git a/src/docs/api-reference.md b/src/docs/api-reference.md index 26c21a9ba2..44ea36d9df 100644 --- a/src/docs/api-reference.md +++ b/src/docs/api-reference.md @@ -13,17 +13,17 @@ History point limits, timestamp cutoffs, incremental cost reads and ETag/304 beh | `GET` | `/api/version` | Dashboard auth/session | Build/version metadata; includes `upgradeMarker` (`target`, `current`, `attempts`, `maxAttempts`, `failed`, `requestedAt`, `lastError`) while a self-upgrade is in flight or has failed ([#6765](https://github.com/hivecommons/hive/issues/6765)), the hub-delivered `upgradePolicy` (`hub_managed`, `spoke_managed`, `schedule`, `schedule_hour`, `schedule_timezone`, `schedule_weekday`, `paused`, `branch`, `channel`, `target_sha`, `target_resolved`, `armed_target`, and `next_update_at` — the hub's RFC3339 ETA for the next promotion into the `stable` channel, omitted when unknown ([#10256](https://github.com/hivecommons/hive/issues/10256))) when available, and an `autoUpdate` object (`enabled`, `state` — one of `disabled`/`up_to_date`/`behind`/`retrying`/`failed`/`unknown` — `healthy`, `period`, `targetBranch`, `targetCommit`, `currentCommit`, `commitsBehind`, `lastAttemptAt`, `lastError`, `detail`, and `nextUpdateAt` — the hub's RFC3339 ETA for the next update, omitted when unknown, disabled or paused ([#10257](https://github.com/hivecommons/hive/issues/10257))) that never reports a failed or unknown update as healthy ([#6962](https://github.com/hivecommons/hive/issues/6962), [#6963](https://github.com/hivecommons/hive/issues/6963)) | `pkg/dashboard/api.go:42` | | `GET` | `/api/version/release-notes` | Dashboard auth/session | Changelog gap between two revisions: `?from=&to=` returns `{from:{sha,version?}, to:{sha,ref,version?}, sections:[{version,date,categories:{added,changed,fixed,security,deprecated}}], unreleased?, truncated, source, error?}` — the `CHANGELOG.md` release sections present at `to` and absent at `from`, newest first, plus (when `to` is not a release tag) the `changelog.d/` fragments new at `to` under an `Unreleased (in this build)` section. Fetch failures return `source: "unavailable"` with `error` rather than an error status; results are cached per (from, to) for an hour ([#11065](https://github.com/hivecommons/hive/issues/11065)) | `pkg/dashboard/api.go:43` | | `POST` | `/api/release-channel` | Owner only | Hosted spoke self-service release-channel selector; relays `stable`/`candidate`/`edge` to the hub's existing switch-branch endpoint with the spoke dashboard-token proof, and reports the requested channel as pending until the Deployment image lands | `pkg/dashboard/api.go:64` | -| `GET` | `/api/health` | Public | Basic health probe | `pkg/dashboard/server.go:1518` | -| `GET` | `/api/health/deep` | Public | Deep health probe | `pkg/dashboard/server.go:1519` | -| `GET` | `/api/livez` | Public | Kubernetes liveness probe | `pkg/dashboard/server.go:1520` | -| `GET` | `/metrics` | Registered only when `HIVE_METRICS_ENABLED`; requires `Authorization: ****** (403 if the token is unset) | Prometheus metrics | `pkg/dashboard/server.go:1526` | -| `GET` | `/api/status` | Dashboard auth/session | Dashboard aggregate status; `labelPresets[]` (`{key,label,description,labels[],states[]}`) lists the built-in label-filter presets for the Projects cards, led by `needs-human` whose `labels[]` are the effective hard-suppress labels plus `needs-signal`, the hold labels, `stage/071-needs-author` and the reporter-trust awaiting label (#12172); accepts `?fields=a,b` for top-level selection and `?omit=repos` to drop heavy sections, and honors `Accept-Encoding: gzip`; `agents[]` includes busy-pane visibility fields (`kicksUndeliverable`, `busySince`, `lastTranscriptActivity`, `condition`, `conditionMessage`) when a Working pane cannot accept kicks; a working agent also carries `agents[].run` — the live per-run line (#14576): `{since, elapsedS, costUSD, tokens, priced, tests, filesChanged}`, the estimated spend/tokens since the kick was delivered (`lastKickAt`, falling back to `busySince`) (delta of the token collector's per-agent cumulative figure; `priced:false` when the agent has none yet), the last visible test-runner result (`pass`/`fail`/empty) and the last `N files changed` git summary (`-1` when none) parsed from the captured pane; top-level `fleetRun` `{running, hourUSD, hourPriced}` is the governor strip's `running N · $X this hour` figure; `sla` (fleet) and `repos[].sla` (per repo) carry the Delivery SLA block derived from the lifecycle timeline with no extra GitHub calls: `windows.{7d,30d}.{open_to_triage,triage_to_start,start_to_pr,pr_to_merge,open_to_merge}` as `{p50_h,p95_h,n}` (hours, nearest-rank), `history[]` of up to 30 daily buckets `{day,start_to_pr_p50,pr_to_merge_p50,open_to_merge_p50,n_started,n_pr,n_merged}` persisted under `/data/sla-history.json`, `stale_in_progress[]` (`{number,repo,started_at,age_h,agent}` — issues carrying `hive/in-progress` longer than the agent claim TTL with no linked PR), `stale_count`, `merged_7d`, `claim_ttl_s` and, on the fleet block, the effective `targets` (`start_to_pr_h`, `pr_to_merge_h`, `open_to_merge_h`); `staleSince` and `repos[].countsStale` / `countsStaleSince` mark totals served from last-good figures — the previous enumeration (#13382) or the ForgeState cache's last-good counts (#13376) | `pkg/dashboard/server.go:1531` | -| `GET` | `/api/status/summary` | Dashboard auth/session | Compact agent/governor/budget status summary for pollers; honors `Accept-Encoding: gzip` | `pkg/dashboard/server.go:1532` | -| `GET` | `/api/overview/issues.csv` | Dashboard auth/session | Cached Overview issue-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1533` | -| `GET` | `/api/overview/issues.json` | Dashboard auth/session | Cached Overview issue-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1534` | -| `GET` | `/api/overview/prs.csv` | Dashboard auth/session | Cached Overview PR-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1535` | -| `GET` | `/api/overview/prs.json` | Dashboard auth/session | Cached Overview PR-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1536` | -| `GET` | `/api/events` | Dashboard auth/session | Server-sent event stream | `pkg/dashboard/server.go:1537` | +| `GET` | `/api/health` | Public | Basic health probe | `pkg/dashboard/server.go:1522` | +| `GET` | `/api/health/deep` | Public | Deep health probe | `pkg/dashboard/server.go:1523` | +| `GET` | `/api/livez` | Public | Kubernetes liveness probe | `pkg/dashboard/server.go:1524` | +| `GET` | `/metrics` | Registered only when `HIVE_METRICS_ENABLED`; requires `Authorization: ****** (403 if the token is unset) | Prometheus metrics | `pkg/dashboard/server.go:1530` | +| `GET` | `/api/status` | Dashboard auth/session | Dashboard aggregate status; `labelPresets[]` (`{key,label,description,labels[],states[]}`) lists the built-in label-filter presets for the Projects cards, led by `needs-human` whose `labels[]` are the effective hard-suppress labels plus `needs-signal`, the hold labels, `stage/071-needs-author` and the reporter-trust awaiting label (#12172); accepts `?fields=a,b` for top-level selection and `?omit=repos` to drop heavy sections, and honors `Accept-Encoding: gzip`; `agents[]` includes busy-pane visibility fields (`kicksUndeliverable`, `busySince`, `lastTranscriptActivity`, `condition`, `conditionMessage`) when a Working pane cannot accept kicks; a working agent also carries `agents[].run` — the live per-run line (#14576): `{since, elapsedS, costUSD, tokens, priced, tests, filesChanged}`, the estimated spend/tokens since the kick was delivered (`lastKickAt`, falling back to `busySince`) (delta of the token collector's per-agent cumulative figure; `priced:false` when the agent has none yet), the last visible test-runner result (`pass`/`fail`/empty) and the last `N files changed` git summary (`-1` when none) parsed from the captured pane; top-level `fleetRun` `{running, hourUSD, hourPriced}` is the governor strip's `running N · $X this hour` figure; `sla` (fleet) and `repos[].sla` (per repo) carry the Delivery SLA block derived from the lifecycle timeline with no extra GitHub calls: `windows.{7d,30d}.{open_to_triage,triage_to_start,start_to_pr,pr_to_merge,open_to_merge}` as `{p50_h,p95_h,n}` (hours, nearest-rank), `history[]` of up to 30 daily buckets `{day,start_to_pr_p50,pr_to_merge_p50,open_to_merge_p50,n_started,n_pr,n_merged}` persisted under `/data/sla-history.json`, `stale_in_progress[]` (`{number,repo,started_at,age_h,agent}` — issues carrying `hive/in-progress` longer than the agent claim TTL with no linked PR), `stale_count`, `merged_7d`, `claim_ttl_s` and, on the fleet block, the effective `targets` (`start_to_pr_h`, `pr_to_merge_h`, `open_to_merge_h`); `staleSince` and `repos[].countsStale` / `countsStaleSince` mark totals served from last-good figures — the previous enumeration (#13382) or the ForgeState cache's last-good counts (#13376) | `pkg/dashboard/server.go:1535` | +| `GET` | `/api/status/summary` | Dashboard auth/session | Compact agent/governor/budget status summary for pollers; honors `Accept-Encoding: gzip` | `pkg/dashboard/server.go:1536` | +| `GET` | `/api/overview/issues.csv` | Dashboard auth/session | Cached Overview issue-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1537` | +| `GET` | `/api/overview/issues.json` | Dashboard auth/session | Cached Overview issue-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1538` | +| `GET` | `/api/overview/prs.csv` | Dashboard auth/session | Cached Overview PR-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1539` | +| `GET` | `/api/overview/prs.json` | Dashboard auth/session | Cached Overview PR-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1540` | +| `GET` | `/api/events` | Dashboard auth/session | Server-sent event stream | `pkg/dashboard/server.go:1541` | | `GET` | `/api/swarm` | Dashboard auth/session | Current swarm status: display name, duration, active repo, expiry, persisted prep metrics when available, and idle-unlock fields. | `pkg/dashboard/swarm.go:310` | | `POST` | `/api/swarm` | Owner only | Start a 24h repo swarm for a configured repo, returning 409 when another swarm is active and 423 when a previous swarm keeps the next swarm locked until enough agents are idle unless body `{"repo":"owner/name","force":true}` is used; saves the active record before repo prep, persists prep metrics, and announces the start to Discord when configured. | `pkg/dashboard/swarm.go:311` | | `DELETE` | `/api/swarm` | Owner only | End the active swarm, score issues closed and PRs merged in the window, append it to history, and announce the result to Discord when configured. | `pkg/dashboard/swarm.go:312` | @@ -66,7 +66,7 @@ History point limits, timestamp cutoffs, incremental cost reads and ETag/304 beh | Method | Path | Auth | Purpose | Source | |---|---|---|---|---| | `GET` | `/api/style` | Public | Sanitized custom dashboard CSS | `pkg/dashboard/api.go:44` | -| `GET` | `/branding/custom.css` | Dashboard auth/session | Operator branding stylesheet override, read per request (see [branding](branding.md)) | `pkg/dashboard/server.go:1598` | +| `GET` | `/branding/custom.css` | Dashboard auth/session | Operator branding stylesheet override, read per request (see [branding](branding.md)) | `pkg/dashboard/server.go:1602` | | `GET` | `/api/snapshot/frame-ancestors` | Public | Snapshot framing allowlist | `pkg/dashboard/api.go:77` | | `GET` | `/api/snapshot` | Public | Snapshot data | `pkg/dashboard/api.go:78` | | `GET` | `/snapshot` | Public | Public read-only snapshot page | `pkg/dashboard/api.go:79` | @@ -94,9 +94,9 @@ History point limits, timestamp cutoffs, incremental cost reads and ETag/304 beh | `GET` | `/api/openrouter/models` | Dashboard auth/session | Open Router Models | `pkg/dashboard/openrouter.go:47` | | `GET` | `/api/openrouter/credit` | Dashboard auth/session | Open Router Credit | `pkg/dashboard/openrouter.go:48` | | `GET` | `/openrouter/callback` | Public | Open Router Callback | `pkg/dashboard/openrouter.go:49` | -| `POST` | `/api/github-app/recheck` | Dashboard auth/session | GitHub App Recheck | `pkg/dashboard/server.go:1538` | -| `POST` | `/api/github-app/install-clicked` | Dashboard auth/session | GitHub App Install Clicked | `pkg/dashboard/server.go:1539` | -| `GET` | `/gh-setup` | Public | GitHub App Setup Callback | `pkg/dashboard/server.go:1540` | +| `POST` | `/api/github-app/recheck` | Dashboard auth/session | GitHub App Recheck | `pkg/dashboard/server.go:1542` | +| `POST` | `/api/github-app/install-clicked` | Dashboard auth/session | GitHub App Install Clicked | `pkg/dashboard/server.go:1543` | +| `GET` | `/gh-setup` | Public | GitHub App Setup Callback | `pkg/dashboard/server.go:1544` | ## Configuration @@ -594,7 +594,7 @@ always resolved server-side from the validated token. | `POST` | `/api/hives/{id}/heartbeat` | Dashboard auth/session | Hives Heartbeat | `pkg/dashboard/api_contribute.go:275` | | `DELETE` | `/api/hives/{id}` | Owner only | Hives Delete | `pkg/dashboard/api_contribute.go:276` | | `POST` | `/api/hives/onboard` | Dashboard auth/session | Hives Onboard | `pkg/dashboard/api_contribute.go:277` | -| `GET` | `/sso` | Public | SSO | `pkg/dashboard/server.go:1545` | +| `GET` | `/sso` | Public | SSO | `pkg/dashboard/server.go:1549` | ## Hub SaaS @@ -724,31 +724,31 @@ always resolved server-side from the validated token. | `GET` | `/api/hub/clusters` | Hub auth | List Clusters | `pkg/hub/saas.go:528` | | `GET` | `/api/hub/image-pulls` | Hub auth | Per-Release Image Pull Series | `pkg/hub/saas.go:377` | | `GET` | `/api/reach` | Hub admin | PR Reach Report (?pr=NNN or ?recent=K) | `pkg/hub/saas.go:519` | -| `GET` | `/fleet` | Hub handler-specific | My-Hives Fleet Page (static; data via `/api/saas/my-hives`) | `pkg/hub/server.go:1750` | -| `GET` | `/my-hives` | Hub handler-specific | 301 redirect to `/fleet` (query preserved) | `pkg/hub/server.go:1751` | -| `POST` | `/api/heartbeat` | Hub handler-specific | Heartbeat | `pkg/hub/server.go:1691` | -| `POST` | `/api/task-status` | Hub handler-specific | Task Status | `pkg/hub/server.go:1692` | -| `GET` | `/api/registry` | Hub handler-specific | Registry | `pkg/hub/server.go:1700` | -| `GET` | `/api/hub/leaderboard` | Hub handler-specific | Leaderboard | `pkg/hub/server.go:1701` | -| `GET` | `/api/hub/stats` | Hub handler-specific | Stats | `pkg/hub/server.go:1702` | -| `GET` | `/api/fleet-stats` | Hub handler-specific | Fleet Stats | `pkg/hub/server.go:1703` | +| `GET` | `/fleet` | Hub handler-specific | My-Hives Fleet Page (static; data via `/api/saas/my-hives`) | `pkg/hub/server.go:1756` | +| `GET` | `/my-hives` | Hub handler-specific | 301 redirect to `/fleet` (query preserved) | `pkg/hub/server.go:1757` | +| `POST` | `/api/heartbeat` | Hub handler-specific | Heartbeat | `pkg/hub/server.go:1697` | +| `POST` | `/api/task-status` | Hub handler-specific | Task Status | `pkg/hub/server.go:1698` | +| `GET` | `/api/registry` | Hub handler-specific | Registry | `pkg/hub/server.go:1706` | +| `GET` | `/api/hub/leaderboard` | Hub handler-specific | Leaderboard | `pkg/hub/server.go:1707` | +| `GET` | `/api/hub/stats` | Hub handler-specific | Stats | `pkg/hub/server.go:1708` | +| `GET` | `/api/fleet-stats` | Hub handler-specific | Fleet Stats | `pkg/hub/server.go:1709` | | `GET` | `/api/hub/evidence` | Hub admin | Read-only cross-hive evidence drill-down from each spoke's stored heartbeat `evidence` summary ([#12116](https://github.com/hivecommons/hive/issues/12116)); non-admins get `403`. Returns `spokes`, `reporting`, pooled `calibration` (bucket 0–5 → `{n, bad, rate, lo, hi}`; the hub view folds the six buckets into four reviewer-confidence ranges), `findings[]` (the plain-language *What this says* sentences, generated from the same fields; [#12456](https://github.com/hivecommons/hive/issues/12456)), `by_acmm_level` (`level`, `hives`, `merged_90d`, `revert_rate`, and `time_to_merge` — `median_of_hive_medians_s`, `hives`, `n` — the median of the level's per-hive median time-to-merge, omitted when no hive at the level reports one; [#12464](https://github.com/hivecommons/hive/issues/12464)), `family_false_approve_rate` (`same`/`cross`/`mixed`, hives classed by their reviewer vs author family sets), fleet `record_coverage` (merged PRs with an evidence record over `prs_merged_90d`), fleet panels ([#12370](https://github.com/hivecommons/hive/issues/12370)) `outcomes_by_author_kind[]` (`kind`, `hives`, `n`, `merged`, `reverted`, `merge_rate`, `revert_rate`), `write_counters[]` (`tier`, `hives`, `attempted`, `blocked`, `block_rate`) and `shadow_gate` (`review_gate` / `intent` → `held` / `passed` outcomes with `revert_rate`), each omitted while no spoke reports it and repeated per `by_acmm_level[]` row, and per hive `hives[]` with `acmm_level`, `prs_merged_90d`, `record_coverage`, `family_mix`, false-approve / false-block / revert rates, families, a calibration series the raw `counts` behind them (including `by_author_kind`) so the drill-down can re-pool a hive / ACMM-level selection client-side, and `criteria[]`: the per-axis criteria table derived hub-side from the quadrant criteria (`axis`, `label`, `scored`, `criteria[]` with `name`, `label`, `status` `met`/`unmet`/`not_reported`, `display`/`score` when measured, `reason` otherwise; offline, evidence-less or unscored hives are `not_reported`, never `unmet`; [#12463](https://github.com/hivecommons/hive/issues/12463)). Every rate is `{n, rate, lo, hi}` with a 95% Wilson interval; `rate`/`lo`/`hi` are null when `n` < 10. Window selection ([RFC #12062](https://github.com/hivecommons/hive/issues/12062)): `?hours=24|168|720` pools each hive's matching heartbeat `windows` block instead of its 90-day block (any other or absent `hours` → 90 days); `?from=YYYY-MM-DD|RFC3339[&to=…]` (date-only `to` is inclusive, `to` defaults to now, `to` ≥ `from`, span ≤ 366 days, else `400`) is served from the smallest reported window that reaches back to `from` and flagged `window.approximate`. The response carries `window` (`hours`, `key`, `label`, `from`, `to`, `approximate`, `coverage_available` — `record_coverage` is only computed for the 90-day window because `prs_merged_90d` is a 90-day count), `window_unreported` and per-hive `window_unreported` for hives whose spoke predates per-window reporting. `?hive=[,…]` (repeatable) and `?level=` restrict the registry server-side to those hives / that ACMM level before pooling. Hub admins open it from the Fleet quadrant card or its **🔬 Evidence** button ([#12117](https://github.com/hivecommons/hive/issues/12117)), pick a window (24h / 7d / 30d / 90d / custom), filter it by hive and ACMM level and read each section's ⓘ explainer; the per-hive author-kind revert counts behind the *Revert rate by author kind* panel are `hives[].counts.by_author_kind` ([#12326](https://github.com/hivecommons/hive/issues/12326)). Never consulted by merge decisions | `pkg/hub/evidence.go` | | `GET` | `/api/hub/evidence/report` | Hub admin | Plain-language report of the hub evidence view ([#12456](https://github.com/hivecommons/hive/issues/12456)), rendered server-side from the same `EvidenceView` as `/api/hub/evidence` so every figure reconciles with the modal; non-admins get `403`. Same `hours=` / `from=` / `to=` window and `hive=` / `level=` filter parameters. `format=md` (default, `text/markdown`) or `format=html` (standalone page, inline CSS, no scripts or external assets, hive names escaped); any other value → `400`. `download=1` sets `Content-Disposition: attachment; filename="hive-evidence---.md|html"`, otherwise inline. Sections: Summary (the generated findings), What we measured and why, The numbers, How to read these, What is not reported yet and why, Grade per section, footer link to `review-evidence.md`. `Cache-Control: no-store`. Read-only; never consulted by merge decisions | `pkg/hub/evidence_report.go` | -| `GET` | `/api/hub/version` | Hub handler-specific | Hub Version | `pkg/hub/server.go:1708` | -| `DELETE` | `/api/hub/registry/{id}` | Hub handler-specific | Registry Delete | `pkg/hub/server.go:1718` | -| `POST` | `/api/contribute/register` | Hub handler-specific | Contribute Proxy | `pkg/hub/server.go:1719` | -| `GET` | `/api/contribute/status` | Hub handler-specific | Contribute Status | `pkg/hub/server.go:1720` | -| `GET` | `/api/contribute/ws` | Hub handler-specific | Contribute WSProxy | `pkg/hub/server.go:1721` | -| `POST` | `/api/github/webhook` | Hub handler-specific | GitHub Webhook | `pkg/hub/server.go:1722` | -| `GET` | `/gh-setup` | Hub handler-specific | GitHub App Setup Router | `pkg/hub/server.go:1723` | -| `GET` | `/learn` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1724` | -| `GET` | `/get-started` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1725` | -| `GET` | `/api/docs` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1728` | -| `GET` | `/api/reading-list` | Hub handler-specific | Reading List | `pkg/hub/server.go:1731` | -| `GET` | `/reading` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1732` | -| `GET` | `/cncf-reference-architecture` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1739` | -| `GET` | `/cncf-reference-architecture/console` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1740` | -| `GET` | `/cncf-reference-architecture/bluefin` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1741` | -| `GET` | `/{$}` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1758` | -| `GET` | `/og-card.png` | Hub handler-specific | OGCard | `pkg/hub/server.go:1763` | -| `GET` | `/` | Public | Static asset fallback (`http.FileServerFS` over the embedded `static/` tree) for any path no other route claims | `pkg/hub/server.go:1765` | +| `GET` | `/api/hub/version` | Hub handler-specific | Hub Version | `pkg/hub/server.go:1714` | +| `DELETE` | `/api/hub/registry/{id}` | Hub handler-specific | Registry Delete | `pkg/hub/server.go:1724` | +| `POST` | `/api/contribute/register` | Hub handler-specific | Contribute Proxy | `pkg/hub/server.go:1725` | +| `GET` | `/api/contribute/status` | Hub handler-specific | Contribute Status | `pkg/hub/server.go:1726` | +| `GET` | `/api/contribute/ws` | Hub handler-specific | Contribute WSProxy | `pkg/hub/server.go:1727` | +| `POST` | `/api/github/webhook` | Hub handler-specific | GitHub Webhook | `pkg/hub/server.go:1728` | +| `GET` | `/gh-setup` | Hub handler-specific | GitHub App Setup Router | `pkg/hub/server.go:1729` | +| `GET` | `/learn` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1730` | +| `GET` | `/get-started` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1731` | +| `GET` | `/api/docs` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1734` | +| `GET` | `/api/reading-list` | Hub handler-specific | Reading List | `pkg/hub/server.go:1737` | +| `GET` | `/reading` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1738` | +| `GET` | `/cncf-reference-architecture` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1745` | +| `GET` | `/cncf-reference-architecture/console` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1746` | +| `GET` | `/cncf-reference-architecture/bluefin` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1747` | +| `GET` | `/{$}` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1764` | +| `GET` | `/og-card.png` | Hub handler-specific | OGCard | `pkg/hub/server.go:1769` | +| `GET` | `/` | Public | Static asset fallback (`http.FileServerFS` over the embedded `static/` tree) for any path no other route claims | `pkg/hub/server.go:1771` | diff --git a/src/docs/operator-reference.md b/src/docs/operator-reference.md index 5cf3107b55..ba16a36e46 100644 --- a/src/docs/operator-reference.md +++ b/src/docs/operator-reference.md @@ -939,6 +939,24 @@ services: To relate an image to source, compare the `` tag published by the same workflow with commits on `v4`, or inspect the Docker workflow run for the commit SHA that produced the digest. +## Scheduled-upgrade progress and history + +A hub-instructed upgrade is a short state machine, and both dashboards expose it as data so a window that does not roll is never silent ([#14856](https://github.com/hivecommons/hive/issues/14856)). The hub derives it per hive on `/api/saas/my-hives` (`upgradeState`, `upgradeHistory`), and the spoke derives the same vocabulary locally on `/api/status.upgradeProgress` from its drain latch, self-upgrade marker, last landed outcome and the hub's heartbeat policy (`upgrade_policy.next_window_at`, `upgrade_policy.upgrade_history`). + +| State | Meaning | Extra fields | +| --- | --- | --- | +| `scheduled` | Nothing in flight. | `nextWindowAt` — the hive's own daily/weekly slot (always, even on `stable`, where `nextUpdateAt` is the channel's promotion ETA instead). | +| `draining` | The spoke received the instruction and is waiting for kicked agent turns to end. | `drainDeadline`, `drainRemainingS`, `agentsInTurn[]` (`agent`, `elapsedS`), `agentsTotal`; `detail` reads `2/5 agents in turn: reviewer 38m, scanner 12m — proceeds in 40m`. | +| `pulling` | Instruction delivered; the spoke acknowledged it, or (older build) may be draining without reporting it. | `reason` qualifies a silent spoke or a Deployment-patch fallback. | +| `rolling` | No heartbeat since the instruction — the pod is restarting onto the target. | | +| `verifying` | The spoke is back and reports the target commit; the hub has not yet closed the attempt. | | +| `done` | The most recent attempt landed (shown for 24 h, then `scheduled`). | `endedAt` | +| `missed` | The attempt or window did not land. | `reason`: drain lapsed, hub restarted, spoke unreachable, image pull failed, abandoned after N attempts, or the schedule gate's hold reason while the window is still open. | + +**Drain window.** When the instruction arrives mid-turn the spoke holds the restart for up to `hub.upgrade_drain_timeout_s` (default 45 min) and reports what it is waiting on every heartbeat (`upgrade_drain.turns`). The hub's orphan sweep leaves a draining spoke alone for that window plus a 20-minute grace; the countdown both dashboards show is anchored on the last heartbeat that carried the report. + +**History.** The hub persists the last `HIVE_HUB_UPGRADE_HISTORY_MAX` (default 14) attempts and held windows per hive under `/data/saas/upgrade-history/.json`, so they survive a hub restart. Each row carries `target`, `from`, `trigger` (`scheduled` / `instant` / `manual` / `observed` / `window`), `startedAt`, `endedAt`, `outcome` (`done` / `missed`) and `reason`. A daily window the gate held records exactly one `window` row per calendar day (ET), updated in place if the hold reason changes; a window that fired records the attempt instead. Deleting a hive drops its history. + ## Governor cadence and budget - Agent cadences are evaluated from persisted state: the last-kick map lives in `/data/hive-state.json` and is honored across pod restarts — a Deployment roll does **not** re-kick every cadenced agent at boot ([#3817](https://github.com/hivecommons/hive/pull/3817)). A fresh install (no persisted state) still kicks every cadenced agent on the first eval. There is no global default interval; a zero/absent interval means the agent is never cadence-kicked. diff --git a/src/docs/release-channels.md b/src/docs/release-channels.md index b7eb145474..deeb47aa3e 100644 --- a/src/docs/release-channels.md +++ b/src/docs/release-channels.md @@ -168,7 +168,13 @@ normal path is: and are held for exactly the window they report; older spokes drain silently and are held for the default window ([#14856](https://github.com/hivecommons/hive/issues/14856)). Only after - that does the hub record a `stale-upgrade-sweep` and retry. + that does the hub record a `stale-upgrade-sweep` and retry. The whole + lifecycle is also exposed as one state (`scheduled` → `draining` → `pulling` → `rolling` → + `verifying` → `done` | `missed`) on `/api/status.upgradeProgress` and, per + hive, on the hub's `/api/saas/my-hives` (`upgradeState`), with the drain + countdown, the agents whose turns are holding it, and a persisted + `upgradeHistory` of the last 14 attempts and held windows — see + [operator-reference.md](operator-reference.md#scheduled-upgrade-progress-and-history). 5. **Need it now?** Once `stable` itself points at the desired build, **Upgrade now** skips the automatic-upgrade wait for that hive and arms the rollout immediately. It does not promote `stable`; it only follows the channel diff --git a/src/pkg/dashboard/server.go b/src/pkg/dashboard/server.go index 50bcb73d56..a72d811565 100644 --- a/src/pkg/dashboard/server.go +++ b/src/pkg/dashboard/server.go @@ -379,6 +379,10 @@ type Server struct { // one, or forever on a spoke the hub does not manage. Guarded by versionMu. hubUpgradePolicy *spoke.HeartbeatUpgradePolicy hubUpgradePolicyAt time.Time + // upgradeDrain reports the live self-upgrade drain latch (cmd/hive's + // selfUpgradeDrain) so /api/status can show the countdown and the agents + // holding it (#14856). Guarded by versionMu; nil when not installed. + upgradeDrain upgradeDrainSource // hubPushedDashboardURL is the vanity dashboard URL the hub last delivered // on a heartbeat (#7451). Non-empty means the hub owns hub.dashboard_url: // an operator edit would be reverted by the next push, and until then a @@ -3575,11 +3579,13 @@ func (s *Server) handleStatus(w http.ResponseWriter, r *http.Request) { statusJSONResponse(w, r, map[string]string{"status": "initializing"}) return } - payload := statusWithWebhooks{StatusPayload: s.statusWithOverviewBands(status, time.Now().UTC()), Webhooks: github.WebhookHealthSnapshot()} + now := time.Now().UTC() + payload := statusWithWebhooks{StatusPayload: s.statusWithOverviewBands(status, now), Webhooks: github.WebhookHealthSnapshot()} if s.deps != nil && s.deps.EvalLoop != nil { loop := s.deps.EvalLoop() payload.EvalLoopStatus = &loop } + payload.UpgradeProgress = s.upgradeProgress(status, now) statusJSONResponse(w, r, filterStatusPayload(&payload, r.URL.Query().Get("fields"), r.URL.Query().Get("omit"))) } @@ -3590,6 +3596,11 @@ type statusWithWebhooks struct { *StatusPayload Webhooks github.WebhookHealth `json:"webhooks"` *EvalLoopStatus + // UpgradeProgress is the spoke's view of the scheduled-upgrade state + // machine (scheduled / draining / pulling / … / missed) with the drain + // countdown, derived at serve time (#14856). Absent on a hive the hub + // does not manage and that has no upgrade in flight. + UpgradeProgress *spoke.UpgradeProgress `json:"upgradeProgress,omitempty"` } func (s *Server) handleStatusSummary(w http.ResponseWriter, r *http.Request) { diff --git a/src/pkg/dashboard/upgrade_progress.go b/src/pkg/dashboard/upgrade_progress.go new file mode 100644 index 0000000000..ba51d32f6b --- /dev/null +++ b/src/pkg/dashboard/upgrade_progress.go @@ -0,0 +1,223 @@ +package dashboard + +import ( + "strconv" + "time" + + "github.com/hivecommons/hive/pkg/hub/spoke" +) + +// upgradeDrainSource is the cmd/hive hook that exposes the live self-upgrade +// drain latch (selfUpgradeDrain.Report) to the dashboard, so /api/status +// can show the same countdown the hub sees on the heartbeat. +type upgradeDrainSource func(now time.Time) *spoke.UpgradeDrainReport + +// SetUpgradeDrainSource installs the drain reporter. nil disables it. +func (s *Server) SetUpgradeDrainSource(fn func(now time.Time) *spoke.UpgradeDrainReport) { + s.versionMu.Lock() + s.upgradeDrain = fn + s.versionMu.Unlock() +} + +// upgradeProgressInput is what buildUpgradeProgress derives from, gathered +// at serve time so the derivation is pure and table-testable. +type upgradeProgressInput struct { + Policy *spoke.HeartbeatUpgradePolicy + Drain *spoke.UpgradeDrainReport + Marker map[string]any + Outcome *upgradeOutcome + RunningSHA string + AgentsTotal int + // StartedAt is process start — an outcome that completed on this boot + // is the upgrade that just landed. + StartedAt time.Time + Now time.Time +} + +// buildUpgradeProgress derives the spoke's view of the upgrade state machine +// from what it knows locally — the drain latch, the self-upgrade marker, the +// last landed outcome — plus the hub's policy (next window, history). It +// uses the same state vocabulary as the hub card (#14856). +// +// drain open → draining (countdown, agents in turn) +// marker, attempts exhausted → missed (last error) +// marker, retrying → pulling +// outcome landed this boot → done +// hub history: recent row → that row's outcome (missed window etc.) +// policy with a schedule → scheduled (next window / next poll) +// no policy → nil (manual, nothing to show) +func buildUpgradeProgress(in upgradeProgressInput) *spoke.UpgradeProgress { + now := in.Now + if now.IsZero() { + now = time.Now() + } + rfc := func(t time.Time) string { + if t.IsZero() { + return "" + } + return t.UTC().Format(time.RFC3339) + } + nextWindow := "" + var history []spoke.UpgradeAttempt + if in.Policy != nil { + nextWindow = in.Policy.NextWindowAt + history = in.Policy.UpgradeHistory + } + + if d := in.Drain; d != nil { + turns := d.Turns + if len(turns) == 0 { + turns = spoke.TurnsFromNames(d.InFlight) + } + remaining := time.Duration(d.RemainingS) * time.Second + p := &spoke.UpgradeProgress{ + State: spoke.UpgradeStateDraining, + Target: shortSHADashboard(d.Target), + From: shortSHADashboard(in.RunningSHA), + StartedAt: rfc(now.Add(-time.Duration(d.WaitedS) * time.Second)), + DrainDeadline: rfc(now.Add(remaining)), + DrainRemainingS: d.RemainingS, + AgentsInTurn: turns, + AgentsTotal: in.AgentsTotal, + NextWindowAt: nextWindow, + } + p.Detail = "draining: " + spoke.FormatTurns(turns, in.AgentsTotal) + + " — proceeds in " + spoke.FormatShortDuration(remaining) + return p + } + + if m := in.Marker; m != nil { + target, _ := m["target"].(string) + current, _ := m["current"].(string) + requestedAt, _ := m["requestedAt"].(string) + lastErr, _ := m["lastError"].(string) + attempts, _ := m["attempts"].(int) + maxAttempts, _ := m["maxAttempts"].(int) + failed, _ := m["failed"].(bool) + p := &spoke.UpgradeProgress{ + Target: shortSHADashboard(target), + From: shortSHADashboard(current), + StartedAt: requestedAt, + NextWindowAt: nextWindow, + Reason: lastErr, + } + if failed { + p.State = spoke.UpgradeStateMissed + if p.Reason == "" { + p.Reason = "image pull or restart failed" + } + p.Detail = "upgrade failed after " + itoaDashboard(attempts) + " attempts: " + p.Reason + return p + } + p.State = spoke.UpgradeStatePulling + p.Detail = "pulling " + orDashDashboard(p.Target) + " (attempt " + itoaDashboard(attempts) + "/" + itoaDashboard(maxAttempts) + ")" + return p + } + + if o := in.Outcome; o != nil && !o.CompletedAt.IsZero() && !in.StartedAt.IsZero() && !o.CompletedAt.Before(in.StartedAt.Add(-time.Minute)) { + return &spoke.UpgradeProgress{ + State: spoke.UpgradeStateDone, + Target: shortSHADashboard(o.TargetSHA), + From: shortSHADashboard(o.CurrentSHA), + StartedAt: rfc(o.RequestedAt), + EndedAt: rfc(o.CompletedAt), + NextWindowAt: nextWindow, + Detail: "upgraded to " + orDashDashboard(shortSHADashboard(o.TargetSHA)) + " " + spoke.FormatShortDuration(now.Sub(o.CompletedAt)) + " ago", + } + } + + if last, ok := latestClosedAttempt(history); ok && now.Sub(last.EndedAt) <= upgradeHistoryRecentFor { + p := &spoke.UpgradeProgress{ + State: last.Outcome, + Target: last.Target, + From: last.From, + StartedAt: rfc(last.StartedAt), + EndedAt: rfc(last.EndedAt), + NextWindowAt: nextWindow, + Reason: last.Reason, + } + ago := spoke.FormatShortDuration(now.Sub(last.EndedAt)) + " ago" + switch last.Outcome { + case spoke.UpgradeOutcomeDone: + p.Detail = "upgraded to " + orDashDashboard(last.Target) + " " + ago + default: + if last.Trigger == spoke.UpgradeTriggerWindow { + p.Detail = "missed window " + ago + ": " + last.Reason + } else { + p.Detail = "missed " + ago + ": " + last.Reason + } + } + return p + } + + if in.Policy == nil || (!in.Policy.HubManaged && !in.Policy.SpokeManaged) { + return nil + } + p := &spoke.UpgradeProgress{State: spoke.UpgradeStateScheduled, From: shortSHADashboard(in.RunningSHA), NextWindowAt: nextWindow} + switch { + case in.Policy.Paused: + p.Detail = "upgrades paused" + case nextWindow != "": + p.Detail = "next window " + nextWindow + default: + p.Detail = "upgrades on the hub's next poll" + } + return p +} + +// upgradeHistoryRecentFor mirrors the hub's window for showing the last +// attempt's outcome as the current state; after it the pill falls back to +// "scheduled". +const upgradeHistoryRecentFor = 24 * time.Hour + +// latestClosedAttempt returns the most recently ended row of a history. +func latestClosedAttempt(history []spoke.UpgradeAttempt) (spoke.UpgradeAttempt, bool) { + var best spoke.UpgradeAttempt + found := false + for _, a := range history { + if a.Open() || a.EndedAt.IsZero() { + continue + } + if !found || a.EndedAt.After(best.EndedAt) { + best, found = a, true + } + } + return best, found +} + +// upgradeProgress gathers the live inputs and derives the state at serve +// time, so the drain countdown is never a cached eval cycle stale. +func (s *Server) upgradeProgress(status *StatusPayload, now time.Time) *spoke.UpgradeProgress { + s.versionMu.RLock() + policy := s.hubUpgradePolicy + drainFn := s.upgradeDrain + s.versionMu.RUnlock() + in := upgradeProgressInput{ + Policy: policy, + Marker: readUpgradeMarker(), + Outcome: readUpgradeOutcome(), + StartedAt: s.startedAt, + Now: now, + } + if drainFn != nil { + in.Drain = drainFn(now) + } + if status != nil { + in.AgentsTotal = len(status.Agents) + } + if versionHash != "unknown" { + in.RunningSHA = versionHash + } + return buildUpgradeProgress(in) +} + +func itoaDashboard(n int) string { + return strconv.Itoa(n) +} + +func orDashDashboard(s string) string { + if s == "" { + return "—" + } + return s +} diff --git a/src/pkg/dashboard/upgrade_progress_test.go b/src/pkg/dashboard/upgrade_progress_test.go new file mode 100644 index 0000000000..8fd681f2ea --- /dev/null +++ b/src/pkg/dashboard/upgrade_progress_test.go @@ -0,0 +1,166 @@ +package dashboard + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/hub/spoke" +) + +// #14856: the spoke dashboard derives the same upgrade state vocabulary the +// hub card uses, from the drain latch, the self-upgrade marker, the landed +// outcome and the hub's policy. +func TestBuildUpgradeProgressStates(t *testing.T) { + now := time.Date(2026, 10, 26, 17, 10, 0, 0, time.UTC) + next := now.Add(24 * time.Hour).Format(time.RFC3339) + managed := &spoke.HeartbeatUpgradePolicy{HubManaged: true, Schedule: "daily", NextWindowAt: next} + + cases := []struct { + name string + in upgradeProgressInput + state string + detail string + nilOK bool + }{ + { + name: "drain open", + in: upgradeProgressInput{Policy: managed, AgentsTotal: 5, RunningSHA: "551ffc34f", + Drain: &spoke.UpgradeDrainReport{Target: "abc1234", WaitedS: 300, RemainingS: 2400, + Turns: []spoke.UpgradeTurn{{Agent: "reviewer", ElapsedS: 38 * 60}, {Agent: "scanner", ElapsedS: 12 * 60}}}}, + state: spoke.UpgradeStateDraining, detail: "2/5 agents in turn: reviewer 38m, scanner 12m — proceeds in 40m", + }, + { + name: "marker exhausted", + in: upgradeProgressInput{Policy: managed, Marker: map[string]any{"target": "abc1234", "current": "551ffc3", + "attempts": 5, "maxAttempts": 5, "failed": true, "lastError": "ErrImagePull"}}, + state: spoke.UpgradeStateMissed, detail: "ErrImagePull", + }, + { + name: "marker retrying", + in: upgradeProgressInput{Policy: managed, Marker: map[string]any{"target": "abc1234", "current": "551ffc3", + "attempts": 2, "maxAttempts": 5, "failed": false}}, + state: spoke.UpgradeStatePulling, detail: "attempt 2/5", + }, + { + name: "outcome landed this boot", + in: upgradeProgressInput{Policy: managed, StartedAt: now.Add(-10 * time.Minute), + Outcome: &upgradeOutcome{TargetSHA: "abc1234", CurrentSHA: "551ffc3", RequestedAt: now.Add(-time.Hour), CompletedAt: now.Add(-9 * time.Minute)}}, + state: spoke.UpgradeStateDone, detail: "upgraded to abc1234 9m ago", + }, + { + name: "old outcome ignored, hub history missed window shown", + in: upgradeProgressInput{StartedAt: now.Add(-10 * time.Minute), + Outcome: &upgradeOutcome{TargetSHA: "abc1234", CompletedAt: now.Add(-48 * time.Hour)}, + Policy: &spoke.HeartbeatUpgradePolicy{HubManaged: true, NextWindowAt: next, UpgradeHistory: []spoke.UpgradeAttempt{ + {Trigger: spoke.UpgradeTriggerWindow, Outcome: spoke.UpgradeOutcomeMissed, Reason: "spoke still draining", + StartedAt: now.Add(-3 * time.Hour), EndedAt: now.Add(-3 * time.Hour)}}}}, + state: spoke.UpgradeStateMissed, detail: "missed window 3h0m ago: spoke still draining", + }, + { + name: "scheduled with window", + in: upgradeProgressInput{Policy: managed}, + state: spoke.UpgradeStateScheduled, detail: "next window " + next, + }, + { + name: "paused", + in: upgradeProgressInput{Policy: &spoke.HeartbeatUpgradePolicy{HubManaged: true, Paused: true}}, + state: spoke.UpgradeStateScheduled, detail: "paused", + }, + { + name: "spoke managed instant", + in: upgradeProgressInput{Policy: &spoke.HeartbeatUpgradePolicy{SpokeManaged: true}}, + state: spoke.UpgradeStateScheduled, detail: "next poll", + }, + {name: "unmanaged", in: upgradeProgressInput{}, nilOK: true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + tc.in.Now = now + p := buildUpgradeProgress(tc.in) + if tc.nilOK { + if p != nil { + t.Fatalf("want nil, got %+v", p) + } + return + } + if p == nil { + t.Fatal("nil progress") + } + if p.State != tc.state { + t.Fatalf("state = %q want %q (%+v)", p.State, tc.state, p) + } + if !strings.Contains(p.Detail, tc.detail) { + t.Fatalf("detail %q lacks %q", p.Detail, tc.detail) + } + if p.State == spoke.UpgradeStateDraining { + if p.DrainRemainingS != 2400 || p.DrainDeadline != now.Add(40*time.Minute).Format(time.RFC3339) { + t.Fatalf("drain countdown wrong: %+v", p) + } + if p.NextWindowAt != next { + t.Fatal("draining must still carry the next window") + } + } + }) + } +} + +func TestStatusCarriesUpgradeProgress(t *testing.T) { + origMarker, origOutcome := upgradeMarkerPath, upgradeOutcomePath + dir := t.TempDir() + upgradeMarkerPath = filepath.Join(dir, "upgrade-requested") + upgradeOutcomePath = filepath.Join(dir, "last-upgrade-outcome") + t.Cleanup(func() { upgradeMarkerPath, upgradeOutcomePath = origMarker, origOutcome }) + if err := os.WriteFile(upgradeMarkerPath, []byte(`{"target_sha":"abc1234","current_sha":"551ffc3","attempts":1}`), 0o600); err != nil { + t.Fatal(err) + } + + s := &Server{status: &StatusPayload{}, startedAt: time.Now()} + s.SetHubUpgradePolicy(&spoke.HeartbeatUpgradePolicy{HubManaged: true, NextWindowAt: "2026-10-27T17:00:00Z"}) + drainCalls := 0 + s.SetUpgradeDrainSource(func(now time.Time) *spoke.UpgradeDrainReport { + drainCalls++ + return &spoke.UpgradeDrainReport{Target: "abc1234", RemainingS: 90, InFlight: []string{"reviewer"}} + }) + + rec := httptest.NewRecorder() + s.handleStatus(rec, httptest.NewRequest(http.MethodGet, "/api/status", nil)) + if rec.Code != http.StatusOK { + t.Fatalf("status %d: %s", rec.Code, rec.Body.String()) + } + var body struct { + UpgradeProgress *spoke.UpgradeProgress `json:"upgradeProgress"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil { + t.Fatal(err) + } + if drainCalls != 1 { + t.Fatalf("drain source called %d times, want 1 (serve-time)", drainCalls) + } + // The live drain outranks the persisted marker. + if body.UpgradeProgress == nil || body.UpgradeProgress.State != spoke.UpgradeStateDraining { + t.Fatalf("upgradeProgress = %+v", body.UpgradeProgress) + } + if body.UpgradeProgress.DrainRemainingS != 90 || len(body.UpgradeProgress.AgentsInTurn) != 1 || body.UpgradeProgress.AgentsInTurn[0].Agent != "reviewer" { + t.Fatalf("drain fields = %+v", body.UpgradeProgress) + } + if body.UpgradeProgress.NextWindowAt != "2026-10-27T17:00:00Z" { + t.Fatalf("nextWindowAt = %q", body.UpgradeProgress.NextWindowAt) + } + + // Without the drain the marker speaks. + s.SetUpgradeDrainSource(nil) + rec = httptest.NewRecorder() + s.handleStatus(rec, httptest.NewRequest(http.MethodGet, "/api/status?fields=upgradeProgress", nil)) + if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil { + t.Fatal(err) + } + if body.UpgradeProgress == nil || body.UpgradeProgress.State != spoke.UpgradeStatePulling { + t.Fatalf("marker-only upgradeProgress = %+v", body.UpgradeProgress) + } +} diff --git a/src/pkg/hub/heartbeat.go b/src/pkg/hub/heartbeat.go index 4553c2498f..ddfe268f81 100644 --- a/src/pkg/hub/heartbeat.go +++ b/src/pkg/hub/heartbeat.go @@ -7,6 +7,7 @@ import ( "github.com/hivecommons/hive/crux" "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/hub/spoke" "github.com/hivecommons/hive/pkg/inferencehealth" "github.com/hivecommons/hive/pkg/nous" "github.com/hivecommons/hive/pkg/tracing" @@ -179,6 +180,9 @@ type UpgradeDrainReport struct { InFlight []string `json:"in_flight,omitempty"` WaitedS int `json:"waited_s"` RemainingS int `json:"remaining_s"` + // Turns mirrors spoke.UpgradeDrainReport.Turns: per-agent turn elapsed + // for the InFlight agents. Absent from older spokes. + Turns []spoke.UpgradeTurn `json:"turns,omitempty"` } // HeartbeatClusterHealthReport contains cluster node and GPU metrics @@ -525,6 +529,13 @@ type HeartbeatUpgradePolicy struct { // instead of only showing the previous upgrade. ScheduledHoldReason string `json:"scheduled_hold_reason,omitempty"` ScheduledHoldAt string `json:"scheduled_hold_at,omitempty"` + // NextWindowAt is this hive's OWN next daily/weekly window (RFC3339 + // UTC), sent regardless of channel so a :stable hive can show when it + // will next be considered rather than only the channel's promotion ETA. + NextWindowAt string `json:"next_window_at,omitempty"` + // UpgradeHistory is the hub's persisted record of this hive's recent + // upgrade attempts and missed windows, newest last (#14856). + UpgradeHistory []spoke.UpgradeAttempt `json:"upgrade_history,omitempty"` } type TaskStatusPayload struct { diff --git a/src/pkg/hub/heartbeat_upgrade_policy.go b/src/pkg/hub/heartbeat_upgrade_policy.go index 7984a59f02..db22ec7ee5 100644 --- a/src/pkg/hub/heartbeat_upgrade_policy.go +++ b/src/pkg/hub/heartbeat_upgrade_policy.go @@ -50,10 +50,21 @@ func (s *HubServer) heartbeatUpgradePolicy(payload *HeartbeatPayload, saasHive * } else if schedule == AutoUpgradeModeDaily || schedule == AutoUpgradeModeWeekly { policy.ScheduleHour = autoUpgradeDailyHour policy.ScheduleTimezone = autoUpgradeTimezone + lastFired := "" + if saasHive != nil { + lastFired = saasHive.AutoUpgradeLastFired + } + // NextWindowAt is always the hive's own window regardless of channel — + // NextUpdateAt on the stable channel is the promotion ETA instead, which + // is what made "overdue since" read as a mystery time (#14856). + policy.NextWindowAt = nextScheduledAutoUpgradeAt(schedule, lastFired, time.Now()) if policy.NextUpdateAt == "" && reach.Channel != ReleaseChannelStable { - policy.NextUpdateAt = nextScheduledAutoUpgradeAt(schedule, saasHive.AutoUpgradeLastFired, time.Now()) + policy.NextUpdateAt = policy.NextWindowAt } } + if payload != nil && s.upgradeHistory != nil { + policy.UpgradeHistory = s.upgradeHistory.recent(payload.HiveID) + } if schedule == AutoUpgradeModeWeekly { policy.ScheduleWeekday = autoUpgradeWeeklyDay.String() } diff --git a/src/pkg/hub/saas_hives_api.go b/src/pkg/hub/saas_hives_api.go index 2bd77c593d..013a0d2c89 100644 --- a/src/pkg/hub/saas_hives_api.go +++ b/src/pkg/hub/saas_hives_api.go @@ -13,6 +13,7 @@ import ( "time" "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/hub/spoke" ) type MyHiveEntry struct { @@ -46,10 +47,18 @@ type MyHiveEntry struct { AutoUpgradeMode string `json:"autoUpgradeMode,omitempty"` // AutoUpgradeSchedule is the EFFECTIVE schedule: the configured mode, or // "candidate-bypass" when a candidate-channel hive ignores the window. - AutoUpgradeSchedule string `json:"autoUpgradeSchedule,omitempty"` - NextUpdateAt string `json:"nextUpdateAt,omitempty"` - NextUpdateStatus string `json:"nextUpdateStatus,omitempty"` - UpgradeWhyNotNow string `json:"upgradeWhyNotNow,omitempty"` + AutoUpgradeSchedule string `json:"autoUpgradeSchedule,omitempty"` + NextUpdateAt string `json:"nextUpdateAt,omitempty"` + NextUpdateStatus string `json:"nextUpdateStatus,omitempty"` + UpgradeWhyNotNow string `json:"upgradeWhyNotNow,omitempty"` + // UpgradeState is the derived upgrade state machine for this hive + // (scheduled / draining / pulling / rolling / verifying / done / missed) + // with the drain countdown and the agents holding it; UpgradeHistory is + // the persisted record of recent attempts and held windows. Both are + // computed on read by upgradeProgressFor so the card and the spoke + // dashboard speak the same words (#14856). + UpgradeState *spoke.UpgradeProgress `json:"upgradeState,omitempty"` + UpgradeHistory []spoke.UpgradeAttempt `json:"upgradeHistory,omitempty"` PendingRequestCount int `json:"pendingRequestCount,omitempty"` PendingRequests []PendingAccessRequest `json:"pending_requests,omitempty"` @@ -1007,6 +1016,8 @@ func (s *HubServer) buildMyHivesResponse(username string, queryValues url.Values } } + result[i].UpgradeState, result[i].UpgradeHistory = s.upgradeProgressForRow(&result[i].RegistryEntry, saasByID[result[i].ID], result[i].TrackedChannel, journeyNow) + st := s.journey.get(result[i].ID) status := JourneyStatusFor(&result[i].RegistryEntry, st, journeyNow) result[i].Journey = &status @@ -1771,6 +1782,7 @@ func (s *HubServer) handleDeleteHive(w http.ResponseWriter, r *http.Request) { // handler. So purging the record here cannot break placeholder recycling. removeHiveRecord(id, s.logger) s.removeRegistryEntry(id, username) + s.upgradeHistory.forget(id) s.logger.Info("audit: hosted hive deleted", "hive_id", id, "by", username, "deprovisioned", cluster != nil) diff --git a/src/pkg/hub/saas_upgrade.go b/src/pkg/hub/saas_upgrade.go index da1645c01f..1fae271dda 100644 --- a/src/pkg/hub/saas_upgrade.go +++ b/src/pkg/hub/saas_upgrade.go @@ -8,6 +8,8 @@ import ( "os" "strings" "time" + + "github.com/hivecommons/hive/pkg/hub/spoke" ) // hubUpgradeDebounce is the minimum gap between hub self-upgrade rollout @@ -613,6 +615,7 @@ func (s *HubServer) handleUpgradeHive(w http.ResponseWriter, r *http.Request) { s.logger.Info("audit: hosted hive upgrade requested", "hive_id", id, "by", username, "cluster", cluster.ID, "mode", mode) s.recordTimeline(id, TimelineUpgradeStarted, "upgrade requested from the hub dashboard ("+mode+")", username) + s.noteUpgradeStarted(id, currentSHA, latestSHA, spoke.UpgradeTriggerManual) w.Header().Set("Content-Type", "application/json") _, _ = w.Write([]byte(`{"status":"upgrading","mode":"` + mode + `"}`)) } @@ -1086,6 +1089,7 @@ func (s *HubServer) triggerAutoUpgrades() { holdScheduled := func(reason string) { if h.AutoUpgrade && scheduledUpgradeWindowOpen(h.AutoUpgradeMode, h.AutoUpgradeLastFired, holdNow) { s.noteScheduledUpgradeHold(&h, reason, holdNow) + s.noteUpgradeWindowMissed(h.ID, currentSHA, reason, holdNow) } } if alreadyUpgrading { @@ -1530,6 +1534,7 @@ func (s *HubServer) triggerAutoUpgrades() { "schedule", effectiveAutoUpgradeSchedule(h.AutoUpgradeMode, reach.Channel), "reason", decision.Reason) s.recordTimeline(h.ID, TimelineUpgradeStarted, fmt.Sprintf("auto-upgrade triggered on %s: %s → %s", branch, orDash(currentSHA), latestSHA), "auto-upgrade") + s.noteUpgradeStarted(h.ID, currentSHA, latestSHA, upgradeTriggerForSchedule(h.AutoUpgradeMode, reach.Channel)) s.mu.Lock() for i := range s.registry.Hives { if s.registry.Hives[i].ID == h.ID { diff --git a/src/pkg/hub/server.go b/src/pkg/hub/server.go index 59a742f979..418196d0f7 100644 --- a/src/pkg/hub/server.go +++ b/src/pkg/hub/server.go @@ -1364,6 +1364,10 @@ type HubServer struct { // leaf mutex and is never guarded by s.mu. See timeline.go. timeline *timelineStore + // upgradeHistory is the per-hive record of upgrade attempts and held + // windows with their outcomes (#14856). Leaf mutex; see upgrade_history.go. + upgradeHistory *upgradeHistoryStore + // journey holds per-hive user-journey nudge state: which banner each owner // was last sent and when, plus admin snoozes. Persisted to the PVC. journey *journeyStore @@ -1570,6 +1574,7 @@ func NewHubServer(port int, logger *slog.Logger, gitHash, gitBranch string) *Hub topRepoRefreshEnabled: true, authRolloutSeen: make(map[string]authRolloutEntry), timeline: newTimelineStore(), + upgradeHistory: newUpgradeHistoryStore(), journey: newJourneyStore(), alerts: newAlertState(), revokedSessions: newRevokedSessions(), @@ -1676,6 +1681,7 @@ func NewHubServer(port int, logger *slog.Logger, gitHash, gitBranch string) *Hub s.recoverArmedUpgrades() s.loadHubBanners() s.timeline.load() + s.upgradeHistory.load() if err := s.journey.load(time.Now()); err != nil { // Losing journey state is a soft failure: the worst case is that a few // owners get one duplicate nudge. Never block hub startup on it. diff --git a/src/pkg/hub/spoke/heartbeat.go b/src/pkg/hub/spoke/heartbeat.go index 48c5ec8085..c989143ae1 100644 --- a/src/pkg/hub/spoke/heartbeat.go +++ b/src/pkg/hub/spoke/heartbeat.go @@ -575,6 +575,10 @@ type UpgradeDrainReport struct { InFlight []string `json:"in_flight,omitempty"` WaitedS int `json:"waited_s"` RemainingS int `json:"remaining_s"` + // Turns carries, for each InFlight agent, how long its current turn has + // been running, so the hub can say "reviewer 38m" rather than just + // "reviewer". Spokes that predate the field send only InFlight. + Turns []UpgradeTurn `json:"turns,omitempty"` } // HeartbeatClusterHealthReport contains cluster node and GPU metrics @@ -2901,6 +2905,14 @@ type HeartbeatUpgradePolicy struct { // it was recorded. Empty when nothing is held or the hub is too old to say. ScheduledHoldReason string `json:"scheduled_hold_reason,omitempty"` ScheduledHoldAt string `json:"scheduled_hold_at,omitempty"` + // NextWindowAt is this hive's OWN next daily/weekly window (RFC3339 + // UTC), independent of the channel promotion estimate in NextUpdateAt. + // Empty for instant/manual policies and for hubs too old to send it. + NextWindowAt string `json:"next_window_at,omitempty"` + // UpgradeHistory is the hub's record of this hive's recent upgrade + // attempts and missed windows, newest last, so the spoke dashboard can + // show why a window did not land without asking the hub. + UpgradeHistory []UpgradeAttempt `json:"upgrade_history,omitempty"` } // Schedule values carried by HeartbeatUpgradePolicy.Schedule. These mirror diff --git a/src/pkg/hub/spoke/upgrade_progress.go b/src/pkg/hub/spoke/upgrade_progress.go new file mode 100644 index 0000000000..b0ffc69496 --- /dev/null +++ b/src/pkg/hub/spoke/upgrade_progress.go @@ -0,0 +1,183 @@ +package spoke + +import ( + "fmt" + "sort" + "strings" + "time" +) + +// Upgrade progress states. One scheduled upgrade walks +// scheduled → draining → pulling → rolling → verifying → done, or ends in +// missed. The hub derives the state from its registry latch plus the spoke's +// heartbeat; the spoke derives the same vocabulary from what it knows locally +// (its drain latch, upgrade marker and the hub's policy), so both dashboards +// speak the same words about the same upgrade. +const ( + // UpgradeStateScheduled — nothing in flight; NextWindowAt says when the + // hub will next consider this hive (or "instant"/"next poll"). + UpgradeStateScheduled = "scheduled" + // UpgradeStateDraining — the spoke received the instruction and is + // waiting for in-flight agent turns before it restarts. + UpgradeStateDraining = "draining" + // UpgradeStatePulling — the instruction was delivered and the spoke is + // (or should be) acting on it: patching its Deployment, waiting for the + // image, or on an older build draining without reporting it. + UpgradeStatePulling = "pulling" + // UpgradeStateRolling — the spoke has gone quiet since the instruction: + // the pod is restarting onto the new image. + UpgradeStateRolling = "rolling" + // UpgradeStateVerifying — the spoke is back and reports the target + // commit, but the hub has not yet closed the attempt. + UpgradeStateVerifying = "verifying" + // UpgradeStateDone — the most recent attempt landed. + UpgradeStateDone = "done" + // UpgradeStateMissed — the most recent attempt or window did not land; + // Reason says why (drain lapsed, hub restarted, unreachable, image pull + // failed, held by the schedule gate, …). + UpgradeStateMissed = "missed" +) + +// Upgrade attempt outcomes recorded in UpgradeAttempt.Outcome. +const ( + UpgradeOutcomeDone = "done" + UpgradeOutcomeMissed = "missed" +) + +// Upgrade attempt triggers recorded in UpgradeAttempt.Trigger. +const ( + UpgradeTriggerScheduled = "scheduled" + UpgradeTriggerInstant = "instant" + UpgradeTriggerManual = "manual" + // UpgradeTriggerObserved — the hub saw the spoke upgrading without + // having armed it (a spoke-managed or operator-restarted rollout). + UpgradeTriggerObserved = "observed" + // UpgradeTriggerWindow — not an attempt at all: a daily/weekly window + // opened and the hub held the upgrade instead of firing it. + UpgradeTriggerWindow = "window" +) + +// UpgradeTurn is one agent whose kicked turn is holding a drain, with how +// long that turn has been running. +type UpgradeTurn struct { + Agent string `json:"agent"` + ElapsedS int `json:"elapsedS"` +} + +// UpgradeAttempt is one row of a hive's upgrade history: what was attempted, +// what triggered it, and how it ended. The hub persists the last N per hive +// so a missed window is still visible the next morning. +type UpgradeAttempt struct { + Target string `json:"target,omitempty"` + From string `json:"from,omitempty"` + Trigger string `json:"trigger,omitempty"` + StartedAt time.Time `json:"startedAt"` + EndedAt time.Time `json:"endedAt,omitempty"` + // Outcome is "" while the attempt is still open. + Outcome string `json:"outcome,omitempty"` + Reason string `json:"reason,omitempty"` +} + +// Open reports whether the attempt has not ended yet. +func (a UpgradeAttempt) Open() bool { return a.Outcome == "" } + +// UpgradeProgress is the derived, render-ready picture of one hive's upgrade +// state machine. Every field is additive; consumers must tolerate absent ones. +type UpgradeProgress struct { + State string `json:"state"` + Target string `json:"target,omitempty"` + From string `json:"from,omitempty"` + // StartedAt is when the current attempt was instructed (RFC3339 UTC). + StartedAt string `json:"startedAt,omitempty"` + // NextWindowAt is the hive's own next scheduled window (RFC3339 UTC) — + // the daily/weekly slot, never the channel's promotion estimate. + NextWindowAt string `json:"nextWindowAt,omitempty"` + // DrainDeadline is when a draining spoke will proceed regardless + // (RFC3339 UTC); DrainRemainingS is the same as a countdown. + DrainDeadline string `json:"drainDeadline,omitempty"` + DrainRemainingS int `json:"drainRemainingS,omitempty"` + // AgentsInTurn names the agents holding the drain; AgentsTotal is the + // hive's agent count so the UI can say "2/5 agents in turn". + AgentsInTurn []UpgradeTurn `json:"agentsInTurn,omitempty"` + AgentsTotal int `json:"agentsTotal,omitempty"` + // Reason explains a missed state, or qualifies an in-flight one. + Reason string `json:"reason,omitempty"` + // Detail is the one-line human summary the pill shows. + Detail string `json:"detail"` + // EndedAt is when a done/missed attempt closed (RFC3339 UTC). + EndedAt string `json:"endedAt,omitempty"` +} + +// InFlight reports whether the state describes an attempt still under way. +func (p *UpgradeProgress) InFlight() bool { + if p == nil { + return false + } + switch p.State { + case UpgradeStateDraining, UpgradeStatePulling, UpgradeStateRolling, UpgradeStateVerifying: + return true + } + return false +} + +// FormatTurns renders "2/5 agents in turn: reviewer 38m, scanner 12m". +// Longest-running turn first; total is omitted when unknown (0). +func FormatTurns(turns []UpgradeTurn, total int) string { + if len(turns) == 0 { + return "no agents in turn" + } + sorted := append([]UpgradeTurn(nil), turns...) + sort.SliceStable(sorted, func(i, j int) bool { + if sorted[i].ElapsedS != sorted[j].ElapsedS { + return sorted[i].ElapsedS > sorted[j].ElapsedS + } + return sorted[i].Agent < sorted[j].Agent + }) + parts := make([]string, 0, len(sorted)) + for _, t := range sorted { + if t.ElapsedS > 0 { + parts = append(parts, t.Agent+" "+FormatShortDuration(time.Duration(t.ElapsedS)*time.Second)) + } else { + parts = append(parts, t.Agent) + } + } + count := fmt.Sprintf("%d", len(turns)) + if total > 0 { + count = fmt.Sprintf("%d/%d", len(turns), total) + } + noun := "agents" + if len(turns) == 1 { + noun = "agent" + } + return count + " " + noun + " in turn: " + strings.Join(parts, ", ") +} + +// FormatShortDuration renders a duration the way the pills do: "45s", "38m", +// "1h12m". Negative durations read as zero. +func FormatShortDuration(d time.Duration) string { + if d < 0 { + d = 0 + } + sec := int(d / time.Second) + if sec < 60 { + return fmt.Sprintf("%ds", sec) + } + min := sec / 60 + if min < 60 { + return fmt.Sprintf("%dm", min) + } + return fmt.Sprintf("%dh%dm", min/60, min%60) +} + +// TurnsFromNames builds turns with unknown elapsed from a bare name list — +// the shape older spokes report in UpgradeDrainReport.InFlight. +func TurnsFromNames(names []string) []UpgradeTurn { + if len(names) == 0 { + return nil + } + out := make([]UpgradeTurn, 0, len(names)) + for _, n := range names { + out = append(out, UpgradeTurn{Agent: n}) + } + return out +} diff --git a/src/pkg/hub/spoke/upgrade_progress_test.go b/src/pkg/hub/spoke/upgrade_progress_test.go new file mode 100644 index 0000000000..b295b2c650 --- /dev/null +++ b/src/pkg/hub/spoke/upgrade_progress_test.go @@ -0,0 +1,44 @@ +package spoke + +import ( + "testing" + "time" +) + +func TestFormatTurns(t *testing.T) { + turns := []UpgradeTurn{{Agent: "scanner", ElapsedS: 12 * 60}, {Agent: "reviewer", ElapsedS: 38 * 60}, {Agent: "triage"}} + if got := FormatTurns(turns, 5); got != "3/5 agents in turn: reviewer 38m, scanner 12m, triage" { + t.Fatalf("FormatTurns = %q", got) + } + if got := FormatTurns(turns[:1], 0); got != "1 agent in turn: scanner 12m" { + t.Fatalf("FormatTurns single = %q", got) + } + if got := FormatTurns(nil, 3); got != "no agents in turn" { + t.Fatalf("FormatTurns empty = %q", got) + } + if turns[0].Agent != "scanner" { + t.Fatal("FormatTurns must not reorder its input") + } +} + +func TestFormatShortDuration(t *testing.T) { + cases := map[time.Duration]string{-time.Second: "0s", 45 * time.Second: "45s", 38 * time.Minute: "38m", 72 * time.Minute: "1h12m"} + for d, want := range cases { + if got := FormatShortDuration(d); got != want { + t.Fatalf("FormatShortDuration(%s) = %q want %q", d, got, want) + } + } +} + +func TestTurnsFromNames(t *testing.T) { + if TurnsFromNames(nil) != nil { + t.Fatal("nil in, nil out") + } + got := TurnsFromNames([]string{"a", "b"}) + if len(got) != 2 || got[1].Agent != "b" || got[1].ElapsedS != 0 { + t.Fatalf("TurnsFromNames = %+v", got) + } + if !(UpgradeAttempt{}).Open() || (UpgradeAttempt{Outcome: UpgradeOutcomeDone}).Open() { + t.Fatal("Open must key on Outcome") + } +} diff --git a/src/pkg/hub/stale_upgrade.go b/src/pkg/hub/stale_upgrade.go index 30187dfed3..63c04aede9 100644 --- a/src/pkg/hub/stale_upgrade.go +++ b/src/pkg/hub/stale_upgrade.go @@ -546,6 +546,7 @@ func (s *HubServer) sweepOrphanedUpgrades() { "upgrade to "+orDash(c.to)+" had already completed (hive is on "+ orDash(c.from)+") — cleared a stale in-flight latch", "stale-upgrade-sweep") + s.noteUpgradeDone(c.id, "completed; stale latch cleared by the sweep", "", c.from) continue } if c.exhausted { @@ -562,6 +563,8 @@ func (s *HubServer) sweepOrphanedUpgrades() { "upgrade to "+orDash(c.to)+" abandoned after "+itoa(c.sweeps)+ " attempts — hive is still on "+orDash(c.from)+ " and is no longer being retried", "stale-upgrade-sweep") + s.noteUpgradeMissed(c.id, + "abandoned after "+itoa(c.sweeps)+" attempts: "+c.reason, c.from, c.to) continue } s.logger.Warn("cleared orphaned upgrade flag — no attempt in flight", @@ -576,6 +579,8 @@ func (s *HubServer) sweepOrphanedUpgrades() { "upgrade never completed after "+roundedDuration(c.elapsed)+ " — cleared stale in-flight state (still on "+orDash(c.from)+ ", target was "+orDash(c.to)+")", "stale-upgrade-sweep") + s.noteUpgradeMissed(c.id, + "never completed after "+roundedDuration(c.elapsed)+": "+c.reason, c.from, c.to) } if len(swept) > 0 { s.requestSave() diff --git a/src/pkg/hub/timeline.go b/src/pkg/hub/timeline.go index 03a6229cce..940f21593f 100644 --- a/src/pkg/hub/timeline.go +++ b/src/pkg/hub/timeline.go @@ -10,6 +10,8 @@ import ( "strings" "sync" "time" + + "github.com/hivecommons/hive/pkg/hub/spoke" ) // timelineMaxEvents caps the number of events retained per hive. Oldest are @@ -524,12 +526,42 @@ func (s *HubServer) recordHeartbeatTransitions(old, new *RegistryEntry) { return } evs := detectTransitions(old, new, time.Now()) + s.recordUpgradeHistoryTransition(old, new) if len(evs) == 0 { return } s.timeline.appendAll(new.ID, evs) } +// recordUpgradeHistoryTransition mirrors the upgrade lifecycle branch of +// detectTransitions into the upgrade history store (#14856). The hub-armed +// paths open their row themselves; a rollout the hub merely observed opens +// one here. Closing is decided from the same evidence the completion chain +// used: a new hash is a landed upgrade, a spoke-reported failure is a miss, +// and a latch cleared with no movement is the hive already being at target. +func (s *HubServer) recordUpgradeHistoryTransition(old, new *RegistryEntry) { + if s == nil || old == nil || new == nil { + return + } + switch { + case !old.Upgrading && new.Upgrading: + s.noteUpgradeStarted(new.ID, old.GitHash, new.UpgradeTarget, spoke.UpgradeTriggerObserved) + case old.Upgrading && !new.Upgrading: + switch { + case new.UpgradeFailed && !old.UpgradeFailed: + reason := new.UpgradeError + if reason == "" { + reason = "spoke reported the upgrade failed" + } + s.noteUpgradeEnded(new.ID, spoke.UpgradeOutcomeMissed, reason, old.GitHash, old.UpgradeTarget) + case new.GitHash != "" && !sameCommit(old.GitHash, new.GitHash): + s.noteUpgradeEnded(new.ID, spoke.UpgradeOutcomeDone, "", old.GitHash, new.GitHash) + default: + s.noteUpgradeEnded(new.ID, spoke.UpgradeOutcomeDone, "latch cleared — hive already at target", old.GitHash, new.GitHash) + } + } +} + // recordOfflineTransitions is the sweep hook. markStaleHives flips Online to // false for hives that stopped beating; that transition can only be observed // there, because a hive that has gone silent by definition sends no diff --git a/src/pkg/hub/upgrade_history.go b/src/pkg/hub/upgrade_history.go new file mode 100644 index 0000000000..a1d0b4ddf6 --- /dev/null +++ b/src/pkg/hub/upgrade_history.go @@ -0,0 +1,365 @@ +package hub + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/hivecommons/hive/pkg/hub/spoke" +) + +// upgradeHistoryDir is where the hub persists each hive's recent upgrade +// attempts, one file per hive beside the timeline store. A var so tests can +// point it at a temp dir. +var upgradeHistoryDir = "/data/saas/upgrade-history" + +// defaultUpgradeHistoryMax is how many attempts are kept per hive. Two weeks +// of daily windows is enough to answer "when did this hive last actually +// roll, and what happened on the days it did not" without a log dive. +const defaultUpgradeHistoryMax = 14 + +// upgradeHistoryMax is the retention cap, overridable with +// HIVE_HUB_UPGRADE_HISTORY_MAX. +func upgradeHistoryMax() int { + return envInt("HIVE_HUB_UPGRADE_HISTORY_MAX", defaultUpgradeHistoryMax) +} + +// upgradeHistoryRecentFor bounds how long a closed attempt still speaks for +// the hive's current upgrade state: a window missed this morning reads as +// "missed" all day; by the time the next window opens it is history only. +const upgradeHistoryRecentFor = 24 * time.Hour + +// hiveUpgradeHistory is the on-disk shape of one hive's history file. +type hiveUpgradeHistory struct { + HiveID string `json:"hive_id"` + Attempts []spoke.UpgradeAttempt `json:"attempts"` +} + +// upgradeHistoryStore records the outcome of every upgrade attempt and every +// held scheduled window per hive, so a missed window is never silent (#14856: +// three hives missed two daily windows each and nothing on either dashboard +// said so). Like timelineStore its mutex is a LEAF — nothing under it reaches +// for HubServer.mu — and it does its own file I/O, so callers may hold s.mu +// only when they already do I/O there (the hold recorder does). +type upgradeHistoryStore struct { + mu sync.RWMutex + byID map[string][]spoke.UpgradeAttempt +} + +func newUpgradeHistoryStore() *upgradeHistoryStore { + return &upgradeHistoryStore{byID: make(map[string][]spoke.UpgradeAttempt)} +} + +func upgradeHistoryPath(hiveID string) string { + if hiveID == "" || !isValidName(hiveID) || + strings.Contains(hiveID, "..") || strings.ContainsAny(hiveID, `/\`) { + return "" + } + return filepath.Join(upgradeHistoryDir, hiveID+".json") +} + +// recent returns a copy of the hive's attempts, oldest first. +func (u *upgradeHistoryStore) recent(hiveID string) []spoke.UpgradeAttempt { + if u == nil { + return nil + } + u.mu.RLock() + defer u.mu.RUnlock() + src := u.byID[hiveID] + if len(src) == 0 { + return nil + } + return append([]spoke.UpgradeAttempt(nil), src...) +} + +// open returns the hive's currently open attempt, if any. +func (u *upgradeHistoryStore) open(hiveID string) (spoke.UpgradeAttempt, bool) { + if u == nil { + return spoke.UpgradeAttempt{}, false + } + u.mu.RLock() + defer u.mu.RUnlock() + list := u.byID[hiveID] + if n := len(list); n > 0 && list[n-1].Open() { + return list[n-1], true + } + return spoke.UpgradeAttempt{}, false +} + +// start opens an attempt toward target. A retry of an attempt that is already +// open (same hive, still unresolved) does not open a second row: the hub +// re-arms the same instruction every stale cycle, and a re-targeted retry +// (the channel moved mid-attempt) is still the same attempt — the target is +// updated so the row names what finally landed. +func (u *upgradeHistoryStore) start(hiveID, from, target, trigger string, at time.Time) { + if u == nil || hiveID == "" { + return + } + u.mu.Lock() + defer u.mu.Unlock() + list := u.byID[hiveID] + if n := len(list); n > 0 && list[n-1].Open() { + if at.Sub(list[n-1].StartedAt) < upgradeHistoryRecentFor { + if target != "" { + list[n-1].Target = target + } + if list[n-1].From == "" { + list[n-1].From = from + } + u.byID[hiveID] = list + u.persistLocked(hiveID) + return + } + // An attempt left open for a day was closed by a path that does not + // report here (latch dropped on a channel switch, hub state reset…); + // never let it swallow the new attempt. + list[n-1].Outcome = spoke.UpgradeOutcomeMissed + list[n-1].Reason = "superseded by a newer attempt" + list[n-1].EndedAt = at.UTC() + } + list = append(list, spoke.UpgradeAttempt{ + Target: target, + From: from, + Trigger: trigger, + StartedAt: at.UTC(), + }) + u.byID[hiveID] = trimUpgradeHistory(list, upgradeHistoryMax()) + u.persistLocked(hiveID) +} + +// end closes the open attempt with an outcome. With nothing open, a done +// outcome records a one-row attempt (the hub observed a rollout it did not +// arm); a missed outcome with nothing open is dropped — there is no window +// to blame. +func (u *upgradeHistoryStore) end(hiveID, outcome, reason, from, target string, at time.Time) { + if u == nil || hiveID == "" || outcome == "" { + return + } + u.mu.Lock() + defer u.mu.Unlock() + list := u.byID[hiveID] + n := len(list) + if n > 0 && list[n-1].Open() { + a := &list[n-1] + a.Outcome = outcome + a.Reason = reason + a.EndedAt = at.UTC() + if a.Target == "" { + a.Target = target + } + if a.From == "" { + a.From = from + } + u.byID[hiveID] = list + u.persistLocked(hiveID) + return + } + if outcome != spoke.UpgradeOutcomeDone { + return + } + list = append(list, spoke.UpgradeAttempt{ + Target: target, + From: from, + Trigger: spoke.UpgradeTriggerObserved, + StartedAt: at.UTC(), + EndedAt: at.UTC(), + Outcome: outcome, + Reason: reason, + }) + u.byID[hiveID] = trimUpgradeHistory(list, upgradeHistoryMax()) + u.persistLocked(hiveID) +} + +// windowMissed records a daily/weekly window the hub held instead of firing. +// One row per window: a hold reason that changes while the same window is +// open updates the row rather than adding another, and an attempt already +// open for this hive means the window did fire, so nothing is recorded. +func (u *upgradeHistoryStore) windowMissed(hiveID, from, reason string, at time.Time) { + if u == nil || hiveID == "" || reason == "" { + return + } + u.mu.Lock() + defer u.mu.Unlock() + list := u.byID[hiveID] + if n := len(list); n > 0 { + last := &list[n-1] + if last.Open() { + return + } + if last.Trigger == spoke.UpgradeTriggerWindow && sameUpgradeWindow(last.StartedAt, at) { + last.Reason = reason + last.EndedAt = at.UTC() + u.byID[hiveID] = list + u.persistLocked(hiveID) + return + } + } + list = append(list, spoke.UpgradeAttempt{ + From: from, + Trigger: spoke.UpgradeTriggerWindow, + StartedAt: at.UTC(), + EndedAt: at.UTC(), + Outcome: spoke.UpgradeOutcomeMissed, + Reason: reason, + }) + u.byID[hiveID] = trimUpgradeHistory(list, upgradeHistoryMax()) + u.persistLocked(hiveID) +} + +// forget drops a hive's history (hive deleted). +func (u *upgradeHistoryStore) forget(hiveID string) { + if u == nil { + return + } + u.mu.Lock() + delete(u.byID, hiveID) + u.mu.Unlock() + if p := upgradeHistoryPath(hiveID); p != "" { + _ = os.Remove(p) + } +} + +// sameUpgradeWindow reports whether two instants fall in the same scheduled +// window: the same calendar day in the schedule's timezone. +func sameUpgradeWindow(a, b time.Time) bool { + loc, err := autoUpgradeLocation() + if err != nil { + loc = time.UTC + } + ay, am, ad := a.In(loc).Date() + by, bm, bd := b.In(loc).Date() + return ay == by && am == bm && ad == bd +} + +func trimUpgradeHistory(list []spoke.UpgradeAttempt, max int) []spoke.UpgradeAttempt { + if max < 1 { + max = 1 + } + if len(list) <= max { + return list + } + trimmed := make([]spoke.UpgradeAttempt, max) + copy(trimmed, list[len(list)-max:]) + return trimmed +} + +// persistLocked writes one hive's history atomically. Caller holds u.mu. +// Failures are swallowed: history must never fail a heartbeat or a sweep. +func (u *upgradeHistoryStore) persistLocked(hiveID string) { + path := upgradeHistoryPath(hiveID) + if path == "" { + return + } + if err := os.MkdirAll(upgradeHistoryDir, 0o755); err != nil { + return + } + data, err := json.MarshalIndent(hiveUpgradeHistory{HiveID: hiveID, Attempts: u.byID[hiveID]}, "", " ") + if err != nil { + return + } + tmp := path + ".tmp" + if err := os.WriteFile(tmp, data, 0o644); err != nil { + return + } + if err := os.Rename(tmp, path); err != nil { + _ = os.Remove(tmp) + } +} + +// load reads every persisted history back at hub startup. +func (u *upgradeHistoryStore) load() { + entries, err := os.ReadDir(upgradeHistoryDir) + if err != nil { + return + } + max := upgradeHistoryMax() + u.mu.Lock() + defer u.mu.Unlock() + for _, e := range entries { + name := e.Name() + if e.IsDir() || !strings.HasSuffix(name, ".json") { + continue + } + data, err := os.ReadFile(filepath.Join(upgradeHistoryDir, name)) + if err != nil { + continue + } + var h hiveUpgradeHistory + if json.Unmarshal(data, &h) != nil || h.HiveID == "" { + continue + } + u.byID[h.HiveID] = trimUpgradeHistory(h.Attempts, max) + } +} + +// Hub-side recording hooks. Each is safe without s.mu. + +// noteUpgradeStarted opens a history row for an instructed upgrade. +func (s *HubServer) noteUpgradeStarted(hiveID, from, target, trigger string) { + if s == nil || s.upgradeHistory == nil { + return + } + s.upgradeHistory.start(hiveID, historyRef(from), historyRef(target), trigger, time.Now()) +} + +// noteUpgradeEnded closes the open row for hiveID. +func (s *HubServer) noteUpgradeEnded(hiveID, outcome, reason, from, target string) { + if s == nil || s.upgradeHistory == nil { + return + } + s.upgradeHistory.end(hiveID, outcome, reason, historyRef(from), historyRef(target), time.Now()) +} + +// noteUpgradeDone / noteUpgradeMissed are the outcome-specific spellings for +// callers that should not need the spoke vocabulary. +func (s *HubServer) noteUpgradeDone(hiveID, reason, from, target string) { + s.noteUpgradeEnded(hiveID, spoke.UpgradeOutcomeDone, reason, from, target) +} + +func (s *HubServer) noteUpgradeMissed(hiveID, reason, from, target string) { + s.noteUpgradeEnded(hiveID, spoke.UpgradeOutcomeMissed, reason, from, target) +} + +// noteUpgradeWindowMissed records a held scheduled window. +func (s *HubServer) noteUpgradeWindowMissed(hiveID, from, reason string, at time.Time) { + if s == nil || s.upgradeHistory == nil { + return + } + s.upgradeHistory.windowMissed(hiveID, historyRef(from), reason, at) +} + +// historyRef trims a commit to its short form for history rows; a channel +// name ("candidate") or an empty target passes through untouched. +func historyRef(ref string) string { + ref = strings.TrimSpace(ref) + if len(ref) > StandardSHALen && isHexString(ref) { + return ref[:StandardSHALen] + } + return ref +} + +func isHexString(s string) bool { + if s == "" { + return false + } + for _, r := range s { + if (r < '0' || r > '9') && (r < 'a' || r > 'f') && (r < 'A' || r > 'F') { + return false + } + } + return true +} + +// upgradeTriggerForSchedule maps an auto-upgrade schedule onto the history +// trigger vocabulary: a daily/weekly window is "scheduled", everything else +// the hub fires on its own is "instant". +func upgradeTriggerForSchedule(mode, channel string) string { + switch effectiveAutoUpgradeSchedule(mode, channel) { + case AutoUpgradeModeDaily, AutoUpgradeModeWeekly: + return spoke.UpgradeTriggerScheduled + } + return spoke.UpgradeTriggerInstant +} diff --git a/src/pkg/hub/upgrade_history_test.go b/src/pkg/hub/upgrade_history_test.go new file mode 100644 index 0000000000..eff10ea6e3 --- /dev/null +++ b/src/pkg/hub/upgrade_history_test.go @@ -0,0 +1,177 @@ +package hub + +import ( + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/hub/spoke" +) + +func useTempUpgradeHistoryDir(t *testing.T) string { + t.Helper() + dir := t.TempDir() + orig := upgradeHistoryDir + upgradeHistoryDir = dir + t.Cleanup(func() { upgradeHistoryDir = orig }) + return dir +} + +func TestUpgradeHistoryStartEndRoundTrip(t *testing.T) { + useTempUpgradeHistoryDir(t) + u := newUpgradeHistoryStore() + t0 := time.Date(2026, 10, 26, 17, 0, 0, 0, time.UTC) + + u.start("hive-a", "551ffc34f1234", "abcdef0123456", spoke.UpgradeTriggerScheduled, t0) + open, ok := u.open("hive-a") + if !ok || !open.Open() || open.Target != "abcdef0123456" || open.Trigger != spoke.UpgradeTriggerScheduled { + t.Fatalf("open attempt = %+v, ok=%v", open, ok) + } + // A retry of the same open attempt re-targets rather than appending. + u.start("hive-a", "551ffc34f1234", "fedcba9876543", spoke.UpgradeTriggerScheduled, t0.Add(time.Minute)) + if got := u.recent("hive-a"); len(got) != 1 || got[0].Target != "fedcba9876543" { + t.Fatalf("retry should update the open row, got %+v", got) + } + u.end("hive-a", spoke.UpgradeOutcomeDone, "", "", "", t0.Add(10*time.Minute)) + got := u.recent("hive-a") + if len(got) != 1 || got[0].Outcome != spoke.UpgradeOutcomeDone || got[0].EndedAt.IsZero() { + t.Fatalf("closed row = %+v", got) + } + // A missed outcome with nothing open has no window to blame: dropped. + u.end("hive-a", spoke.UpgradeOutcomeMissed, "drain lapsed", "", "", t0.Add(time.Hour)) + if got := u.recent("hive-a"); len(got) != 1 { + t.Fatalf("missed with nothing open must be dropped, got %d rows", len(got)) + } + // A done outcome with nothing open is an observed rollout: one row. + u.end("hive-a", spoke.UpgradeOutcomeDone, "", "fedcba9876543", "0123456789abc", t0.Add(2*time.Hour)) + got = u.recent("hive-a") + if len(got) != 2 || got[1].Trigger != spoke.UpgradeTriggerObserved || got[1].Target != "0123456789abc" { + t.Fatalf("observed row = %+v", got) + } +} + +func TestUpgradeHistoryStaleOpenRowIsSuperseded(t *testing.T) { + useTempUpgradeHistoryDir(t) + u := newUpgradeHistoryStore() + t0 := time.Date(2026, 10, 25, 17, 0, 0, 0, time.UTC) + u.start("h", "aaa", "bbb", spoke.UpgradeTriggerManual, t0) + u.start("h", "aaa", "ccc", spoke.UpgradeTriggerScheduled, t0.Add(upgradeHistoryRecentFor+time.Hour)) + got := u.recent("h") + if len(got) != 2 { + t.Fatalf("want stale row closed + new row, got %+v", got) + } + if got[0].Outcome != spoke.UpgradeOutcomeMissed || !strings.Contains(got[0].Reason, "superseded") { + t.Fatalf("stale row = %+v", got[0]) + } + if !got[1].Open() || got[1].Target != "ccc" { + t.Fatalf("new row = %+v", got[1]) + } +} + +func TestUpgradeHistoryWindowMissedOneRowPerDay(t *testing.T) { + useTempUpgradeHistoryDir(t) + u := newUpgradeHistoryStore() + loc, err := autoUpgradeLocation() + if err != nil { + t.Fatal(err) + } + day1 := time.Date(2026, 10, 26, 13, 0, 0, 0, loc) + u.windowMissed("h", "551ffc3", "hub restarted; latch recovering", day1) + u.windowMissed("h", "551ffc3", "spoke still draining", day1.Add(30*time.Minute)) + got := u.recent("h") + if len(got) != 1 { + t.Fatalf("same window must dedupe to one row, got %d", len(got)) + } + if got[0].Trigger != spoke.UpgradeTriggerWindow || got[0].Outcome != spoke.UpgradeOutcomeMissed || got[0].Reason != "spoke still draining" { + t.Fatalf("window row = %+v", got[0]) + } + u.windowMissed("h", "551ffc3", "spoke still draining", day1.Add(24*time.Hour)) + if got := u.recent("h"); len(got) != 2 { + t.Fatalf("next day's window must be its own row, got %d", len(got)) + } + // A window that fired has an open attempt — nothing to record. + u.start("h", "551ffc3", "abc1234", spoke.UpgradeTriggerScheduled, day1.Add(48*time.Hour)) + u.windowMissed("h", "551ffc3", "late reason", day1.Add(48*time.Hour+time.Minute)) + if got := u.recent("h"); len(got) != 3 || !got[2].Open() { + t.Fatalf("open attempt must not be shadowed by a window row, got %+v", got) + } +} + +func TestUpgradeHistoryRotationAndEnvCap(t *testing.T) { + useTempUpgradeHistoryDir(t) + t.Setenv("HIVE_HUB_UPGRADE_HISTORY_MAX", "3") + u := newUpgradeHistoryStore() + t0 := time.Date(2026, 10, 1, 17, 0, 0, 0, time.UTC) + for i := 0; i < 6; i++ { + at := t0.Add(time.Duration(i) * 24 * time.Hour) + u.start("h", "aaa", "bbb", spoke.UpgradeTriggerScheduled, at) + u.end("h", spoke.UpgradeOutcomeDone, "", "", "", at.Add(5*time.Minute)) + } + got := u.recent("h") + if len(got) != 3 { + t.Fatalf("cap 3, got %d rows", len(got)) + } + if !got[0].StartedAt.Equal(t0.Add(3 * 24 * time.Hour)) { + t.Fatalf("oldest surviving row should be day 3, got %s", got[0].StartedAt) + } + if upgradeHistoryMax() != 3 { + t.Fatalf("env cap not honoured: %d", upgradeHistoryMax()) + } +} + +func TestUpgradeHistoryPersistsAndLoads(t *testing.T) { + dir := useTempUpgradeHistoryDir(t) + u := newUpgradeHistoryStore() + t0 := time.Date(2026, 10, 26, 17, 0, 0, 0, time.UTC) + u.start("fma", "551ffc3", "abc1234", spoke.UpgradeTriggerScheduled, t0) + u.end("fma", spoke.UpgradeOutcomeMissed, "drain lapsed", "", "", t0.Add(time.Hour)) + u.windowMissed("fma", "551ffc3", "hub restarted", t0.Add(24*time.Hour)) + if _, err := os.Stat(filepath.Join(dir, "fma.json")); err != nil { + t.Fatalf("history file not written: %v", err) + } + + reloaded := newUpgradeHistoryStore() + reloaded.load() + got := reloaded.recent("fma") + if len(got) != 2 { + t.Fatalf("reload: want 2 rows, got %+v", got) + } + if got[0].Reason != "drain lapsed" || got[1].Trigger != spoke.UpgradeTriggerWindow { + t.Fatalf("reload rows = %+v", got) + } + + reloaded.forget("fma") + if reloaded.recent("fma") != nil { + t.Fatal("forget must drop in-memory rows") + } + if _, err := os.Stat(filepath.Join(dir, "fma.json")); !os.IsNotExist(err) { + t.Fatalf("forget must remove the file, stat err=%v", err) + } + // Path traversal in a hive id never escapes the directory. + if p := upgradeHistoryPath("../etc"); p != "" { + t.Fatalf("traversal id produced a path: %q", p) + } +} + +func TestHistoryRefTrimsCommitsOnly(t *testing.T) { + if got := historyRef("551ffc34f0123456789"); got != "551ffc34f012" && len(got) != StandardSHALen { + t.Fatalf("historyRef(long sha) = %q", got) + } + if got := historyRef("candidate"); got != "candidate" { + t.Fatalf("channel name must pass through, got %q", got) + } + if got := historyRef(""); got != "" { + t.Fatalf("empty must stay empty, got %q", got) + } +} + +func TestUpgradeTriggerForSchedule(t *testing.T) { + if got := upgradeTriggerForSchedule(AutoUpgradeModeDaily, ReleaseChannelStable); got != spoke.UpgradeTriggerScheduled { + t.Fatalf("daily stable = %q", got) + } + if got := upgradeTriggerForSchedule(AutoUpgradeModeInstant, ""); got != spoke.UpgradeTriggerInstant { + t.Fatalf("instant = %q", got) + } +} diff --git a/src/pkg/hub/upgrade_progress.go b/src/pkg/hub/upgrade_progress.go new file mode 100644 index 0000000000..f1745c6065 --- /dev/null +++ b/src/pkg/hub/upgrade_progress.go @@ -0,0 +1,218 @@ +package hub + +import ( + "time" + + "github.com/hivecommons/hive/pkg/hub/spoke" +) + +// upgradeProgressInput is everything upgradeProgressFor needs, gathered by +// the caller so the derivation itself is pure and table-testable. +type upgradeProgressInput struct { + Entry *RegistryEntry + // AutoUpgrade / Mode / Channel / LastFired describe the hive's schedule; + // zero values mean "manual hive". + AutoUpgrade bool + Mode string + Channel string + LastFired string + // HoldReason is the schedule gate's most recent reason for holding an + // open window (SaaSHive.AutoUpgradeHoldReason), with its timestamp. + HoldReason string + HoldAt time.Time + History []spoke.UpgradeAttempt + Now time.Time +} + +// upgradeProgressFor derives the render-ready upgrade state machine for one +// hive from the hub's latch, the spoke's last heartbeat, the schedule and +// the persisted history. It is the single place both the hive card and the +// heartbeat policy read, so the two dashboards cannot disagree (#14856). +// +// Precedence, in flight first: +// +// UpgradeFailed → missed (spoke-reported error) +// Upgrading + UpgradeDrain → draining (countdown, agents) +// Upgrading + beat reports the target → verifying +// Upgrading + spoke reports upgrading → pulling +// Upgrading + no beat since instruction→ rolling +// Upgrading (older build, silent) → pulling, qualified +// open window held by the gate → missed (hold reason) +// last history row ended recently → that row's outcome +// daily/weekly schedule → scheduled (next window) +// instant / spoke-managed → scheduled (next poll) +// manual → scheduled, no window +func upgradeProgressFor(in upgradeProgressInput) *spoke.UpgradeProgress { + e := in.Entry + if e == nil { + return nil + } + now := in.Now + if now.IsZero() { + now = time.Now() + } + rfc := func(t time.Time) string { + if t.IsZero() { + return "" + } + return t.UTC().Format(time.RFC3339) + } + + if e.UpgradeFailed { + reason := e.UpgradeError + if reason == "" { + reason = "the spoke reported the upgrade failed" + } + return &spoke.UpgradeProgress{ + State: spoke.UpgradeStateMissed, + Target: historyRef(e.UpgradeTarget), + From: historyRef(e.GitHash), + Reason: reason, + Detail: "upgrade failed: " + reason, + EndedAt: rfc(e.UpgradeFailedAt), + } + } + + if e.Upgrading { + p := &spoke.UpgradeProgress{ + Target: historyRef(e.UpgradeTarget), + From: historyRef(e.GitHash), + StartedAt: rfc(e.UpgradeStartedAt), + AgentsTotal: len(e.Agents), + } + lastBeat := parseHeartbeatTime(e.LastHeartbeat) + switch { + case e.UpgradeDrain != nil: + p.State = spoke.UpgradeStateDraining + turns := e.UpgradeDrain.Turns + if len(turns) == 0 { + turns = spoke.TurnsFromNames(e.UpgradeDrain.InFlight) + } + p.AgentsInTurn = turns + // The countdown is anchored on the beat that carried the report: + // the spoke's RemainingS was true at send time, not at render time. + remaining := time.Duration(e.UpgradeDrain.RemainingS) * time.Second + anchor := lastBeat + if anchor.IsZero() { + anchor = now + } + deadline := anchor.Add(remaining) + p.DrainDeadline = rfc(deadline) + if left := deadline.Sub(now); left > 0 { + p.DrainRemainingS = int(left / time.Second) + } + p.Detail = "draining: " + spoke.FormatTurns(turns, p.AgentsTotal) + + " — proceeds in " + spoke.FormatShortDuration(deadline.Sub(now)) + case !lastBeat.IsZero() && !e.UpgradeStartedAt.IsZero() && + lastBeat.After(e.UpgradeStartedAt) && sameCommit(e.GitHash, e.UpgradeTarget): + p.State = spoke.UpgradeStateVerifying + p.Detail = "verifying: spoke is back on " + orDash(p.Target) + case e.SpokeReportedUpgrading: + p.State = spoke.UpgradeStatePulling + p.Detail = "pulling: spoke acknowledged the upgrade to " + orDash(p.Target) + if e.UpgradeRestartFallback != "" { + p.Reason = e.UpgradeRestartFallback + } + case !e.UpgradeStartedAt.IsZero() && (lastBeat.IsZero() || !lastBeat.After(e.UpgradeStartedAt)): + p.State = spoke.UpgradeStateRolling + p.Detail = "rolling: no heartbeat since the instruction — pod is restarting onto " + orDash(p.Target) + default: + p.State = spoke.UpgradeStatePulling + p.Detail = "instruction delivered; an older build may be draining silently" + p.Reason = "spoke has not acknowledged the upgrade" + } + return p + } + + // Nothing in flight. A window the gate is holding right now is a live + // miss, before any history row. + daily := false + if in.AutoUpgrade { + switch effectiveAutoUpgradeSchedule(in.Mode, in.Channel) { + case AutoUpgradeModeDaily, AutoUpgradeModeWeekly: + daily = true + } + } + nextWindow := "" + if daily { + nextWindow = nextScheduledAutoUpgradeAt(in.Mode, in.LastFired, now) + } + if daily && in.HoldReason != "" && scheduledUpgradeWindowOpen(in.Mode, in.LastFired, now) { + return &spoke.UpgradeProgress{ + State: spoke.UpgradeStateMissed, + From: historyRef(e.GitHash), + NextWindowAt: nextWindow, + Reason: in.HoldReason, + Detail: "window held: " + in.HoldReason, + EndedAt: rfc(in.HoldAt), + } + } + + if last, ok := latestClosedAttempt(in.History); ok && now.Sub(last.EndedAt) <= upgradeHistoryRecentFor { + p := &spoke.UpgradeProgress{ + State: last.Outcome, + Target: last.Target, + From: last.From, + StartedAt: rfc(last.StartedAt), + EndedAt: rfc(last.EndedAt), + NextWindowAt: nextWindow, + Reason: last.Reason, + } + ago := spoke.FormatShortDuration(now.Sub(last.EndedAt)) + " ago" + switch last.Outcome { + case spoke.UpgradeOutcomeDone: + p.Detail = "upgraded to " + orDash(last.Target) + " " + ago + default: + if last.Trigger == spoke.UpgradeTriggerWindow { + p.Detail = "missed window " + ago + ": " + last.Reason + } else { + p.Detail = "missed " + ago + ": " + last.Reason + } + } + return p + } + + p := &spoke.UpgradeProgress{State: spoke.UpgradeStateScheduled, From: historyRef(e.GitHash), NextWindowAt: nextWindow} + switch { + case !in.AutoUpgrade: + p.Detail = "manual upgrades only" + case daily: + p.Detail = "next window " + nextWindow + default: + p.Detail = "upgrades on the hub's next poll" + } + return p +} + +// latestClosedAttempt returns the most recently ended row of a history. +func latestClosedAttempt(history []spoke.UpgradeAttempt) (spoke.UpgradeAttempt, bool) { + var best spoke.UpgradeAttempt + found := false + for _, a := range history { + if a.Open() || a.EndedAt.IsZero() { + continue + } + if !found || a.EndedAt.After(best.EndedAt) { + best, found = a, true + } + } + return best, found +} + +// upgradeProgressForRow is the hive-card wiring: it pulls schedule state off +// the hub-owned SaaSHive record (nil for an unmanaged spoke) and the +// persisted history for the row. +func (s *HubServer) upgradeProgressForRow(entry *RegistryEntry, sh *SaaSHive, channel string, now time.Time) (*spoke.UpgradeProgress, []spoke.UpgradeAttempt) { + in := upgradeProgressInput{Entry: entry, Channel: channel, Now: now} + if sh != nil { + in.AutoUpgrade = sh.AutoUpgrade + in.Mode = sh.AutoUpgradeMode + in.LastFired = sh.AutoUpgradeLastFired + in.HoldReason = sh.AutoUpgradeHoldReason + in.HoldAt = sh.AutoUpgradeHoldAt + } + if s != nil && s.upgradeHistory != nil { + in.History = s.upgradeHistory.recent(entry.ID) + } + return upgradeProgressFor(in), in.History +} diff --git a/src/pkg/hub/upgrade_progress_test.go b/src/pkg/hub/upgrade_progress_test.go new file mode 100644 index 0000000000..afd23afa1a --- /dev/null +++ b/src/pkg/hub/upgrade_progress_test.go @@ -0,0 +1,241 @@ +package hub + +import ( + "strings" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/hub/spoke" +) + +func TestUpgradeProgressForStates(t *testing.T) { + loc, err := autoUpgradeLocation() + if err != nil { + t.Fatal(err) + } + // 14:00 ET on a Monday: the daily window is open. + now := time.Date(2026, 10, 26, 14, 0, 0, 0, loc) + started := now.Add(-20 * time.Minute) + agents := []AgentSummary{{Name: "reviewer"}, {Name: "scanner"}, {Name: "triage"}} + + cases := []struct { + name string + in upgradeProgressInput + state string + detail string // substring + remain int + inTurn int + nextWin bool + }{ + { + name: "spoke-reported failure is missed", + in: upgradeProgressInput{Entry: &RegistryEntry{UpgradeFailed: true, UpgradeError: "image pull backoff", UpgradeTarget: "abc1234"}}, + state: spoke.UpgradeStateMissed, detail: "image pull backoff", + }, + { + name: "drain report is draining with countdown and agents", + in: upgradeProgressInput{Entry: &RegistryEntry{ + Upgrading: true, UpgradeTarget: "abc1234", GitHash: "551ffc3", UpgradeStartedAt: started, + LastHeartbeat: now.Add(-30 * time.Second).UTC().Format(time.RFC3339), Agents: agents, + UpgradeDrain: &UpgradeDrainReport{Target: "abc1234", RemainingS: 600, WaitedS: 300, + Turns: []spoke.UpgradeTurn{{Agent: "reviewer", ElapsedS: 38 * 60}, {Agent: "scanner", ElapsedS: 12 * 60}}}, + }}, + state: spoke.UpgradeStateDraining, detail: "2/3 agents in turn: reviewer 38m, scanner 12m", remain: 570, inTurn: 2, + }, + { + name: "legacy drain report without turns still names agents", + in: upgradeProgressInput{Entry: &RegistryEntry{ + Upgrading: true, UpgradeTarget: "abc1234", UpgradeStartedAt: started, Agents: agents, + UpgradeDrain: &UpgradeDrainReport{Target: "abc1234", RemainingS: 60, InFlight: []string{"reviewer"}}, + }}, + state: spoke.UpgradeStateDraining, detail: "1/3 agent in turn: reviewer", inTurn: 1, + }, + { + name: "beat after instruction on target is verifying", + in: upgradeProgressInput{Entry: &RegistryEntry{ + Upgrading: true, UpgradeTarget: "abc1234", GitHash: "abc1234", UpgradeStartedAt: started, + LastHeartbeat: now.Add(-time.Minute).UTC().Format(time.RFC3339), + }}, + state: spoke.UpgradeStateVerifying, detail: "verifying", + }, + { + name: "spoke acknowledged is pulling", + in: upgradeProgressInput{Entry: &RegistryEntry{ + Upgrading: true, UpgradeTarget: "abc1234", GitHash: "551ffc3", UpgradeStartedAt: started, SpokeReportedUpgrading: true, + LastHeartbeat: now.Add(-time.Minute).UTC().Format(time.RFC3339), + }}, + state: spoke.UpgradeStatePulling, detail: "acknowledged", + }, + { + name: "silent since instruction is rolling", + in: upgradeProgressInput{Entry: &RegistryEntry{ + Upgrading: true, UpgradeTarget: "abc1234", GitHash: "551ffc3", UpgradeStartedAt: started, + LastHeartbeat: started.Add(-time.Minute).UTC().Format(time.RFC3339), + }}, + state: spoke.UpgradeStateRolling, detail: "restarting", + }, + { + name: "beating on old hash without ack is pulling with a qualifier", + in: upgradeProgressInput{Entry: &RegistryEntry{ + Upgrading: true, UpgradeTarget: "abc1234", GitHash: "551ffc3", UpgradeStartedAt: started, + LastHeartbeat: now.Add(-time.Minute).UTC().Format(time.RFC3339), + }}, + state: spoke.UpgradeStatePulling, detail: "draining silently", + }, + { + name: "open window held by the gate is missed now", + in: upgradeProgressInput{Entry: &RegistryEntry{GitHash: "551ffc3"}, + AutoUpgrade: true, Mode: AutoUpgradeModeDaily, Channel: ReleaseChannelStable, + HoldReason: "hub restarted; stuck-upgrade latch recovering", HoldAt: now.Add(-10 * time.Minute)}, + state: spoke.UpgradeStateMissed, detail: "window held", nextWin: true, + }, + { + name: "recent missed window from history", + in: upgradeProgressInput{Entry: &RegistryEntry{GitHash: "551ffc3"}, + AutoUpgrade: true, Mode: AutoUpgradeModeDaily, LastFired: now.Format(autoUpgradeDateFormat), + History: []spoke.UpgradeAttempt{{Trigger: spoke.UpgradeTriggerWindow, Outcome: spoke.UpgradeOutcomeMissed, + Reason: "drain lapsed", StartedAt: now.Add(-2 * time.Hour), EndedAt: now.Add(-2 * time.Hour)}}}, + state: spoke.UpgradeStateMissed, detail: "missed window 2h0m ago: drain lapsed", nextWin: true, + }, + { + name: "recent done from history", + in: upgradeProgressInput{Entry: &RegistryEntry{GitHash: "abc1234"}, + History: []spoke.UpgradeAttempt{{Target: "abc1234", Outcome: spoke.UpgradeOutcomeDone, + StartedAt: now.Add(-3 * time.Hour), EndedAt: now.Add(-3 * time.Hour)}}}, + state: spoke.UpgradeStateDone, detail: "upgraded to abc1234 3h0m ago", + }, + { + name: "old history falls back to scheduled", + in: upgradeProgressInput{Entry: &RegistryEntry{GitHash: "abc1234"}, + AutoUpgrade: true, Mode: AutoUpgradeModeDaily, LastFired: now.Format(autoUpgradeDateFormat), + History: []spoke.UpgradeAttempt{{Outcome: spoke.UpgradeOutcomeDone, + StartedAt: now.Add(-48 * time.Hour), EndedAt: now.Add(-48 * time.Hour)}}}, + state: spoke.UpgradeStateScheduled, detail: "next window", nextWin: true, + }, + { + name: "instant mode is scheduled on next poll", + in: upgradeProgressInput{Entry: &RegistryEntry{}, AutoUpgrade: true, Mode: AutoUpgradeModeInstant}, + state: spoke.UpgradeStateScheduled, detail: "next poll", + }, + { + name: "manual hive", + in: upgradeProgressInput{Entry: &RegistryEntry{}}, + state: spoke.UpgradeStateScheduled, detail: "manual", + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + tc.in.Now = now + p := upgradeProgressFor(tc.in) + if p == nil { + t.Fatal("nil progress") + } + if p.State != tc.state { + t.Fatalf("state = %q want %q (%+v)", p.State, tc.state, p) + } + if !strings.Contains(p.Detail, tc.detail) { + t.Fatalf("detail %q lacks %q", p.Detail, tc.detail) + } + if tc.remain != 0 && p.DrainRemainingS != tc.remain { + t.Fatalf("remaining = %d want %d", p.DrainRemainingS, tc.remain) + } + if len(p.AgentsInTurn) != tc.inTurn { + t.Fatalf("agents in turn = %d want %d", len(p.AgentsInTurn), tc.inTurn) + } + if tc.nextWin != (p.NextWindowAt != "") { + t.Fatalf("nextWindowAt = %q, want present=%v", p.NextWindowAt, tc.nextWin) + } + if p.State == spoke.UpgradeStateDraining && p.DrainDeadline == "" { + t.Fatal("draining must carry a deadline") + } + }) + } + if upgradeProgressFor(upgradeProgressInput{}) != nil { + t.Fatal("nil entry must yield nil") + } +} + +func TestUpgradeProgressInFlight(t *testing.T) { + for _, st := range []string{spoke.UpgradeStateDraining, spoke.UpgradeStatePulling, spoke.UpgradeStateRolling, spoke.UpgradeStateVerifying} { + if !(&spoke.UpgradeProgress{State: st}).InFlight() { + t.Fatalf("%s should be in flight", st) + } + } + for _, st := range []string{spoke.UpgradeStateScheduled, spoke.UpgradeStateDone, spoke.UpgradeStateMissed} { + if (&spoke.UpgradeProgress{State: st}).InFlight() { + t.Fatalf("%s should not be in flight", st) + } + } + var nilP *spoke.UpgradeProgress + if nilP.InFlight() { + t.Fatal("nil must not be in flight") + } +} + +func TestHeartbeatUpgradePolicyCarriesWindowAndHistory(t *testing.T) { + useTempUpgradeHistoryDir(t) + stubBranchHead(t, "v5", "5193426") + stubChannelRevisions(t, map[string]string{"stable": "5193426"}) + s := &HubServer{logger: targetingLogger(), upgradeHistory: newUpgradeHistoryStore()} + s.upgradeHistory.windowMissed("fma", "551ffc3", "hub restarted", time.Now().Add(-time.Hour)) + sh := &SaaSHive{ID: "fma", AutoUpgrade: true, AutoUpgradeMode: AutoUpgradeModeDaily, TrackedChannel: ReleaseChannelStable} + payload := &HeartbeatPayload{HiveID: "fma"} + policy := s.heartbeatUpgradePolicy(payload, sh, false, false, "v5", "") + if policy == nil { + t.Fatal("nil policy") + } + if policy.NextWindowAt == "" { + t.Fatal("stable daily hive must still report its own next window (#14856)") + } + if _, err := time.Parse(time.RFC3339, policy.NextWindowAt); err != nil { + t.Fatalf("NextWindowAt not RFC3339: %q", policy.NextWindowAt) + } + if len(policy.UpgradeHistory) != 1 || policy.UpgradeHistory[0].Reason != "hub restarted" { + t.Fatalf("history not delivered on the policy: %+v", policy.UpgradeHistory) + } +} + +func TestUpgradeProgressForRowReadsSaaSHiveAndHistory(t *testing.T) { + useTempUpgradeHistoryDir(t) + s := &HubServer{logger: targetingLogger(), upgradeHistory: newUpgradeHistoryStore()} + now := time.Now() + s.upgradeHistory.start("h", "551ffc3", "abc1234", spoke.UpgradeTriggerScheduled, now.Add(-time.Hour)) + s.upgradeHistory.end("h", spoke.UpgradeOutcomeDone, "", "", "", now.Add(-50*time.Minute)) + entry := &RegistryEntry{ID: "h", GitHash: "abc1234"} + sh := &SaaSHive{ID: "h", AutoUpgrade: true, AutoUpgradeMode: AutoUpgradeModeDaily} + p, hist := s.upgradeProgressForRow(entry, sh, "", now) + if p == nil || p.State != spoke.UpgradeStateDone { + t.Fatalf("progress = %+v", p) + } + if len(hist) != 1 { + t.Fatalf("history = %+v", hist) + } + if p2, _ := s.upgradeProgressForRow(&RegistryEntry{ID: "other"}, nil, "", now); p2 == nil || p2.State != spoke.UpgradeStateScheduled { + t.Fatalf("unmanaged spoke progress = %+v", p2) + } +} + +func TestRecordUpgradeHistoryTransition(t *testing.T) { + useTempUpgradeHistoryDir(t) + s := &HubServer{logger: targetingLogger(), upgradeHistory: newUpgradeHistoryStore()} + idle := &RegistryEntry{ID: "h", GitHash: "551ffc3"} + rolling := &RegistryEntry{ID: "h", GitHash: "551ffc3", Upgrading: true, UpgradeTarget: "abc1234"} + s.recordUpgradeHistoryTransition(idle, rolling) + if open, ok := s.upgradeHistory.open("h"); !ok || open.Trigger != spoke.UpgradeTriggerObserved || open.Target != "abc1234" { + t.Fatalf("observed start not recorded: %+v ok=%v", open, ok) + } + landed := &RegistryEntry{ID: "h", GitHash: "abc1234"} + s.recordUpgradeHistoryTransition(rolling, landed) + got := s.upgradeHistory.recent("h") + if len(got) != 1 || got[0].Outcome != spoke.UpgradeOutcomeDone || got[0].Target != "abc1234" { + t.Fatalf("landed row = %+v", got) + } + + s.recordUpgradeHistoryTransition(idle, rolling) + failed := &RegistryEntry{ID: "h", GitHash: "551ffc3", UpgradeFailed: true, UpgradeError: "ErrImagePull"} + s.recordUpgradeHistoryTransition(rolling, failed) + got = s.upgradeHistory.recent("h") + if len(got) != 2 || got[1].Outcome != spoke.UpgradeOutcomeMissed || got[1].Reason != "ErrImagePull" { + t.Fatalf("failed row = %+v", got) + } +}