Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/added-15788-roll-upgrade-animation.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- The dashboard's upgrade animation now plays for every pod roll, not only self-upgrades ([#15788](https://github.com/hivecommons/hive/issues/15788)): the hub announces hub-managed rolls through the heartbeat (`upgrade_announce`), operators can announce theirs with `POST /api/upgrade/announce` (owner-only; one-liner in `dashboard-standalone-upgrades.md`), and a SIGTERM with no announce still publishes an SSE `upgrading` event with `phase: restarting` before the stream drops. The banner survives the reconnect, shows the new pod's boot phase from `/api/version.startupPhase`, and reads **Upgraded to `<sha>` ✓** or **Restarted ✓** by comparing build hash and process `startedAt`. Preview it any time with `?demo=upgrade` or Settings → Diagnostics → *Preview upgrade animation*.
67 changes: 67 additions & 0 deletions dashboard/openapi.json
Original file line number Diff line number Diff line change
Expand Up @@ -29621,6 +29621,73 @@
}
}
},
"/api/upgrade/announce": {
"post": {
"tags": [
"System"
],
"summary": "Announce a pod roll before it happens",
"description": "Marks this spoke's upgradeProgress as state=pulling/phase=announced with the given target and publishes an SSE `upgrading` event so every open dashboard shows the upgrade animation before the pod is killed (#15788). Used by the hub (also delivered via the heartbeat response) and by operator scripts ahead of `kubectl rollout restart`. A SIGTERM afterwards publishes phase=restarting from the same record; an announce with no kill expires after 15 minutes. Requires owner role.",
"requestBody": {
"required": false,
"content": {
"application/json": {
"schema": {
"type": "object",
"properties": {
"target": {
"type": "string",
"description": "Commit SHA or image tag the pod restarts onto; omit for a plain restart"
},
"version": {
"type": "string",
"description": "Alias of target"
},
"source": {
"type": "string",
"enum": [
"hub",
"operator"
],
"description": "Who is rolling the pod; anything else is recorded as operator"
},
"reason": {
"type": "string"
}
}
}
}
}
},
"responses": {
"200": {
"description": "Announce recorded",
"content": {
"application/json": {
"schema": {
"type": "object",
"properties": {
"ok": {
"type": "boolean"
},
"upgradeProgress": {
"type": "object",
"description": "The spoke's upgradeProgress as now reported by /api/status and /api/version (state, phase, source, target, from, detail, startedAt)"
}
}
}
}
}
},
"400": {
"description": "Invalid JSON body"
},
"403": {
"description": "Owner access required"
}
}
}
},
"/api/snapshot": {
"get": {
"tags": [
Expand Down
11 changes: 9 additions & 2 deletions src/cmd/hive/boot_heartbeat_deps_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -180,8 +180,9 @@ func TestBootHeartbeatWith_PublishesIdentityBeforeTheLoopAndWiresEveryCallback(t
heartbeatCallback[spoke.GatewayConfigCallback](t, f)
heartbeatCallback[spoke.ClusterIDCallback](t, f)
heartbeatCallback[spoke.FreshStatusCollector](t, f)
if len(f.callbacks) != 14 {
t.Fatalf("registered %d callbacks, want 14 (one of each)", len(f.callbacks))
heartbeatCallback[spoke.UpgradeAnnounceCallback](t, f)
if len(f.callbacks) != 15 {
t.Fatalf("registered %d callbacks, want 15 (one of each)", len(f.callbacks))
}

// The collector honours a runtime hub.enabled=false without being torn
Expand Down Expand Up @@ -230,6 +231,12 @@ func TestBootHeartbeatWith_CallbacksReconcileRunningConfig(t *testing.T) {
banner(nil)
heartbeatCallback[spoke.UpgradePolicyCallback](t, f)(&spoke.HeartbeatUpgradePolicy{HubManaged: true})
heartbeatCallback[spoke.UpgradePolicyCallback](t, f)(nil)
// A hub roll announce (#15788) lands on the dashboard as an in-flight
// upgrade so the operator sees the animation before the kill.
heartbeatCallback[spoke.UpgradeAnnounceCallback](t, f)(&spoke.UpgradeAnnounce{Target: "0123456789abcdef", Source: spoke.UpgradeSourceHub})
if p := b.dashSrv.AnnounceRestart(time.Now()); p.Target != "0123456" || p.Source != spoke.UpgradeSourceHub {
t.Fatalf("announce did not reach the dashboard: %+v", p)
}

// Unknown agent: warn, never panic — the hub may name an agent this
// spoke's pack no longer has.
Expand Down
27 changes: 26 additions & 1 deletion src/cmd/hive/boot_sequence.go
Original file line number Diff line number Diff line change
Expand Up @@ -66,11 +66,36 @@ func defaultBootSequenceNames() []string {
func defaultBootSequence() bootSequence {
seq := bootSequence{config: (*boot).bootConfig, loop: (*boot).runLoop}
for _, p := range defaultBootPhases() {
seq.phases = append(seq.phases, p.run)
seq.phases = append(seq.phases, notingPhase(p))
}
return seq
}

// bootPhaseFirstEval is the phase name runLoop reports once every boot
// phase has run and the governor is about to publish its first status.
const bootPhaseFirstEval = "firstEval"

// notingPhase wraps a phase so the dashboard learns its name as it starts
// (#15788): a browser reconnecting onto a fresh pod reads startupPhase from
// /api/status and shows "starting: opening stores" instead of a bare
// spinner. Phases before bootDashboard have no server to tell and are
// skipped by noteBootPhase.
func notingPhase(p namedPhase) bootPhase {
return func(b *boot) bool {
b.noteBootPhase(p.name)
return p.run(b)
}
}

// noteBootPhase records the running boot phase on the dashboard server,
// when one exists yet.
func (b *boot) noteBootPhase(name string) {
if b == nil || b.dashSrv == nil {
return
}
b.dashSrv.SetStartupPhase(name)
}

// runBoot drives one hive process through seq. b.cleanup runs on every
// return path, including a config veto.
func runBoot(b *boot, seq bootSequence) {
Expand Down
31 changes: 27 additions & 4 deletions src/cmd/hive/boot_sequence_test.go
Original file line number Diff line number Diff line change
@@ -1,10 +1,14 @@
package main

import (
"io"
"log/slog"
"reflect"
"runtime"
"strings"
"testing"

"github.com/hivecommons/hive/pkg/dashboard"
)

func funcName(f interface{}) string {
Expand All @@ -29,10 +33,11 @@ func TestDefaultBootSequenceOrder(t *testing.T) {
if len(seq.phases) != len(want) {
t.Fatalf("phase count = %d, want %d", len(seq.phases), len(want))
}
// step() wraps non-veto phases in a closure, so their names are not
// recoverable through the func value; only the veto-capable bootAdvisory
// is pinned by name. defaultBootSequenceNames pins the rest.
if got := funcName(seq.phases[2]); got != "bootAdvisory" {
// step() and notingPhase() wrap every phase in a closure, so names are
// not recoverable through the func values; the veto-capable bootAdvisory
// is pinned through the phase table and defaultBootSequenceNames pins
// the order.
if got := funcName(defaultBootPhases()[2].run); got != "bootAdvisory" {
t.Fatalf("phases[2] = %s, want bootAdvisory (the veto-capable phase)", got)
}
if got := defaultBootSequenceNames(); strings.Join(got, ",") != strings.Join(want, ",") {
Expand Down Expand Up @@ -90,3 +95,21 @@ func TestRunBootPhaseVetoSkipsLaterPhasesAndLoop(t *testing.T) {
t.Fatalf("trace = %s", got)
}
}

// #15788: each phase reports its name to the dashboard as it starts so a
// browser reconnecting onto a fresh pod can say what it is waiting on; a
// phase before bootDashboard has nowhere to report and must not panic.
func TestNotingPhaseReportsStartupPhaseToDashboard(t *testing.T) {
b := &boot{}
ran := false
phase := notingPhase(namedPhase{name: "bootGitHub", run: func(*boot) bool { ran = true; return false }})
if phase(b) || !ran {
t.Fatal("wrapper must run the phase and return its verdict")
}

b.dashSrv = dashboard.NewServer(0, slog.New(slog.NewTextHandler(io.Discard, nil)))
notingPhase(namedPhase{name: "bootStores", run: func(*boot) bool { return true }})(b)
if got := b.dashSrv.StartupPhase(); got != "bootStores" {
t.Fatalf("StartupPhase = %q, want bootStores", got)
}
}
18 changes: 18 additions & 0 deletions src/cmd/hive/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -2428,6 +2428,18 @@ func (b *boot) bootDashboardWith(deps bootDashboardDeps) {
b.preShutdownHooks.addUrgent("drain-contributor-websockets", func() {
b.dashSrv.DrainContributorsForShutdown()
})
// Tell the dashboard the pod is going down BEFORE the stream drops
// (#15788): an SSE "upgrading" event with phase=restarting and the
// target the hub armed (when known), so the operator sees the upgrade
// animation instead of a bare "reconnecting". Registered after the
// contributor drain with addUrgent so it runs first — the frame is one
// write on sockets that are already open, and the drain is what takes
// the time. restartAnnounceSettle gives the SSE handler goroutines a
// moment to flush before the hooks that follow race the grace period.
b.preShutdownHooks.addUrgent("announce-restart", func() {
b.dashSrv.AnnounceRestart(time.Now())
<-time.After(restartAnnounceSettle)
})
// Stop the contributor hub's background cleanup loop on the way out
// (#6272). On v4 this is a `defer dashSrv.CloseContributeHub()` in main();
// v5 has no such single-function scope, so it rides the same shutdown-hook
Expand Down Expand Up @@ -5317,6 +5329,11 @@ func (b *boot) bootHeartbeatWith(deps bootHeartbeatDeps) {
// the hub will actually roll us to and renders the hub's schedule.
b.dashSrv.SetHubUpgradePolicy(p)
}),
spoke.UpgradeAnnounceCallback(func(a *spoke.UpgradeAnnounce) {
// The hub is about to roll (or has rolled) this pod itself
// (#15788): show the upgrade animation before the kill.
b.dashSrv.AnnounceUpgrade(*a, time.Now())
}),
spoke.VisibilityCallback(func(isPublic bool) {
if b.cfg.Hub.IsPublic != isPublic {
b.logger.Info("hub overrode visibility via heartbeat",
Expand Down Expand Up @@ -5766,6 +5783,7 @@ func (b *boot) runLoop() { b.runLoopWith(defaultRunLoopDeps()) }
// runLoopWith is runLoop with its timers and per-tick IO injected; see
// runLoopDeps.
func (b *boot) runLoopWith(deps runLoopDeps) {
b.noteBootPhase(bootPhaseFirstEval)
initialEvalInterval, initialMode := evalIntervalAndModeForConfig(b.cfg, b.ghClient, b.logger)
b.logger.Info("entering governor loop", "interval_seconds", int(initialEvalInterval.Seconds()), "eval_mode", initialMode)
lastEvalInterval := initialEvalInterval
Expand Down
5 changes: 5 additions & 0 deletions src/cmd/hive/self_upgrade_drain.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,11 @@ import (
// upgrade was cancelled or superseded — and kicks must not stay withheld.
const selfUpgradeDrainStaleAfter = 3 * heartbeatSendInterval

// restartAnnounceSettle is how long the SIGTERM hook lets the SSE handler
// goroutines flush the "upgrading" frame before the next hook runs
// (#15788). One write on already-open sockets; well inside the grace period.
const restartAnnounceSettle = 100 * time.Millisecond

// selfUpgradeDrainNow is the drain clock, a seam for tests.
var selfUpgradeDrainNow = time.Now

Expand Down
Loading
Loading