Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/added-upgrade-progress-visibility.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- Scheduled upgrades now expose their progress as data on both dashboards ([#14950](https://github.com/hivecommons/hive/issues/14950), motivated by [#14856](https://github.com/hivecommons/hive/issues/14856)): the hub's `/api/saas/my-hives` carries `upgradeState` (`scheduled` → `draining` → `pulling` → `rolling` → `verifying` → `done` | `missed`, with the drain deadline/countdown and the agents whose turns are holding it — "2/5 agents in turn: reviewer 38m, scanner 12m") and `upgradeHistory`; the spoke's `/api/status.upgradeProgress` derives the same state locally and the heartbeat policy gains `next_window_at` (the hive's own window, even on `stable`) plus `upgrade_history`. The hub persists the last 14 attempts and held windows per hive (`HIVE_HUB_UPGRADE_HISTORY_MAX`) under `/data/saas/upgrade-history/`, so a missed window survives a hub restart and is never silent. The spoke's drain report now includes per-agent turn elapsed (`upgrade_drain.turns`).
5 changes: 4 additions & 1 deletion src/cmd/hive/main.go
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
package main

import (
Expand Down Expand Up @@ -3721,6 +3721,7 @@
})

b.dashSrv.SetGitHubAppRequired(b.githubAppRequired)
b.dashSrv.SetUpgradeDrainSource(selfUpgradeDrain.Report)
// Order matters: SetGitHubAppRequired(false) clears both fields, so the
// classified state is applied only after it, and only when a failure was
// actually detected.
Expand Down Expand Up @@ -5156,7 +5157,8 @@
// heartbeat) and withhold new kicks; proceed once the turns end or
// hub.upgrade_drain_timeout_s elapses.
if b.agentMgr != nil {
inFlight, wedged := agentKicksInFlight(b.agentMgr.AllStatuses(), b.startTime, selfUpgradeDrainNow(), func(name string) time.Duration {
statuses := b.agentMgr.AllStatuses()
inFlight, wedged := agentKicksInFlight(statuses, b.startTime, selfUpgradeDrainNow(), func(name string) time.Duration {
if ac, ok := b.cfg.Agents[name]; ok {
return ac.EffectiveContinuousTurnCeiling()
}
Expand All @@ -5173,6 +5175,7 @@
ignoredWedged := wedgedKickAgents(wedged)
drainTimeout := b.cfg.Hub.EffectiveUpgradeDrainTimeout()
dec := selfUpgradeDrain.decide(targetSHA, inFlight, drainTimeout, selfUpgradeDrainNow())
selfUpgradeDrain.setKickTimes(agentKickTimes(statuses, inFlight))
switch {
case !dec.Proceed:
b.logger.Info("self-upgrade deferred: draining in-flight agent kicks",
Expand Down
43 changes: 42 additions & 1 deletion src/cmd/hive/self_upgrade_drain.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,10 @@ type upgradeDrain struct {
// tell the hub WHAT the drain is waiting on and for how much longer.
inFlight []string
timeout time.Duration
// kickAt is when each in-flight agent's current turn was kicked, so
// Report can say "reviewer 38m" (#14856). Names absent from the map are
// reported with an unknown elapsed time.
kickAt map[string]time.Time
// released is set once decide lets the upgrade proceed. Every path after
// that point exits the process or blocks until SIGTERM, so kicks stay
// withheld for good: a kick started now would be cut by the restart.
Expand Down Expand Up @@ -135,12 +139,48 @@ func (d *upgradeDrain) Report(now time.Time) *spoke.UpgradeDrainReport {
if remaining < 0 {
remaining = 0
}
return &spoke.UpgradeDrainReport{
rep := &spoke.UpgradeDrainReport{
Target: d.target,
InFlight: append([]string(nil), d.inFlight...),
WaitedS: int(waited / time.Second),
RemainingS: int(remaining / time.Second),
}
for _, name := range d.inFlight {
turn := spoke.UpgradeTurn{Agent: name}
if at, ok := d.kickAt[name]; ok && !at.IsZero() && now.After(at) {
turn.ElapsedS = int(now.Sub(at) / time.Second)
}
rep.Turns = append(rep.Turns, turn)
}
return rep
}

// setKickTimes records when the in-flight agents' turns started. Called next
// to decide with the same statuses it classified; a drain that is not open
// ignores it.
func (d *upgradeDrain) setKickTimes(kickAt map[string]time.Time) {
d.mu.Lock()
defer d.mu.Unlock()
if d.released || d.target == "" {
d.kickAt = nil
return
}
d.kickAt = kickAt
}

// agentKickTimes returns the kick time of each named agent, for
// upgradeDrain.setKickTimes.
func agentKickTimes(statuses map[string]*crux.Process, names []string) map[string]time.Time {
if len(names) == 0 {
return nil
}
out := make(map[string]time.Time, len(names))
for _, name := range names {
if proc := statuses[name]; proc != nil && proc.LastKick != nil {
out[name] = *proc.LastKick
}
}
return out
}

// releaseLocked closes the drain window and latches the kick hold until the
Expand All @@ -150,6 +190,7 @@ func (d *upgradeDrain) releaseLocked(target string, now time.Time) {
d.startedAt = time.Time{}
d.checkedAt = now
d.inFlight, d.timeout = nil, 0
d.kickAt = nil
d.released = true
}

Expand Down
47 changes: 46 additions & 1 deletion src/cmd/hive/self_upgrade_drain_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -245,7 +245,8 @@ func TestUpgradeDrain_ReportReflectsDeferralForHub(t *testing.T) {
if r == nil {
t.Fatal("deferring drain must report")
}
want := &spoke.UpgradeDrainReport{Target: "4f8be0b", InFlight: []string{"brainstorm"}, WaitedS: 600, RemainingS: 2100}
want := &spoke.UpgradeDrainReport{Target: "4f8be0b", InFlight: []string{"brainstorm"}, WaitedS: 600, RemainingS: 2100,
Turns: []spoke.UpgradeTurn{{Agent: "brainstorm"}}}
if !reflect.DeepEqual(r, want) {
t.Fatalf("Report = %+v, want %+v", r, want)
}
Expand All @@ -259,3 +260,47 @@ func TestUpgradeDrain_ReportReflectsDeferralForHub(t *testing.T) {
t.Fatalf("released drain must report nil, got %+v", r)
}
}

// #14856: the drain report names how long each held turn has been running,
// so the hub card can say "reviewer 38m" instead of just "reviewer".
func TestUpgradeDrain_ReportCarriesTurnElapsed(t *testing.T) {
d := &upgradeDrain{}
now := time.Date(2026, 10, 26, 17, 10, 0, 0, time.UTC)
reviewerKick := now.Add(-38 * time.Minute)
scannerKick := now.Add(-12 * time.Minute)
statuses := map[string]*crux.Process{
"reviewer": {State: crux.StateRunning, LastKick: &reviewerKick},
"scanner": {State: crux.StateRunning, LastKick: &scannerKick},
"idle": {State: crux.StateRunning},
}
inFlight := []string{"reviewer", "scanner"}

// Not open yet: kick times are ignored rather than retained.
d.setKickTimes(agentKickTimes(statuses, inFlight))
if d.kickAt != nil {
t.Fatal("kick times must not stick to a closed drain")
}

d.decide("abc1234", inFlight, 45*time.Minute, now)
d.setKickTimes(agentKickTimes(statuses, inFlight))
r := d.Report(now.Add(time.Minute))
if r == nil {
t.Fatal("open drain must report")
}
want := []spoke.UpgradeTurn{{Agent: "reviewer", ElapsedS: 39 * 60}, {Agent: "scanner", ElapsedS: 13 * 60}}
if !reflect.DeepEqual(r.Turns, want) {
t.Fatalf("Turns = %+v, want %+v", r.Turns, want)
}
if got := spoke.FormatTurns(r.Turns, 3); got != "2/3 agents in turn: reviewer 39m, scanner 13m" {
t.Fatalf("FormatTurns = %q", got)
}

// Release clears the kick times with the rest of the drain inputs.
d.decide("abc1234", nil, 45*time.Minute, now.Add(2*time.Minute))
if d.kickAt != nil || d.Report(now.Add(2*time.Minute)) != nil {
t.Fatal("released drain must drop kick times and report nil")
}
if agentKickTimes(statuses, nil) != nil {
t.Fatal("no names, no map")
}
}
Loading
Loading