diff --git a/changelog.d/added-agent-session-recycle.md b/changelog.d/added-agent-session-recycle.md new file mode 100644 index 0000000000..e508621929 --- /dev/null +++ b/changelog.d/added-agent-session-recycle.md @@ -0,0 +1 @@ +- Agents: idle CLI sessions over an RSS (`agents.session_recycle.max_rss_mb`, default 1536 MiB) or age (`max_age`, default 6h) budget are recycled through the normal restart path, bounding the V8 heap growth that was OOM-killing the pod; the dashboard shows each agent's session RSS and age (#14842). diff --git a/src/crux/agent/manager.go b/src/crux/agent/manager.go index a17c323489..89207f2ebc 100644 --- a/src/crux/agent/manager.go +++ b/src/crux/agent/manager.go @@ -371,6 +371,12 @@ type AgentProcess struct { lastInferKickVisible string // stall watchdog: visible pane text just after kick delivery stallNudgeSent bool // stall watchdog: at most one nudge per kick lastSessionStallRestart time.Time // session-liveness: last restart for a hung Copilot session + // SessionRSSMB is the last sampled RSS of the CLI process tree (MiB); + // SessionRecycles counts #14842 policy recycles; lastSessionRecycle + // enforces the per-agent backoff after one. + SessionRSSMB int + SessionRecycles int + lastSessionRecycle time.Time // busyNoActivityRestartAt is when busyNoActivityStall last restarted this // agent, and busyNoActivityRestarts how many times it has in the current // episode. Both survive the restart's busy-visibility reset and clear on diff --git a/src/crux/agent/manager_pane_signals.go b/src/crux/agent/manager_pane_signals.go index 0692f87fce..648ddcf83a 100644 --- a/src/crux/agent/manager_pane_signals.go +++ b/src/crux/agent/manager_pane_signals.go @@ -517,6 +517,8 @@ func (a *AgentProcess) snapshot() crux.Process { WatchdogConditions: conds, StallNudges: a.StallNudges, StallRecoveries: a.StallRecoveries, + SessionRSSMB: a.SessionRSSMB, + SessionRecycles: a.SessionRecycles, CIPollNudges: a.CIPollNudges, AwaitingCI: a.AwaitingCI, ActionNudges: a.ActionNudges, diff --git a/src/crux/agent/manager_restart.go b/src/crux/agent/manager_restart.go index 9ba9826542..94eba1c472 100644 --- a/src/crux/agent/manager_restart.go +++ b/src/crux/agent/manager_restart.go @@ -78,6 +78,7 @@ func (m *Manager) CheckAndRestartCrashedAgents(ctx context.Context) []string { busyNoActivity bool } var hungSessions []sessionRestart + var recycleCandidates []sessionRecycleCandidate for name, agent := range m.agents { if agent.State != StateRunning { continue @@ -155,6 +156,10 @@ func (m *Manager) CheckAndRestartCrashedAgents(ctx context.Context) []string { // No replayed kick: the governor re-kicks once the pane is // idle, and the storm breaker bounds repeat restarts. hungSessions = append(hungSessions, sessionRestart{name: name, reason: reason, busyNoActivity: true}) + } else if agent.Config.SessionRecycle.IsEnabled() { + // #14842: a live, non-stalled session is sampled for RSS and + // may be recycled if over budget and idle (session_recycle.go). + recycleCandidates = append(recycleCandidates, sessionRecycleCandidate{name: name, pane: pane}) } // Plain claude agents launch with --dangerously-skip-permissions too, // so they can park on the same bypass consent screen (#14307). @@ -217,6 +222,8 @@ func (m *Manager) CheckAndRestartCrashedAgents(ctx context.Context) []string { } } + m.sampleAndRecycleSessions(ctx, recycleCandidates) + var restarted []string for _, crash := range crashed { name := crash.name diff --git a/src/crux/agent/session_recycle.go b/src/crux/agent/session_recycle.go new file mode 100644 index 0000000000..41137aeee1 --- /dev/null +++ b/src/crux/agent/session_recycle.go @@ -0,0 +1,323 @@ +package agent + +import ( + "bytes" + "context" + "fmt" + "os" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/hivecommons/hive/crux/config" +) + +// Session recycle (#14842). A CLI session's V8 heap grows with every turn +// and never shrinks on idle: on the hosted hive, active sessions gained +// 35–65 MiB per six minutes until the pod OOM'd. The health tick samples +// each agent's CLI process-tree RSS and, once a session is over its RSS or +// age budget AND idle at the input prompt, ends it through the ordinary +// restart path so claims survive exactly as they do on any other restart. + +const ( + sessionRecycleReason = "session recycle" + // sessionRecycleStartupGrace keeps a freshly launched session out of + // the policy while the CLI is still booting, regardless of config. + sessionRecycleStartupGrace = 5 * time.Minute +) + +// procTreeRSSMB sums VmRSS (MiB) over rootPid and every descendant found +// under root (a /proc mount). It scans the tree once, so the cost is one +// readdir plus one small read per process, not one per agent per process. +// Missing or unreadable entries (a child that exited mid-scan) are skipped. +func procTreeRSSMB(root string, rootPid int) int { + if rootPid <= 0 { + return 0 + } + entries, err := os.ReadDir(root) + if err != nil { + return 0 + } + children := map[int][]int{} + for _, e := range entries { + pid, err := strconv.Atoi(e.Name()) + if err != nil || pid <= 0 { + continue + } + ppid, ok := procPPID(filepath.Join(root, e.Name(), "stat")) + if !ok { + continue + } + children[ppid] = append(children[ppid], pid) + } + totalKB := 0 + seen := map[int]bool{} + stack := []int{rootPid} + for len(stack) > 0 { + pid := stack[len(stack)-1] + stack = stack[:len(stack)-1] + if seen[pid] { + continue + } + seen[pid] = true + totalKB += procVmRSSKB(filepath.Join(root, strconv.Itoa(pid), "status")) + stack = append(stack, children[pid]...) + } + return totalKB / 1024 +} + +// procPPID parses the parent pid (field 4) out of /proc//stat. The +// comm field is parenthesised and may itself contain spaces or parens, so +// the fields are taken from after the LAST ')'. +func procPPID(statPath string) (int, bool) { + data, err := os.ReadFile(statPath) + if err != nil { + return 0, false + } + i := bytes.LastIndexByte(data, ')') + if i < 0 { + return 0, false + } + fields := strings.Fields(string(data[i+1:])) + // fields[0] is state, fields[1] is ppid. + if len(fields) < 2 { + return 0, false + } + ppid, err := strconv.Atoi(fields[1]) + if err != nil { + return 0, false + } + return ppid, true +} + +// procVmRSSKB reads the VmRSS line of /proc//status; 0 when absent +// (kernel threads, or a process that exited between readdir and read). +func procVmRSSKB(statusPath string) int { + data, err := os.ReadFile(statusPath) + if err != nil { + return 0 + } + for _, line := range strings.Split(string(data), "\n") { + if !strings.HasPrefix(line, "VmRSS:") { + continue + } + fields := strings.Fields(strings.TrimPrefix(line, "VmRSS:")) + if len(fields) == 0 { + return 0 + } + kb, err := strconv.Atoi(fields[0]) + if err != nil { + return 0 + } + return kb + } + return 0 +} + +// tmuxPanePIDForAgent asks tmux for the shell pid of the agent's pane; the +// CLI and its helpers are that shell's descendants. +func (m *Manager) tmuxPanePIDForAgent(agent *AgentProcess) int { + if agent == nil || agent.tmuxSession == "" { + return 0 + } + out, err := m.tmuxCmd(agent, "display-message", "-p", "-t", agent.tmuxSession, "#{pane_pid}").Output() + if err != nil { + return 0 + } + pid, err := strconv.Atoi(strings.TrimSpace(string(out))) + if err != nil { + return 0 + } + return pid +} + +// sessionRecycleInput is everything the pure policy needs about one agent, +// gathered by the health tick so the decision itself has no clock, lock or +// tmux dependency and is table-testable. +type sessionRecycleInput struct { + Policy config.SessionRecycleConfig + RSSMB int + // StartedAt is when the current CLI session launched; zero means + // unknown, which never recycles on age. + StartedAt time.Time + // LastActivity is the latest of transcript activity, pane change and + // kick delivery; zero means "no evidence", which counts as idle only if + // the session is older than MinIdle. + LastActivity time.Time + LastRecycle time.Time + // Pane is the visible pane capture; the idle test is made against it. + Pane string + // KickInFlight is true while a kick is being typed, dispatched + // asynchronously, or queued for startup — the turn has not started + // in the pane yet, so the pane alone would look idle. + KickInFlight bool +} + +// sessionRecycleDecision returns the reason to recycle, or "" to leave the +// session alone. Over-budget is necessary but never sufficient: the session +// must also be idle at the CLI's input prompt with no turn pending, past +// the per-agent backoff, and silent for at least MinIdle. +func sessionRecycleDecision(in sessionRecycleInput, now time.Time) string { + p := in.Policy.Effective() + if p.Enabled == nil || !*p.Enabled { + return "" + } + if in.StartedAt.IsZero() || now.Sub(in.StartedAt) < sessionRecycleStartupGrace { + return "" + } + if !in.LastRecycle.IsZero() && now.Sub(in.LastRecycle) < config.SessionRecycleBackoff { + return "" + } + age := now.Sub(in.StartedAt) + var reason string + switch { + case in.RSSMB > p.MaxRSSMB: + reason = fmt.Sprintf("rss %d MiB over %d MiB budget", in.RSSMB, p.MaxRSSMB) + case age >= p.MaxAge: + reason = fmt.Sprintf("session age %s over %s budget", roundDurationForHumans(age), p.MaxAge) + default: + return "" + } + if in.KickInFlight { + return "" + } + if !paneIdleAtInputPrompt(in.Pane) { + return "" + } + last := in.LastActivity + if last.IsZero() { + last = in.StartedAt + } + if now.Sub(last) < p.MinIdle { + return "" + } + return reason +} + +// paneIdleAtInputPrompt is the "CLI reached its input prompt and is doing +// nothing" test: the CLI is up, shows its prompt, and shows none of the +// working, active-work, consent or blocking-prompt signatures. +func paneIdleAtInputPrompt(pane string) bool { + if !paneHasCLIMarker(pane) { + return false + } + if paneShowsAgentWorking(pane) || paneShowsActiveWork(pane) { + return false + } + if paneShowsConsentScreen(pane) || paneHasBlockingPrompt(pane) { + return false + } + return paneShowsInputPrompt(pane) +} + +// Seams for tests, like procRoot: the pane-pid lookup shells out to tmux and +// the restart tears down a real session. +var ( + sessionRecyclePanePID = (*Manager).tmuxPanePIDForAgent + sessionRecycleRestart = (*Manager).restartWithStormBreaker +) + +// sessionRecycleCandidate is one running agent the health tick saw with a +// live pane, carried across the RLock release so RSS sampling (a tmux exec +// and a /proc scan) happens unlocked. +type sessionRecycleCandidate struct { + name string + pane string +} + +// sampleAndRecycleSessions is the health-tick half of the policy: sample +// RSS for every candidate, publish it on the AgentProcess for the +// dashboard, and restart the ones the policy says to. Restarts go through +// the storm breaker like every other automatic restart. +func (m *Manager) sampleAndRecycleSessions(ctx context.Context, candidates []sessionRecycleCandidate) { + now := time.Now() + type recycle struct { + name, reason string + rssMB int + age time.Duration + } + var toRecycle []recycle + for _, c := range candidates { + m.mu.RLock() + agent := m.agents[c.name] + m.mu.RUnlock() + if agent == nil { + continue + } + rssMB := procTreeRSSMB(procRoot, sessionRecyclePanePID(m, agent)) + + m.mu.Lock() + agent, ok := m.agents[c.name] + if !ok || agent == nil || agent.State != StateRunning || agent.Paused { + m.mu.Unlock() + continue + } + if rssMB > 0 { + agent.SessionRSSMB = rssMB + } + in := sessionRecycleInput{ + Policy: agent.Config.SessionRecycle.Effective(), + RSSMB: agent.SessionRSSMB, + LastRecycle: agent.lastSessionRecycle, + Pane: c.pane, + KickInFlight: m.kickInFlightLocked(agent), + } + if agent.StartedAt != nil { + in.StartedAt = *agent.StartedAt + } + agent.paneMu.RLock() + paneChange := agent.LastPaneChange + agent.paneMu.RUnlock() + var lastKick time.Time + if agent.LastKick != nil { + lastKick = *agent.LastKick + } + in.LastActivity = latestTime(agent.LastTranscriptActivity, paneChange, lastKick) + reason := sessionRecycleDecision(in, now) + if reason != "" { + agent.lastSessionRecycle = now + agent.SessionRecycles++ + agent.LastError = "" + toRecycle = append(toRecycle, recycle{name: c.name, reason: reason, rssMB: in.RSSMB, age: now.Sub(in.StartedAt)}) + } + m.mu.Unlock() + } + for _, r := range toRecycle { + didRestart, err := sessionRecycleRestart(m, ctx, r.name, sessionRecycleReason, r.reason) + switch { + case err != nil: + m.logger.Error("agent session recycle failed", "name", r.name, "reason", r.reason, "error", err) + case didRestart: + m.logger.Info("agent session recycled", + "name", r.name, + "rss_mb", r.rssMB, + "age", roundDurationForHumans(r.age), + "reason", r.reason, + ) + } + } +} + +// kickInFlightLocked reports whether a kick is anywhere between "decided" +// and "visible in the pane": being typed, dispatched asynchronously, or +// queued for startup. Caller holds m.mu. +func (m *Manager) kickInFlightLocked(agent *AgentProcess) bool { + if agent.kickDelivering.Load() || agent.startupKickInFlight || agent.pendingStartupKick != "" { + return true + } + if d, ok := m.kickDispatches.get(agent.Name); ok && d.Pending() { + return true + } + return false +} + +func latestTime(ts ...time.Time) time.Time { + var out time.Time + for _, t := range ts { + if t.After(out) { + out = t + } + } + return out +} diff --git a/src/crux/agent/session_recycle_test.go b/src/crux/agent/session_recycle_test.go new file mode 100644 index 0000000000..b44adbb781 --- /dev/null +++ b/src/crux/agent/session_recycle_test.go @@ -0,0 +1,249 @@ +package agent + +import ( + "context" + "fmt" + "os" + "path/filepath" + "testing" + "time" + + "github.com/hivecommons/hive/crux/config" +) + +// writeFakeProc lays down /proc//{stat,status} for one process. comm +// deliberately contains a space and a ')' so the ppid parse has to anchor +// on the last ')' rather than the first. +func writeFakeProc(t *testing.T, root string, pid, ppid, rssKB int) { + t.Helper() + dir := filepath.Join(root, fmt.Sprint(pid)) + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + stat := fmt.Sprintf("%d (copilot (arm64) x) S %d %d %d 0 -1 4194560 0", pid, ppid, pid, pid) + if err := os.WriteFile(filepath.Join(dir, "stat"), []byte(stat), 0o644); err != nil { + t.Fatal(err) + } + status := fmt.Sprintf("Name:\tcopilot\nPid:\t%d\nPPid:\t%d\nVmPeak:\t%d kB\nVmRSS:\t%d kB\nThreads:\t12\n", pid, ppid, rssKB*2, rssKB) + if err := os.WriteFile(filepath.Join(dir, "status"), []byte(status), 0o644); err != nil { + t.Fatal(err) + } +} + +func TestProcTreeRSSMBSumsDescendantsOnly(t *testing.T) { + root := t.TempDir() + // pane shell 100 → copilot 200 → {helper 300, helper 301}; 400 is an + // unrelated sibling under init and must not be counted. 500 has no + // VmRSS line (kernel thread shape) and sits under the shell. + writeFakeProc(t, root, 1, 0, 1024) + writeFakeProc(t, root, 100, 1, 2*1024) + writeFakeProc(t, root, 200, 100, 900*1024) + writeFakeProc(t, root, 300, 200, 50*1024) + writeFakeProc(t, root, 301, 200, 60*1024) + writeFakeProc(t, root, 400, 1, 700*1024) + if err := os.MkdirAll(filepath.Join(root, "500"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, "500", "stat"), []byte("500 (kworker) S 100 0 0"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, "500", "status"), []byte("Name:\tkworker\n"), 0o644); err != nil { + t.Fatal(err) + } + // Non-pid entries (self, sys, …) are ignored. + if err := os.MkdirAll(filepath.Join(root, "self"), 0o755); err != nil { + t.Fatal(err) + } + + if got, want := procTreeRSSMB(root, 100), 2+900+50+60; got != want { + t.Fatalf("procTreeRSSMB(100) = %d, want %d", got, want) + } + if got, want := procTreeRSSMB(root, 200), 900+50+60; got != want { + t.Fatalf("procTreeRSSMB(200) = %d, want %d", got, want) + } + if got := procTreeRSSMB(root, 0); got != 0 { + t.Fatalf("procTreeRSSMB(0) = %d, want 0", got) + } + if got := procTreeRSSMB(filepath.Join(root, "missing"), 100); got != 0 { + t.Fatalf("procTreeRSSMB(missing root) = %d, want 0", got) + } + if got := procTreeRSSMB(root, 999); got != 0 { + t.Fatalf("procTreeRSSMB(unknown pid) = %d, want 0", got) + } +} + +func TestProcPPIDAnchorsOnLastParen(t *testing.T) { + dir := t.TempDir() + p := filepath.Join(dir, "stat") + if err := os.WriteFile(p, []byte("42 (a) b (c) d) R 7 42 42 0"), 0o644); err != nil { + t.Fatal(err) + } + if ppid, ok := procPPID(p); !ok || ppid != 7 { + t.Fatalf("procPPID = %d,%v want 7,true", ppid, ok) + } + if err := os.WriteFile(p, []byte("garbage"), 0o644); err != nil { + t.Fatal(err) + } + if _, ok := procPPID(p); ok { + t.Fatal("procPPID accepted garbage") + } + if _, ok := procPPID(filepath.Join(dir, "nope")); ok { + t.Fatal("procPPID accepted a missing file") + } +} + +const ( + recycleIdlePane = " Copilot CLI\n\n❯ \n / commands · ? help\n" + recycleBusyPane = " Copilot CLI\n\n◉ Working on it… esc interrupt\n" + recycleBarePane = "bash-5.2$ \n" +) + +func TestSessionRecycleDecision(t *testing.T) { + now := time.Date(2026, 10, 23, 3, 0, 0, 0, time.UTC) + policy := config.SessionRecycleConfig{MaxRSSMB: 1000, MaxAge: 6 * time.Hour, MinIdle: 2 * time.Minute} + off := false + base := func() sessionRecycleInput { + return sessionRecycleInput{ + Policy: policy, + RSSMB: 500, + StartedAt: now.Add(-time.Hour), + LastActivity: now.Add(-10 * time.Minute), + Pane: recycleIdlePane, + } + } + cases := []struct { + name string + mutate func(*sessionRecycleInput) + want bool + }{ + {"under both budgets", func(in *sessionRecycleInput) {}, false}, + {"over rss and idle", func(in *sessionRecycleInput) { in.RSSMB = 1500 }, true}, + {"over rss but pane busy", func(in *sessionRecycleInput) { in.RSSMB = 1500; in.Pane = recycleBusyPane }, false}, + {"over rss but no CLI in pane", func(in *sessionRecycleInput) { in.RSSMB = 1500; in.Pane = recycleBarePane }, false}, + {"over rss but kick in flight", func(in *sessionRecycleInput) { in.RSSMB = 1500; in.KickInFlight = true }, false}, + {"over rss but recent activity", func(in *sessionRecycleInput) { in.RSSMB = 1500; in.LastActivity = now.Add(-30 * time.Second) }, false}, + {"over rss, no activity evidence, old session", func(in *sessionRecycleInput) { in.RSSMB = 1500; in.LastActivity = time.Time{} }, true}, + {"over age and idle", func(in *sessionRecycleInput) { in.StartedAt = now.Add(-7 * time.Hour) }, true}, + {"over age but busy", func(in *sessionRecycleInput) { in.StartedAt = now.Add(-7 * time.Hour); in.Pane = recycleBusyPane }, false}, + {"over rss within backoff", func(in *sessionRecycleInput) { in.RSSMB = 1500; in.LastRecycle = now.Add(-5 * time.Minute) }, false}, + {"over rss past backoff", func(in *sessionRecycleInput) { in.RSSMB = 1500; in.LastRecycle = now.Add(-11 * time.Minute) }, true}, + {"over rss but within startup grace", func(in *sessionRecycleInput) { in.RSSMB = 1500; in.StartedAt = now.Add(-time.Minute) }, false}, + {"over rss but unknown start", func(in *sessionRecycleInput) { in.RSSMB = 1500; in.StartedAt = time.Time{} }, false}, + {"over rss but disabled", func(in *sessionRecycleInput) { in.RSSMB = 1500; in.Policy.Enabled = &off }, false}, + {"rss exactly at budget", func(in *sessionRecycleInput) { in.RSSMB = 1000 }, false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + in := base() + tc.mutate(&in) + got := sessionRecycleDecision(in, now) + if (got != "") != tc.want { + t.Fatalf("decision = %q, want recycle=%v", got, tc.want) + } + }) + } +} + +func TestSessionRecycleDecisionReasonNamesBudget(t *testing.T) { + now := time.Now() + in := sessionRecycleInput{ + Policy: config.SessionRecycleConfig{MaxRSSMB: 1000}, + RSSMB: 1200, + StartedAt: now.Add(-time.Hour), + LastActivity: now.Add(-time.Hour), + Pane: recycleIdlePane, + } + if got := sessionRecycleDecision(in, now); got != "rss 1200 MiB over 1000 MiB budget" { + t.Fatalf("rss reason = %q", got) + } + in.RSSMB = 10 + in.StartedAt = now.Add(-7 * time.Hour) + if got := sessionRecycleDecision(in, now); got != "session age 7h0m0s over 6h0m0s budget" { + t.Fatalf("age reason = %q", got) + } +} + +func TestSessionRecycleConfigEffectiveDefaults(t *testing.T) { + var nilCfg *config.SessionRecycleConfig + e := nilCfg.Effective() + if e.MaxRSSMB != config.DefaultSessionRecycleMaxRSSMB || e.MaxAge != config.DefaultSessionRecycleMaxAge || e.MinIdle != config.DefaultSessionRecycleMinIdle { + t.Fatalf("defaults = %+v", e) + } + if !nilCfg.IsEnabled() { + t.Fatal("nil policy should be enabled by default") + } + off := false + if (&config.SessionRecycleConfig{Enabled: &off}).IsEnabled() { + t.Fatal("enabled: false should disable") + } + partial := (&config.SessionRecycleConfig{MaxRSSMB: 2048}).Effective() + if partial.MaxRSSMB != 2048 || partial.MaxAge != config.DefaultSessionRecycleMaxAge { + t.Fatalf("partial = %+v", partial) + } +} + +func TestSampleAndRecycleSessionsPublishesRSSAndRestarts(t *testing.T) { + root := t.TempDir() + writeFakeProc(t, root, 100, 1, 1024) + writeFakeProc(t, root, 200, 100, 1700*1024) + oldRoot := procRoot + procRoot = root + t.Cleanup(func() { procRoot = oldRoot }) + + m := testManager(0) + started := time.Now().Add(-time.Hour) + agent := &AgentProcess{ + Name: "scanner", + State: StateRunning, + StartedAt: &started, + LastTranscriptActivity: time.Now().Add(-10 * time.Minute), + tmuxSession: "hive-scanner", + } + m.agents["scanner"] = agent + fakePanePID := 100 + oldPID, oldRestart := sessionRecyclePanePID, sessionRecycleRestart + t.Cleanup(func() { sessionRecyclePanePID, sessionRecycleRestart = oldPID, oldRestart }) + sessionRecyclePanePID = func(*Manager, *AgentProcess) int { return fakePanePID } + var restarted []string + sessionRecycleRestart = func(_ *Manager, _ context.Context, name, reason, detail string) (bool, error) { + restarted = append(restarted, name+"|"+reason+"|"+detail) + return true, nil + } + + m.sampleAndRecycleSessions(t.Context(), []sessionRecycleCandidate{{name: "scanner", pane: recycleIdlePane}}) + if agent.SessionRSSMB != 1701 { + t.Fatalf("SessionRSSMB = %d, want 1701", agent.SessionRSSMB) + } + if len(restarted) != 1 || restarted[0] != "scanner|session recycle|rss 1701 MiB over 1536 MiB budget" { + t.Fatalf("restarts = %v", restarted) + } + if agent.SessionRecycles != 1 || agent.lastSessionRecycle.IsZero() { + t.Fatalf("recycle bookkeeping not recorded: %+v", agent) + } + + // Immediately over budget again: the per-agent backoff holds. + m.sampleAndRecycleSessions(t.Context(), []sessionRecycleCandidate{{name: "scanner", pane: recycleIdlePane}}) + if len(restarted) != 1 { + t.Fatalf("backoff ignored: restarts = %v", restarted) + } + + // Busy pane, past backoff: RSS is still published but no restart. + agent.lastSessionRecycle = time.Now().Add(-config.SessionRecycleBackoff - time.Minute) + agent.SessionRSSMB = 0 + m.sampleAndRecycleSessions(t.Context(), []sessionRecycleCandidate{{name: "scanner", pane: recycleBusyPane}}) + if agent.SessionRSSMB != 1701 || len(restarted) != 1 { + t.Fatalf("busy pane: rss=%d restarts=%v", agent.SessionRSSMB, restarted) + } + + // Pane pid unknown: the last good sample is kept, nothing restarts. + fakePanePID = 0 + m.sampleAndRecycleSessions(t.Context(), []sessionRecycleCandidate{{name: "scanner", pane: recycleBusyPane}}) + if agent.SessionRSSMB != 1701 { + t.Fatalf("stale sample overwritten: %d", agent.SessionRSSMB) + } + + // Snapshot carries the fields to the dashboard. + if p := agent.snapshot(); p.SessionRSSMB != 1701 || p.SessionRecycles != 1 { + t.Fatalf("snapshot = rss %d recycles %d", p.SessionRSSMB, p.SessionRecycles) + } +} diff --git a/src/crux/config/agent_config.go b/src/crux/config/agent_config.go index 7c0edcb551..5e2c11a94f 100644 --- a/src/crux/config/agent_config.go +++ b/src/crux/config/agent_config.go @@ -38,10 +38,10 @@ type SandboxJobConfig struct { // gate device-requesting pods behind a custom scheduler via an admission // webhook (e.g. `A pod must use "schedulerName: spyre-scheduler"`); // without this the Job create is denied before any pod exists. - SchedulerName string `yaml:"scheduler_name,omitempty" json:"scheduler_name,omitempty"` - EnvFromSecrets []string `yaml:"env_from_secrets,omitempty" json:"env_from_secrets,omitempty"` - Volumes []SandboxJobVolume `yaml:"volumes,omitempty" json:"volumes,omitempty"` - TTLSeconds int `yaml:"ttl_seconds,omitempty" json:"ttl_seconds,omitempty"` + SchedulerName string `yaml:"scheduler_name,omitempty" json:"scheduler_name,omitempty"` + EnvFromSecrets []string `yaml:"env_from_secrets,omitempty" json:"env_from_secrets,omitempty"` + Volumes []SandboxJobVolume `yaml:"volumes,omitempty" json:"volumes,omitempty"` + TTLSeconds int `yaml:"ttl_seconds,omitempty" json:"ttl_seconds,omitempty"` } // SandboxJobToleration mirrors the subset of a Kubernetes toleration an @@ -131,6 +131,10 @@ type AgentRuntimeConfig struct { // (the default cap). Exported because the Config YAML hooks in // pkg/config set and read it; `yaml:"-"` keeps it off the wire. AgentsGitHubAPIHourlyCapSet bool `yaml:"-" json:"-"` + // AgentsSessionRecycle is the fleet default for the CLI session RSS/age + // recycle policy (#14842). It is encoded under agents.session_recycle by + // Config's YAML hooks, like AgentsGitHubAPIHourlyCap. + AgentsSessionRecycle SessionRecycleConfig `yaml:"-" json:"-"` // Backends is the spoke-wide backend allow/deny list (#11310), enforced // at pack apply and by PUT /api/config/agent/{name}/models. Backends BackendsConfig `yaml:"backends,omitempty" json:"backends,omitempty"` @@ -313,8 +317,13 @@ type AgentConfig struct { // turn. 0 disables it; exceeding it raises busy-over-ceiling but never // interrupts the agent. MaxTurnDuration time.Duration `yaml:"max_turn_duration,omitempty" json:"max_turn_duration,omitempty"` - DisplayName string `yaml:"display_name" json:"display_name,omitempty"` - Description string `yaml:"description" json:"description,omitempty"` + // SessionRecycle overrides the fleet agents.session_recycle policy for + // this agent (#14842). Nil inherits the fleet value; EnabledAgents() + // stamps the fleet block into nil pointers so the agent manager sees one + // resolved policy per agent. + SessionRecycle *SessionRecycleConfig `yaml:"session_recycle,omitempty" json:"session_recycle,omitempty"` + DisplayName string `yaml:"display_name" json:"display_name,omitempty"` + Description string `yaml:"description" json:"description,omitempty"` // Bob holds bob-backend-only per-agent options. Empty is default-off. Bob AgentBobConfig `yaml:"bob,omitempty" json:"bob,omitempty"` // BobDisplayName is the legacy flat spelling for Bob.SessionLabel. It is diff --git a/src/crux/config/session_recycle.go b/src/crux/config/session_recycle.go new file mode 100644 index 0000000000..cfa2e0f793 --- /dev/null +++ b/src/crux/config/session_recycle.go @@ -0,0 +1,71 @@ +package config + +import "time" + +// Session-recycle defaults (#14842). A CLI session's V8 heap grows with +// every turn and never shrinks on idle, so an agent that keeps working +// climbs past a gigabyte within hours and the pod eventually OOMs. The +// recycle policy ends an idle session once it is over an RSS or age budget +// and relaunches it through the ordinary restart path. +const ( + DefaultSessionRecycleMaxRSSMB = 1536 + DefaultSessionRecycleMaxAge = 6 * time.Hour + DefaultSessionRecycleMinIdle = 2 * time.Minute + // SessionRecycleBackoff is how long after a recycle the same agent is + // left alone, even if it is immediately over budget again. + SessionRecycleBackoff = 10 * time.Minute +) + +// SessionRecycleConfig bounds the resident memory and age of an agent's CLI +// session. It is written fleet-wide under agents.session_recycle and may be +// overridden per agent under agents..session_recycle. Zero fields take +// the defaults; the policy is on unless Enabled is explicitly false. +type SessionRecycleConfig struct { + // Enabled turns the policy off when explicitly false. Nil means on. + Enabled *bool `yaml:"enabled,omitempty" json:"enabled,omitempty"` + // MaxRSSMB is the resident-set budget for the whole CLI process tree + // under the agent's tmux pane, in MiB. 0 means the default (1536). + MaxRSSMB int `yaml:"max_rss_mb,omitempty" json:"max_rss_mb,omitempty"` + // MaxAge recycles a session older than this even if under the RSS + // budget. 0 means the default (6h). + MaxAge time.Duration `yaml:"max_age,omitempty" json:"max_age,omitempty"` + // MinIdle is how long the pane and transcript must have been silent at + // the input prompt before a recycle is allowed. 0 means the default (2m). + MinIdle time.Duration `yaml:"min_idle,omitempty" json:"min_idle,omitempty"` +} + +// IsZero reports whether no field was set, so an absent block encodes as +// nothing and a per-agent nil pointer inherits the fleet value. +func (c SessionRecycleConfig) IsZero() bool { + return c.Enabled == nil && c.MaxRSSMB == 0 && c.MaxAge == 0 && c.MinIdle == 0 +} + +// Effective returns the policy with defaults filled in. The receiver may be +// nil (an agent with no override and no fleet block), which yields the +// defaults with the policy enabled. +func (c *SessionRecycleConfig) Effective() SessionRecycleConfig { + out := SessionRecycleConfig{} + if c != nil { + out = *c + } + if out.MaxRSSMB <= 0 { + out.MaxRSSMB = DefaultSessionRecycleMaxRSSMB + } + if out.MaxAge <= 0 { + out.MaxAge = DefaultSessionRecycleMaxAge + } + if out.MinIdle <= 0 { + out.MinIdle = DefaultSessionRecycleMinIdle + } + if out.Enabled == nil { + on := true + out.Enabled = &on + } + return out +} + +// IsEnabled is Effective().Enabled as a plain bool. +func (c *SessionRecycleConfig) IsEnabled() bool { + e := c.Effective() + return e.Enabled != nil && *e.Enabled +} diff --git a/src/crux/process.go b/src/crux/process.go index 0f31958a1c..04d55d8401 100644 --- a/src/crux/process.go +++ b/src/crux/process.go @@ -16,50 +16,55 @@ import ( // The field meanings are documented on agent.AgentProcess. A snapshot is a // read-only display value; writing to one does not change the runtime. type Process struct { - Name string - ID string - Config config.AgentConfig - State ProcessState - PID int - UID int - StartedAt *time.Time - LastKick *time.Time - KicksUndeliverable int - BusySince time.Time - LastTranscriptActivity time.Time - BusyCondition string - BusyConditionMessage string - Paused bool - PausedAt time.Time - PausedReason string - PausedTrigger string - PausedBy string - OperatorStopped bool - PinnedCLI string - PinnedModel string - ModelOverride string - BackendOverride string - RestartCount int - RestartEvents []RestartEvent - LastRestartReason string - OutputBuffer *RingBuffer - SubAgentModels []SubAgentModel - KickHistory []KickRecord - LastKickMessage string - KickRefused bool - KickRefusalReason string - LaunchedMode AgentMode - HasLaunched bool - BootstrapOverride string - LastError string - NeedsLogin bool - LoginURL string - Starting bool - QuotaExhausted bool - WatchdogConditions []Condition - LastPaneChange time.Time - StallNudges int - StallRecoveries int + Name string + ID string + Config config.AgentConfig + State ProcessState + PID int + UID int + StartedAt *time.Time + LastKick *time.Time + KicksUndeliverable int + BusySince time.Time + LastTranscriptActivity time.Time + BusyCondition string + BusyConditionMessage string + Paused bool + PausedAt time.Time + PausedReason string + PausedTrigger string + PausedBy string + OperatorStopped bool + PinnedCLI string + PinnedModel string + ModelOverride string + BackendOverride string + RestartCount int + RestartEvents []RestartEvent + LastRestartReason string + OutputBuffer *RingBuffer + SubAgentModels []SubAgentModel + KickHistory []KickRecord + LastKickMessage string + KickRefused bool + KickRefusalReason string + LaunchedMode AgentMode + HasLaunched bool + BootstrapOverride string + LastError string + NeedsLogin bool + LoginURL string + Starting bool + QuotaExhausted bool + WatchdogConditions []Condition + LastPaneChange time.Time + StallNudges int + StallRecoveries int + // SessionRSSMB is the resident memory of the CLI process tree under the + // agent's tmux pane, in MiB, as of the last health tick (#14842). 0 when + // never sampled. SessionRecycles counts policy recycles since boot. + SessionRSSMB int + SessionRecycles int CIPollNudges int AwaitingCI bool TransientNudges int diff --git a/src/docs/agent-configuration.md b/src/docs/agent-configuration.md index 8883cd3be5..de50d07405 100644 --- a/src/docs/agent-configuration.md +++ b/src/docs/agent-configuration.md @@ -287,6 +287,10 @@ spec omits `launch_cmd`, Hive still builds the backend command normally. # kick_failed_repeatedly self-report is filed # only if kicks still fail after that restart max_turn_duration: 0s # optional visibility-only Working ceiling; 0 disables + session_recycle: # optional override of the fleet agents.session_recycle + max_rss_mb: 1536 # policy: recycle this agent's CLI session (a normal + max_age: 6h # restart) once idle at the prompt and over the RSS + min_idle: 2m # or age budget; enabled: false turns it off restart_strategy: immediate # how to bring a dead session back beads_dir: /data/beads/scanner # work-record (bead) storage; default per-agent replicas: 3 # materialize scanner, scanner-2, scanner-3 (max 5) diff --git a/src/docs/api-reference.md b/src/docs/api-reference.md index dd06ec1418..5f3bcb006f 100644 --- a/src/docs/api-reference.md +++ b/src/docs/api-reference.md @@ -13,17 +13,17 @@ History point limits, timestamp cutoffs, incremental cost reads and ETag/304 beh | `GET` | `/api/version` | Dashboard auth/session | Build/version metadata; includes `upgradeMarker` (`target`, `current`, `attempts`, `maxAttempts`, `failed`, `requestedAt`, `lastError`) while a self-upgrade is in flight or has failed ([#6765](https://github.com/hivecommons/hive/issues/6765)), the hub-delivered `upgradePolicy` (`hub_managed`, `spoke_managed`, `schedule`, `schedule_hour`, `schedule_timezone`, `schedule_weekday`, `paused`, `branch`, `channel`, `target_sha`, `target_resolved`, `armed_target`, and `next_update_at` — the hub's RFC3339 ETA for the next promotion into the `stable` channel, omitted when unknown ([#10256](https://github.com/hivecommons/hive/issues/10256))) when available, and an `autoUpdate` object (`enabled`, `state` — one of `disabled`/`up_to_date`/`behind`/`retrying`/`failed`/`unknown` — `healthy`, `period`, `targetBranch`, `targetCommit`, `currentCommit`, `commitsBehind`, `lastAttemptAt`, `lastError`, `detail`, and `nextUpdateAt` — the hub's RFC3339 ETA for the next update, omitted when unknown, disabled or paused ([#10257](https://github.com/hivecommons/hive/issues/10257))) that never reports a failed or unknown update as healthy ([#6962](https://github.com/hivecommons/hive/issues/6962), [#6963](https://github.com/hivecommons/hive/issues/6963)) | `pkg/dashboard/api.go:42` | | `GET` | `/api/version/release-notes` | Dashboard auth/session | Changelog gap between two revisions: `?from=&to=` returns `{from:{sha,version?}, to:{sha,ref,version?}, sections:[{version,date,categories:{added,changed,fixed,security,deprecated}}], unreleased?, truncated, source, error?}` — the `CHANGELOG.md` release sections present at `to` and absent at `from`, newest first, plus (when `to` is not a release tag) the `changelog.d/` fragments new at `to` under an `Unreleased (in this build)` section. Fetch failures return `source: "unavailable"` with `error` rather than an error status; results are cached per (from, to) for an hour ([#11065](https://github.com/hivecommons/hive/issues/11065)) | `pkg/dashboard/api.go:43` | | `POST` | `/api/release-channel` | Owner only | Hosted spoke self-service release-channel selector; relays `stable`/`candidate`/`edge` to the hub's existing switch-branch endpoint with the spoke dashboard-token proof, and reports the requested channel as pending until the Deployment image lands | `pkg/dashboard/api.go:64` | -| `GET` | `/api/health` | Public | Basic health probe | `pkg/dashboard/server.go:1511` | -| `GET` | `/api/health/deep` | Public | Deep health probe | `pkg/dashboard/server.go:1512` | -| `GET` | `/api/livez` | Public | Kubernetes liveness probe | `pkg/dashboard/server.go:1513` | -| `GET` | `/metrics` | Registered only when `HIVE_METRICS_ENABLED`; requires `Authorization: ****** (403 if the token is unset) | Prometheus metrics | `pkg/dashboard/server.go:1519` | -| `GET` | `/api/status` | Dashboard auth/session | Dashboard aggregate status; `labelPresets[]` (`{key,label,description,labels[],states[]}`) lists the built-in label-filter presets for the Projects cards, led by `needs-human` whose `labels[]` are the effective hard-suppress labels plus `needs-signal`, the hold labels, `stage/071-needs-author` and the reporter-trust awaiting label (#12172); accepts `?fields=a,b` for top-level selection and `?omit=repos` to drop heavy sections, and honors `Accept-Encoding: gzip`; `agents[]` includes busy-pane visibility fields (`kicksUndeliverable`, `busySince`, `lastTranscriptActivity`, `condition`, `conditionMessage`) when a Working pane cannot accept kicks; a working agent also carries `agents[].run` — the live per-run line (#14576): `{since, elapsedS, costUSD, tokens, priced, tests, filesChanged}`, the estimated spend/tokens since the kick was delivered (`lastKickAt`, falling back to `busySince`) (delta of the token collector's per-agent cumulative figure; `priced:false` when the agent has none yet), the last visible test-runner result (`pass`/`fail`/empty) and the last `N files changed` git summary (`-1` when none) parsed from the captured pane; top-level `fleetRun` `{running, hourUSD, hourPriced}` is the governor strip's `running N · $X this hour` figure; `sla` (fleet) and `repos[].sla` (per repo) carry the Delivery SLA block derived from the lifecycle timeline with no extra GitHub calls: `windows.{7d,30d}.{open_to_triage,triage_to_start,start_to_pr,pr_to_merge,open_to_merge}` as `{p50_h,p95_h,n}` (hours, nearest-rank), `history[]` of up to 30 daily buckets `{day,start_to_pr_p50,pr_to_merge_p50,open_to_merge_p50,n_started,n_pr,n_merged}` persisted under `/data/sla-history.json`, `stale_in_progress[]` (`{number,repo,started_at,age_h,agent}` — issues carrying `hive/in-progress` longer than the agent claim TTL with no linked PR), `stale_count`, `merged_7d`, `claim_ttl_s` and, on the fleet block, the effective `targets` (`start_to_pr_h`, `pr_to_merge_h`, `open_to_merge_h`); `staleSince` and `repos[].countsStale` / `countsStaleSince` mark totals served from last-good figures — the previous enumeration (#13382) or the ForgeState cache's last-good counts (#13376) | `pkg/dashboard/server.go:1524` | -| `GET` | `/api/status/summary` | Dashboard auth/session | Compact agent/governor/budget status summary for pollers; honors `Accept-Encoding: gzip` | `pkg/dashboard/server.go:1525` | -| `GET` | `/api/overview/issues.csv` | Dashboard auth/session | Cached Overview issue-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1526` | -| `GET` | `/api/overview/issues.json` | Dashboard auth/session | Cached Overview issue-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1527` | -| `GET` | `/api/overview/prs.csv` | Dashboard auth/session | Cached Overview PR-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1528` | -| `GET` | `/api/overview/prs.json` | Dashboard auth/session | Cached Overview PR-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1529` | -| `GET` | `/api/events` | Dashboard auth/session | Server-sent event stream | `pkg/dashboard/server.go:1530` | +| `GET` | `/api/health` | Public | Basic health probe | `pkg/dashboard/server.go:1518` | +| `GET` | `/api/health/deep` | Public | Deep health probe | `pkg/dashboard/server.go:1519` | +| `GET` | `/api/livez` | Public | Kubernetes liveness probe | `pkg/dashboard/server.go:1520` | +| `GET` | `/metrics` | Registered only when `HIVE_METRICS_ENABLED`; requires `Authorization: ****** (403 if the token is unset) | Prometheus metrics | `pkg/dashboard/server.go:1526` | +| `GET` | `/api/status` | Dashboard auth/session | Dashboard aggregate status; `labelPresets[]` (`{key,label,description,labels[],states[]}`) lists the built-in label-filter presets for the Projects cards, led by `needs-human` whose `labels[]` are the effective hard-suppress labels plus `needs-signal`, the hold labels, `stage/071-needs-author` and the reporter-trust awaiting label (#12172); accepts `?fields=a,b` for top-level selection and `?omit=repos` to drop heavy sections, and honors `Accept-Encoding: gzip`; `agents[]` includes busy-pane visibility fields (`kicksUndeliverable`, `busySince`, `lastTranscriptActivity`, `condition`, `conditionMessage`) when a Working pane cannot accept kicks; a working agent also carries `agents[].run` — the live per-run line (#14576): `{since, elapsedS, costUSD, tokens, priced, tests, filesChanged}`, the estimated spend/tokens since the kick was delivered (`lastKickAt`, falling back to `busySince`) (delta of the token collector's per-agent cumulative figure; `priced:false` when the agent has none yet), the last visible test-runner result (`pass`/`fail`/empty) and the last `N files changed` git summary (`-1` when none) parsed from the captured pane; top-level `fleetRun` `{running, hourUSD, hourPriced}` is the governor strip's `running N · $X this hour` figure; `sla` (fleet) and `repos[].sla` (per repo) carry the Delivery SLA block derived from the lifecycle timeline with no extra GitHub calls: `windows.{7d,30d}.{open_to_triage,triage_to_start,start_to_pr,pr_to_merge,open_to_merge}` as `{p50_h,p95_h,n}` (hours, nearest-rank), `history[]` of up to 30 daily buckets `{day,start_to_pr_p50,pr_to_merge_p50,open_to_merge_p50,n_started,n_pr,n_merged}` persisted under `/data/sla-history.json`, `stale_in_progress[]` (`{number,repo,started_at,age_h,agent}` — issues carrying `hive/in-progress` longer than the agent claim TTL with no linked PR), `stale_count`, `merged_7d`, `claim_ttl_s` and, on the fleet block, the effective `targets` (`start_to_pr_h`, `pr_to_merge_h`, `open_to_merge_h`); `staleSince` and `repos[].countsStale` / `countsStaleSince` mark totals served from last-good figures — the previous enumeration (#13382) or the ForgeState cache's last-good counts (#13376) | `pkg/dashboard/server.go:1531` | +| `GET` | `/api/status/summary` | Dashboard auth/session | Compact agent/governor/budget status summary for pollers; honors `Accept-Encoding: gzip` | `pkg/dashboard/server.go:1532` | +| `GET` | `/api/overview/issues.csv` | Dashboard auth/session | Cached Overview issue-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1533` | +| `GET` | `/api/overview/issues.json` | Dashboard auth/session | Cached Overview issue-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1534` | +| `GET` | `/api/overview/prs.csv` | Dashboard auth/session | Cached Overview PR-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1535` | +| `GET` | `/api/overview/prs.json` | Dashboard auth/session | Cached Overview PR-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1536` | +| `GET` | `/api/events` | Dashboard auth/session | Server-sent event stream | `pkg/dashboard/server.go:1537` | | `GET` | `/api/swarm` | Dashboard auth/session | Current swarm status: display name, duration, active repo, expiry, persisted prep metrics when available, and idle-unlock fields. | `pkg/dashboard/swarm.go:310` | | `POST` | `/api/swarm` | Owner only | Start a 24h repo swarm for a configured repo, returning 409 when another swarm is active and 423 when a previous swarm keeps the next swarm locked until enough agents are idle unless body `{"repo":"owner/name","force":true}` is used; saves the active record before repo prep, persists prep metrics, and announces the start to Discord when configured. | `pkg/dashboard/swarm.go:311` | | `DELETE` | `/api/swarm` | Owner only | End the active swarm, score issues closed and PRs merged in the window, append it to history, and announce the result to Discord when configured. | `pkg/dashboard/swarm.go:312` | @@ -66,7 +66,7 @@ History point limits, timestamp cutoffs, incremental cost reads and ETag/304 beh | Method | Path | Auth | Purpose | Source | |---|---|---|---|---| | `GET` | `/api/style` | Public | Sanitized custom dashboard CSS | `pkg/dashboard/api.go:44` | -| `GET` | `/branding/custom.css` | Dashboard auth/session | Operator branding stylesheet override, read per request (see [branding](branding.md)) | `pkg/dashboard/server.go:1591` | +| `GET` | `/branding/custom.css` | Dashboard auth/session | Operator branding stylesheet override, read per request (see [branding](branding.md)) | `pkg/dashboard/server.go:1598` | | `GET` | `/api/snapshot/frame-ancestors` | Public | Snapshot framing allowlist | `pkg/dashboard/api.go:77` | | `GET` | `/api/snapshot` | Public | Snapshot data | `pkg/dashboard/api.go:78` | | `GET` | `/snapshot` | Public | Public read-only snapshot page | `pkg/dashboard/api.go:79` | @@ -94,9 +94,9 @@ History point limits, timestamp cutoffs, incremental cost reads and ETag/304 beh | `GET` | `/api/openrouter/models` | Dashboard auth/session | Open Router Models | `pkg/dashboard/openrouter.go:47` | | `GET` | `/api/openrouter/credit` | Dashboard auth/session | Open Router Credit | `pkg/dashboard/openrouter.go:48` | | `GET` | `/openrouter/callback` | Public | Open Router Callback | `pkg/dashboard/openrouter.go:49` | -| `POST` | `/api/github-app/recheck` | Dashboard auth/session | GitHub App Recheck | `pkg/dashboard/server.go:1531` | -| `POST` | `/api/github-app/install-clicked` | Dashboard auth/session | GitHub App Install Clicked | `pkg/dashboard/server.go:1532` | -| `GET` | `/gh-setup` | Public | GitHub App Setup Callback | `pkg/dashboard/server.go:1533` | +| `POST` | `/api/github-app/recheck` | Dashboard auth/session | GitHub App Recheck | `pkg/dashboard/server.go:1538` | +| `POST` | `/api/github-app/install-clicked` | Dashboard auth/session | GitHub App Install Clicked | `pkg/dashboard/server.go:1539` | +| `GET` | `/gh-setup` | Public | GitHub App Setup Callback | `pkg/dashboard/server.go:1540` | ## Configuration @@ -593,7 +593,7 @@ always resolved server-side from the validated token. | `POST` | `/api/hives/{id}/heartbeat` | Dashboard auth/session | Hives Heartbeat | `pkg/dashboard/api_contribute.go:275` | | `DELETE` | `/api/hives/{id}` | Owner only | Hives Delete | `pkg/dashboard/api_contribute.go:276` | | `POST` | `/api/hives/onboard` | Dashboard auth/session | Hives Onboard | `pkg/dashboard/api_contribute.go:277` | -| `GET` | `/sso` | Public | SSO | `pkg/dashboard/server.go:1538` | +| `GET` | `/sso` | Public | SSO | `pkg/dashboard/server.go:1545` | ## Hub SaaS diff --git a/src/docs/operator-reference.md b/src/docs/operator-reference.md index 82c2bc9a49..02046c1f2a 100644 --- a/src/docs/operator-reference.md +++ b/src/docs/operator-reference.md @@ -206,7 +206,7 @@ The keys the agent runtime owns — the part of Hive being carved out as **Crux* | Struct | Embedded in | YAML keys (unchanged position) | |---|---|---| -| `AgentRuntimeConfig` (`pkg/config/agent_config.go`) | `Config` | `agents` (including the `agents.github_api_hourly_cap` scalar), `backends`, `providers`, `tool_approval`, `agent_sandbox`, `turn` | +| `AgentRuntimeConfig` (`pkg/config/agent_config.go`) | `Config` | `agents` (including the `agents.github_api_hourly_cap` scalar and the `agents.session_recycle` block), `backends`, `providers`, `tool_approval`, `agent_sandbox`, `turn` | | `AgentRuntimeGovernorConfig` (`pkg/config/agent_config.go`) | `GovernorConfig` | `governor.watchdog`, `governor.litellm`, `governor.vllm`, `governor.llm-d`, `governor.bob`, `governor.trajectory`, `governor.kick_limits`, `governor.gateways`, `governor.rotation`, `governor.provider_budget` | Go callers keep reading `cfg.Agents`, `cfg.Governor.Gateways` and so on through field promotion; only composite literals name the embedded struct (`config.Config{AgentRuntimeConfig: config.AgentRuntimeConfig{Agents: …}}`). `pkg/config/testdata/agent_runtime_golden.{yaml,json}` pins the parse of a representative file so the boundary cannot drift the schema. @@ -1138,6 +1138,33 @@ dashboard shows "Self-reports: N open (last: → #NNN)" in the Fleet Report panel and the toggle/repo/cap under Settings → Governor → "Self-Reported Issues". Full details: [fleet-report.md](fleet-report.md). +## Agent CLI session recycling + +Each agent's CLI (Copilot, Claude, …) is a long-lived Node/V8 process whose +heap grows with every turn and never shrinks while idle. The health tick +samples the resident memory of the whole process tree under each agent's +tmux pane and recycles a session that is over budget — but only while the +agent is idle at its input prompt with no kick pending, so a turn is never +interrupted. A recycle is an ordinary restart (`last_restart_reason: session +recycle`): claims, queues and the kick log are preserved exactly as on any +other restart, and the governor re-kicks on the next cadence. After a recycle +the same agent is left alone for ten minutes even if it is over budget again. + +```yaml +agents: + session_recycle: # fleet default; on unless enabled: false + max_rss_mb: 1536 # recycle an idle session whose process tree exceeds this (MiB) + max_age: 6h # …or whose CLI session is older than this + min_idle: 2m # pane + transcript must be silent this long first + scanner: + session_recycle: # optional per-agent override; absent fields inherit the fleet value + max_rss_mb: 2048 +``` + +The sampled value is shown as **Session RSS** (with the session age and a +`↻N` recycle count) in the agent detail panel. The log line is +`agent session recycled` with `rss_mb`, `age` and `reason`. + ## GitHub API quota Agents reach `api.github.com` through the Hive GitHub proxy, which injects the diff --git a/src/internal/settingsparity/settingsparity.go b/src/internal/settingsparity/settingsparity.go index 3fcfdfaa47..c97bd4f842 100644 --- a/src/internal/settingsparity/settingsparity.go +++ b/src/internal/settingsparity/settingsparity.go @@ -117,6 +117,12 @@ func ConfigLeaves(repoRoot string) ([]Leaf, error) { } walk(structs["Config"], "") leaves = append(leaves, Leaf{Path: "agents.github_api_hourly_cap", Type: "int"}) + leaves = append(leaves, + Leaf{Path: "agents.session_recycle.enabled", Type: "*bool"}, + Leaf{Path: "agents.session_recycle.max_rss_mb", Type: "int"}, + Leaf{Path: "agents.session_recycle.max_age", Type: "time.Duration"}, + Leaf{Path: "agents.session_recycle.min_idle", Type: "time.Duration"}, + ) byPath := map[string]Leaf{} for _, leaf := range leaves { if leaf.Path == "" || strings.Contains(leaf.Path, ".-") { diff --git a/src/internal/settingsparity/settingsparity_test.go b/src/internal/settingsparity/settingsparity_test.go index 2849514a9c..625ce85d7f 100644 --- a/src/internal/settingsparity/settingsparity_test.go +++ b/src/internal/settingsparity/settingsparity_test.go @@ -98,6 +98,10 @@ func TestConfigLeaves(t *testing.T) { want := []Leaf{ {Path: "agents..model", Type: "string"}, {Path: "agents.github_api_hourly_cap", Type: "int"}, + {Path: "agents.session_recycle.enabled", Type: "*bool"}, + {Path: "agents.session_recycle.max_age", Type: "time.Duration"}, + {Path: "agents.session_recycle.max_rss_mb", Type: "int"}, + {Path: "agents.session_recycle.min_idle", Type: "time.Duration"}, {Path: "callback", Type: "unknown"}, {Path: "clock", Type: "time.Duration"}, {Path: "hooks[].url", Type: "string"}, diff --git a/src/internal/tools/settingsparity/main_test.go b/src/internal/tools/settingsparity/main_test.go index 3497b5ce1f..febd8eced7 100644 --- a/src/internal/tools/settingsparity/main_test.go +++ b/src/internal/tools/settingsparity/main_test.go @@ -68,6 +68,10 @@ func TestRunMarkdown(t *testing.T) { "| key | type | default | exposed-in-UI | where | notes |", "| --- | --- | --- | --- | --- | --- |", "| `agents.github_api_hourly_cap` | `int` | | none | | may require restart or reconnect |", + "| `agents.session_recycle.enabled` | `*bool` | | none | | |", + "| `agents.session_recycle.max_age` | `time.Duration` | | none | | |", + "| `agents.session_recycle.max_rss_mb` | `int` | | none | | |", + "| `agents.session_recycle.min_idle` | `time.Duration` | | none | | |", "| `hub.token` | `string` | | none | | secret/credential — expose as masked or write-only |", "| `hub.url` | `string` | https://hub.example | read/write | Settings dialog reference | may require restart or reconnect |", "| `name` | `string` | hive | read/write | Settings dialog reference | |", @@ -89,12 +93,12 @@ func TestRunJSON(t *testing.T) { if err := json.Unmarshal(stdout.Bytes(), &rows); err != nil { t.Fatalf("decode JSON output: %v\n%s", err, stdout.String()) } - if len(rows) != 5 { - t.Fatalf("got %d rows, want 5: %+v", len(rows), rows) + if len(rows) != 9 { + t.Fatalf("got %d rows, want 9: %+v", len(rows), rows) } want := row{Key: "hub.url", Type: "string", Default: "https://hub.example", UI: "read/write", Where: "Settings dialog reference", Notes: "may require restart or reconnect"} - if rows[2] != want { - t.Fatalf("rows[2] = %+v, want %+v", rows[2], want) + if rows[6] != want { + t.Fatalf("rows[6] = %+v, want %+v", rows[6], want) } } diff --git a/src/pkg/config/config.go b/src/pkg/config/config.go index a0a981f81d..660e44c5c4 100644 --- a/src/pkg/config/config.go +++ b/src/pkg/config/config.go @@ -238,19 +238,26 @@ type Config struct { func (c *Config) UnmarshalYAML(value *yaml.Node) error { type plain Config node := cloneYAMLNode(value) + var recycle SessionRecycleConfig if agents := mappingValueNode(&node, "agents"); agents != nil && agents.Kind == yaml.MappingNode { - for i := 0; i < len(agents.Content)-1; i += 2 { - if agents.Content[i].Value != "github_api_hourly_cap" { - continue - } - var cap int - if err := agents.Content[i+1].Decode(&cap); err != nil { - return fmt.Errorf("agents.github_api_hourly_cap: %w", err) + for i := 0; i < len(agents.Content)-1; { + switch agents.Content[i].Value { + case "github_api_hourly_cap": + var cap int + if err := agents.Content[i+1].Decode(&cap); err != nil { + return fmt.Errorf("agents.github_api_hourly_cap: %w", err) + } + c.AgentsGitHubAPIHourlyCap = cap + c.AgentsGitHubAPIHourlyCapSet = true + agents.Content = append(agents.Content[:i], agents.Content[i+2:]...) + case "session_recycle": + if err := agents.Content[i+1].Decode(&recycle); err != nil { + return fmt.Errorf("agents.session_recycle: %w", err) + } + agents.Content = append(agents.Content[:i], agents.Content[i+2:]...) + default: + i += 2 } - c.AgentsGitHubAPIHourlyCap = cap - c.AgentsGitHubAPIHourlyCapSet = true - agents.Content = append(agents.Content[:i], agents.Content[i+2:]...) - break } } var decoded plain @@ -261,6 +268,7 @@ func (c *Config) UnmarshalYAML(value *yaml.Node) error { *c = Config(decoded) c.AgentsGitHubAPIHourlyCap = cap c.AgentsGitHubAPIHourlyCapSet = capSet + c.AgentsSessionRecycle = recycle return nil } @@ -6653,13 +6661,30 @@ func (c Config) MarshalYAML() (interface{}, error) { {Kind: yaml.ScalarNode, Tag: "!!int", Value: strconv.Itoa(c.AgentsGitHubAPIHourlyCap)}, }, agents.Content...) } + if !c.AgentsSessionRecycle.IsZero() && agents.Kind == yaml.MappingNode { + var recycleNode yaml.Node + if err := recycleNode.Encode(c.AgentsSessionRecycle); err != nil { + return nil, err + } + agents.Content = append(agents.Content, + &yaml.Node{Kind: yaml.ScalarNode, Value: "session_recycle"}, + &recycleNode) + } return &node, nil } +// EnabledAgents returns the enabled agents with fleet-level defaults that +// the agent manager needs resolved onto each definition: an agent without +// its own session_recycle block carries the agents.session_recycle value +// (#14842), so the manager never has to reach back into Config. func (c *Config) EnabledAgents() map[string]AgentConfig { result := make(map[string]AgentConfig) for name, agent := range c.Agents { if agent.Enabled { + if agent.SessionRecycle == nil && !c.AgentsSessionRecycle.IsZero() { + fleet := c.AgentsSessionRecycle + agent.SessionRecycle = &fleet + } result[name] = agent } } diff --git a/src/pkg/config/crux_aliases.go b/src/pkg/config/crux_aliases.go index a18adfe79d..59b3693e5a 100644 --- a/src/pkg/config/crux_aliases.go +++ b/src/pkg/config/crux_aliases.go @@ -6,6 +6,7 @@ type ( AgentBobConfig = cruxconfig.AgentBobConfig AgentConfig = cruxconfig.AgentConfig AgentRuntimeConfig = cruxconfig.AgentRuntimeConfig + SessionRecycleConfig = cruxconfig.SessionRecycleConfig AgentRuntimeGovernorConfig = cruxconfig.AgentRuntimeGovernorConfig AgentSandboxConfig = cruxconfig.AgentSandboxConfig AgentSandboxOverride = cruxconfig.AgentSandboxOverride diff --git a/src/pkg/config/session_recycle_test.go b/src/pkg/config/session_recycle_test.go new file mode 100644 index 0000000000..a74523b0da --- /dev/null +++ b/src/pkg/config/session_recycle_test.go @@ -0,0 +1,105 @@ +package config + +import ( + "strings" + "testing" + "time" + + cruxconfig "github.com/hivecommons/hive/crux/config" + "gopkg.in/yaml.v3" +) + +func TestSessionRecycleFleetBlockDecodesAndStampsAgents(t *testing.T) { + raw := ` +agents: + session_recycle: + max_rss_mb: 2048 + max_age: 4h + scanner: + enabled: true + quality: + enabled: true + session_recycle: + enabled: false + disabled: + enabled: false +` + var cfg Config + if err := yaml.NewDecoder(strings.NewReader(raw)).Decode(&cfg); err != nil { + t.Fatalf("Decode: %v", err) + } + if cfg.AgentsSessionRecycle.MaxRSSMB != 2048 || cfg.AgentsSessionRecycle.MaxAge != 4*time.Hour { + t.Fatalf("fleet block = %+v", cfg.AgentsSessionRecycle) + } + if _, leaked := cfg.Agents["session_recycle"]; leaked { + t.Fatal("session_recycle leaked into the agents map as an agent") + } + enabled := cfg.EnabledAgents() + if len(enabled) != 2 { + t.Fatalf("EnabledAgents = %d agents, want 2", len(enabled)) + } + scanner := enabled["scanner"].SessionRecycle + if scanner == nil || scanner.MaxRSSMB != 2048 || scanner.MaxAge != 4*time.Hour { + t.Fatalf("scanner did not inherit the fleet block: %+v", scanner) + } + if eff := scanner.Effective(); eff.MinIdle != cruxconfig.DefaultSessionRecycleMinIdle || !scanner.IsEnabled() { + t.Fatalf("scanner effective = %+v", eff) + } + quality := enabled["quality"].SessionRecycle + if quality == nil || quality.IsEnabled() { + t.Fatalf("quality override lost: %+v", quality) + } + // Stamping is a copy, not a shared pointer. + scanner.MaxRSSMB = 1 + if cfg.AgentsSessionRecycle.MaxRSSMB != 2048 { + t.Fatal("stamped pointer aliases the fleet value") + } +} + +func TestSessionRecycleAbsentBlockLeavesAgentsNil(t *testing.T) { + var cfg Config + if err := yaml.NewDecoder(strings.NewReader("agents:\n scanner:\n enabled: true\n")).Decode(&cfg); err != nil { + t.Fatalf("Decode: %v", err) + } + if got := cfg.EnabledAgents()["scanner"].SessionRecycle; got != nil { + t.Fatalf("absent fleet block stamped %+v", got) + } + if !cfg.EnabledAgents()["scanner"].SessionRecycle.IsEnabled() { + t.Fatal("nil policy must read as enabled with defaults") + } +} + +func TestSessionRecycleRoundTripsThroughMarshal(t *testing.T) { + cfg := Config{AgentRuntimeConfig: AgentRuntimeConfig{ + Agents: map[string]AgentConfig{"scanner": {Enabled: true}}, + AgentsSessionRecycle: SessionRecycleConfig{MaxRSSMB: 900, MinIdle: 5 * time.Minute}, + }} + cfg.Project.Org = "org" + out, err := yaml.Marshal(cfg) + if err != nil { + t.Fatalf("Marshal: %v", err) + } + if !strings.Contains(string(out), "session_recycle:") || !strings.Contains(string(out), "max_rss_mb: 900") { + t.Fatalf("fleet block not encoded:\n%s", out) + } + var back Config + if err := yaml.Unmarshal(out, &back); err != nil { + t.Fatalf("Unmarshal: %v", err) + } + if back.AgentsSessionRecycle.MaxRSSMB != 900 || back.AgentsSessionRecycle.MinIdle != 5*time.Minute { + t.Fatalf("round trip = %+v", back.AgentsSessionRecycle) + } + if _, leaked := back.Agents["session_recycle"]; leaked { + t.Fatal("session_recycle decoded as an agent after round trip") + } + + // A zero fleet block encodes nothing under agents. + cfg.AgentsSessionRecycle = SessionRecycleConfig{} + out, err = yaml.Marshal(cfg) + if err != nil { + t.Fatalf("Marshal: %v", err) + } + if strings.Contains(string(out), "session_recycle") { + t.Fatalf("zero fleet block encoded:\n%s", out) + } +} diff --git a/src/pkg/dashboard/server.go b/src/pkg/dashboard/server.go index ef3376a527..50bcb73d56 100644 --- a/src/pkg/dashboard/server.go +++ b/src/pkg/dashboard/server.go @@ -833,12 +833,19 @@ type FrontendAgent struct { // from it (sandbox_health.go). Present only for sandboxed agents; the // SPA reads health from here instead of State, which is "idle" between // kicks for an agent that has no session to be down. - SandboxRun *FrontendSandboxRun `json:"sandboxRun,omitempty"` - LastError string `json:"lastError,omitempty"` - StallNudges int `json:"stallNudges,omitempty"` - StallRecoveries int `json:"stallRecoveries,omitempty"` - ActionNudges int `json:"actionNudges,omitempty"` - TransientNudges int `json:"transientNudges,omitempty"` + SandboxRun *FrontendSandboxRun `json:"sandboxRun,omitempty"` + LastError string `json:"lastError,omitempty"` + StallNudges int `json:"stallNudges,omitempty"` + StallRecoveries int `json:"stallRecoveries,omitempty"` + ActionNudges int `json:"actionNudges,omitempty"` + TransientNudges int `json:"transientNudges,omitempty"` + // SessionRssMb is the resident memory of the agent's CLI process tree + // (MiB) and SessionAge how long the current CLI session has been up, + // both sampled on the health tick (#14842). SessionRecycles counts + // RSS/age-budget recycles since boot. + SessionRssMb int `json:"sessionRssMb,omitempty"` + SessionAge string `json:"sessionAge,omitempty"` + SessionRecycles int `json:"sessionRecycles,omitempty"` // LastAction is the last issue or PR the agent acted on through the hive // (#10925); see FrontendAgentLastAction. LastAction *FrontendAgentLastAction `json:"lastAction,omitempty"` diff --git a/src/pkg/dashboard/settings_parity_allowlist.txt b/src/pkg/dashboard/settings_parity_allowlist.txt index 5910aa3917..b8da760dd5 100644 --- a/src/pkg/dashboard/settings_parity_allowlist.txt +++ b/src/pkg/dashboard/settings_parity_allowlist.txt @@ -115,6 +115,10 @@ agents..sandbox.job.workspace_claim_mount # TODO #11854 agents..sandbox.network_mode # TODO #11854 agents..sandbox.runtime # TODO #11854 agents..sandbox.timeout_s # TODO #11854 +agents..session_recycle.enabled # TODO #11854 +agents..session_recycle.max_age # TODO #11854 +agents..session_recycle.max_rss_mb # TODO #11854 +agents..session_recycle.min_idle # TODO #11854 agents..skills[] # TODO #11854 agents..sort_order # TODO #11854 agents..stale_timeout # TODO #11854 @@ -140,6 +144,10 @@ agents..variables..type # TODO #11854 agents..variables..url # TODO #11854 agents..variables..value # TODO #11854 agents.github_api_hourly_cap # TODO #11854 +agents.session_recycle.enabled # TODO #11854 +agents.session_recycle.max_age # TODO #11854 +agents.session_recycle.max_rss_mb # TODO #11854 +agents.session_recycle.min_idle # TODO #11854 auto_merge.credit_issue_authors # merge-time Co-authored-by credit (#14581); yaml-only auto_merge.allow_unprotected_base[] # TODO #11854 auto_merge.human_merge_paths. # TODO #11854 diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html index 6ab85b153f..e9d25f520d 100644 --- a/src/pkg/dashboard/static/index.html +++ b/src/pkg/dashboard/static/index.html @@ -10616,6 +10616,7 @@

Where to get help

Next Kick
${isOnDemand ? 'on demand' : isPaused ? 'paused' : isOff ? esc(agentCadenceOffLabel()) : esc(agentKickDisplay(a.nextKick, a.nextKickAt))}
Tokens (24h)
${_tokTotal > 0 ? fmtTokens(_tokTotal) : (_tok.sessions > 0 ? _tok.sessions + ' sess' : '—')}
Restarts
${a.restarts ?? 0}24h${(a.restarts || 0) > 0 ? ` ` : ''}${restartSparkSvg(a.name)}
+ ${a.sessionRssMb ? `
Session RSS
${a.sessionRssMb} MiB${a.sessionAge ? ` ${esc(a.sessionAge)}` : ''}${a.sessionRecycles ? ` ↻${a.sessionRecycles}` : ''}
` : ''}
Avg/Pass
${_tokAvg > 0 ? fmtTokens(_tokAvg) : '—'}
Sessions
${_tok.sessions ?? '—'}
Messages
${_tok.messages ?? '—'}
diff --git a/src/pkg/dashboard/status_builder.go b/src/pkg/dashboard/status_builder.go index 2767eb61da..d23422cc5b 100644 --- a/src/pkg/dashboard/status_builder.go +++ b/src/pkg/dashboard/status_builder.go @@ -1208,6 +1208,9 @@ func buildAgentsWithHidden(statuses map[string]*crux.Process, cfg *config.Config LoginURL: proc.LoginURL, StallNudges: proc.StallNudges, StallRecoveries: proc.StallRecoveries, + SessionRssMb: proc.SessionRSSMB, + SessionAge: formatSessionAge(proc.StartedAt, time.Now()), + SessionRecycles: proc.SessionRecycles, ActionNudges: proc.ActionNudges, // #4697: transient-API-error retry nudges, surfaced beside the // other two nudge counters. @@ -3421,3 +3424,19 @@ func apiBudgetFromRateLimits(rateLimits map[string]any) map[string]any { } return map[string]any{"mode": "normal", "skipped_steps": []string{}} } + +// formatSessionAge renders how long the agent's current CLI session has +// been up (#14842); "" when the launch time is unknown. +func formatSessionAge(startedAt *time.Time, now time.Time) string { + if startedAt == nil || startedAt.IsZero() { + return "" + } + age := now.Sub(*startedAt) + if age < 0 { + age = 0 + } + if age < time.Minute { + return age.Round(time.Second).String() + } + return age.Round(time.Minute).String() +} diff --git a/src/pkg/dashboard/status_builder_session_age_test.go b/src/pkg/dashboard/status_builder_session_age_test.go new file mode 100644 index 0000000000..5567771f63 --- /dev/null +++ b/src/pkg/dashboard/status_builder_session_age_test.go @@ -0,0 +1,29 @@ +package dashboard + +import ( + "testing" + "time" +) + +func TestFormatSessionAge(t *testing.T) { + now := time.Date(2026, 10, 23, 3, 0, 0, 0, time.UTC) + if got := formatSessionAge(nil, now); got != "" { + t.Fatalf("nil = %q", got) + } + zero := time.Time{} + if got := formatSessionAge(&zero, now); got != "" { + t.Fatalf("zero = %q", got) + } + recent := now.Add(-42 * time.Second) + if got := formatSessionAge(&recent, now); got != "42s" { + t.Fatalf("seconds = %q", got) + } + old := now.Add(-3*time.Hour - 7*time.Minute - 20*time.Second) + if got := formatSessionAge(&old, now); got != "3h7m0s" { + t.Fatalf("hours = %q", got) + } + future := now.Add(time.Minute) + if got := formatSessionAge(&future, now); got != "0s" { + t.Fatalf("future = %q", got) + } +}