From 6ff633a282774446fbc48b6aa2b601f825d337d5 Mon Sep 17 00:00:00 2001 From: Andrew Anderson Date: Wed, 28 Oct 2026 00:59:38 -0400 Subject: [PATCH 1/2] dashboard: derive sandboxed agent health from the last run, not the missing session MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Sandboxed agents (Podman or Job runtime) sit in the idle state between kicks and have no tmux session, so the SPA painted them red with "Stopped — see the agent's log" / "session down" and the token collector never saw their usage. Persist a SandboxRun record per kick (outcome, summary, usage) under /sandbox-runs/, restore it on restart, expose it as sandboxRun on /api/status with an ok/running/failed/overdue verdict derived from the run and the cadence, make the SPA use that instead of the session check, and ingest the run history in the token collector. Fixes #14844 Signed-off-by: Andrew Anderson --- .../fixed-14844-sandbox-agent-health.md | 1 + src/cmd/hive/main.go | 1 + src/crux/agent/manager.go | 18 +- src/crux/agent/manager_pane_signals.go | 1 + src/crux/agent/manager_sandbox.go | 22 +- src/crux/agent/sandbox_run.go | 234 ++++++++++++++++++ src/crux/agent/sandbox_run_test.go | 128 ++++++++++ src/crux/process.go | 1 + src/crux/process_types.go | 43 ++++ src/crux/sandbox_run.go | 37 +++ src/docs/api-reference.md | 32 +-- src/docs/sandbox-isolation.md | 19 ++ .../agent_activity_section_10935_test.go | 5 + .../dashboard/disabled_agent_enable_test.go | 5 + src/pkg/dashboard/sandbox_health.go | 149 +++++++++++ src/pkg/dashboard/sandbox_health_test.go | 110 ++++++++ src/pkg/dashboard/server.go | 7 +- src/pkg/dashboard/spoke_health_truth_test.go | 5 +- src/pkg/dashboard/static/index.html | 67 ++++- src/pkg/dashboard/status_builder.go | 11 + src/pkg/tokens/collector.go | 19 ++ src/pkg/tokens/sandbox_scanner.go | 127 ++++++++++ src/pkg/tokens/sandbox_scanner_test.go | 79 ++++++ 23 files changed, 1091 insertions(+), 30 deletions(-) create mode 100644 src/changelog.d/fixed-14844-sandbox-agent-health.md create mode 100644 src/crux/agent/sandbox_run.go create mode 100644 src/crux/agent/sandbox_run_test.go create mode 100644 src/crux/sandbox_run.go create mode 100644 src/pkg/dashboard/sandbox_health.go create mode 100644 src/pkg/dashboard/sandbox_health_test.go create mode 100644 src/pkg/tokens/sandbox_scanner.go create mode 100644 src/pkg/tokens/sandbox_scanner_test.go diff --git a/src/changelog.d/fixed-14844-sandbox-agent-health.md b/src/changelog.d/fixed-14844-sandbox-agent-health.md new file mode 100644 index 0000000000..2ec7f733a8 --- /dev/null +++ b/src/changelog.d/fixed-14844-sandbox-agent-health.md @@ -0,0 +1 @@ +- Sandboxed agents (`sandbox.runtime: job` or Podman) no longer render as "Stopped — see the agent's log" / "session down" with a red dot and empty TOKENS/SESSIONS between kicks. Their health now derives from the last run: green when it succeeded and the next cadence occurrence is still ahead, amber while a kick is in flight, red only when the last run failed/timed out or a scheduled occurrence was missed past a grace. Each run's outcome, summary and token usage are persisted under `/sandbox-runs/`, restored on restart, and ingested by the token collector so tokens, sessions, cost and the activity summary reflect sandboxed work. diff --git a/src/cmd/hive/main.go b/src/cmd/hive/main.go index d858649305..bdf302cff3 100644 --- a/src/cmd/hive/main.go +++ b/src/cmd/hive/main.go @@ -3020,6 +3020,7 @@ func (b *boot) bootCollectorsWith(deps bootCollectorsDeps) { b.tokenCollector.SetClaudeSessionsDir(b.cfg.Data.ClaudeSessionsDir) b.tokenCollector.SetCopilotSessionsDir(b.cfg.Data.CopilotSessionsDir) b.tokenCollector.SetBobSessionsDir(b.cfg.Data.BobSessionsDir) + b.tokenCollector.SetSandboxRunHistoryPath(crux.SandboxRunHistoryPath(crux.WorkDir())) tokenStop := make(chan struct{}) deps.startTokenCollector(b.tokenCollector, tokenStop) b.cleanup.push(func() { close(tokenStop) }) diff --git a/src/crux/agent/manager.go b/src/crux/agent/manager.go index 22d63ae6e4..a17c323489 100644 --- a/src/crux/agent/manager.go +++ b/src/crux/agent/manager.go @@ -441,6 +441,12 @@ type AgentProcess struct { // report, or plain "ended". Settled(LastKick) is false while the current // turn is still running. Guarded by m.mu; see kick_outcome.go. KickOutcome KickOutcome + // SandboxLastRun is the outcome of this agent's most recent sandboxed + // kick (podman or Job runtime). Sandboxed agents sit in StateIdle between + // kicks with no session to probe, so health and the activity summary are + // read from here. Restored from the sandbox run file on Start. Guarded by + // m.mu; see manager_sandbox.go. + SandboxLastRun *crux.SandboxRun // sandboxResumeAfterCancel is set when an operator resumes a paused // sandbox agent while the canceled sandbox goroutine is still draining. // The completion handler then turns the expected cancellation into Idle @@ -809,10 +815,7 @@ func NewManager(agents map[string]config.AgentConfig, logger *slog.Logger, proje } func NewManagerWithOptions(agents map[string]config.AgentConfig, logger *slog.Logger, project ProjectContext, options ...ManagerOption) *Manager { - workDir := os.Getenv("HIVE_WORK_DIR") - if workDir == "" { - workDir = "/data/agents" - } + workDir := crux.WorkDir() var opts managerOptions for _, option := range options { if option != nil { @@ -997,6 +1000,13 @@ func (m *Manager) Start(ctx context.Context, name string) error { agent.HasLaunched = true agent.LaunchedMode = m.agentMode(agent) agent.startupLaunchQueued = false + if agent.SandboxLastRun == nil { + if run, err := loadSandboxRun(m.workDir, name); err != nil { + m.logger.Warn("sandbox last run not restored", "name", name, "error", err) + } else if run != nil { + agent.SandboxLastRun = run + } + } m.logger.Info("audit: sandbox agent ready", "name", name) m.mu.Unlock() return nil diff --git a/src/crux/agent/manager_pane_signals.go b/src/crux/agent/manager_pane_signals.go index 9ae4ebcc6b..0692f87fce 100644 --- a/src/crux/agent/manager_pane_signals.go +++ b/src/crux/agent/manager_pane_signals.go @@ -508,6 +508,7 @@ func (a *AgentProcess) snapshot() crux.Process { BootstrapOverride: a.BootstrapOverride, LastError: a.LastError, KickOutcome: a.KickOutcome, + SandboxLastRun: cloneSandboxRun(a.SandboxLastRun), NeedsLogin: needsLogin, LoginURL: loginURL, Starting: a.startupLaunchQueued || a.launching, diff --git a/src/crux/agent/manager_sandbox.go b/src/crux/agent/manager_sandbox.go index 0349763a0d..c85f261555 100644 --- a/src/crux/agent/manager_sandbox.go +++ b/src/crux/agent/manager_sandbox.go @@ -161,7 +161,7 @@ func (m *Manager) startSandboxKickLocked(agent *AgentProcess, message string) er prClient = m.sandboxPRClient pushEnabled = true } - go m.runSandboxKick(runCtx, agent.Name, spec, launcher, runner, cloneMinter, pushMinter, pushEnabled, prClient, mutationBoundary) + go m.runSandboxKick(runCtx, agent.Name, runtime, spec, launcher, runner, cloneMinter, pushMinter, pushEnabled, prClient, mutationBoundary) return nil } @@ -189,7 +189,8 @@ func (m *Manager) tieredSandboxMinterLocked(tier string) pushbroker.TokenMinter } } -func (m *Manager) runSandboxKick(ctx context.Context, name string, spec SandboxKickSpec, launcher sandbox.Launcher, runner sandboxCommandRunner, cloneMinter, minter pushbroker.TokenMinter, pushEnabled bool, prClient PRCreator, mutationBoundary effects.Boundary) { +func (m *Manager) runSandboxKick(ctx context.Context, name, runtime string, spec SandboxKickSpec, launcher sandbox.Launcher, runner sandboxCommandRunner, cloneMinter, minter pushbroker.TokenMinter, pushEnabled bool, prClient PRCreator, mutationBoundary effects.Boundary) { + started := time.Now() exec := &SandboxExecutor{ Launcher: launcher, Runner: runner, @@ -201,6 +202,7 @@ func (m *Manager) runSandboxKick(ctx context.Context, name string, spec SandboxK Logger: m.logger, } res, err := exec.Run(ctx, spec) + ended := time.Now() m.mu.Lock() defer m.mu.Unlock() agent, ok := m.agents[name] @@ -208,6 +210,22 @@ func (m *Manager) runSandboxKick(ctx context.Context, name string, spec SandboxK return } agent.cancel = nil + // The run record is what a sandboxed agent's health and activity derive + // from: there is no session to probe between kicks. Recorded on every + // outcome but an operator cancel, and persisted so a pod roll does not + // turn yesterday's successful run into "never ran". + if err == nil || !agent.sandboxResumeAfterCancel { + run := buildSandboxRun(runtime, started, ended, res, err) + agent.SandboxLastRun = &run + agent.KickOutcome = KickOutcome{Kind: KickOutcomeEnded, At: ended, KickAt: started} + if agent.LastKick != nil { + agent.KickOutcome.KickAt = *agent.LastKick + } + entry := SandboxRunHistoryEntry{Agent: name, Backend: effectiveBackend(agent), Model: agent.Config.Model, Run: run} + if perr := persistSandboxRun(m.workDir, entry); perr != nil { + m.logger.Warn("sandbox run record not persisted", "name", name, "error", perr) + } + } if err != nil { if agent.sandboxResumeAfterCancel { agent.sandboxResumeAfterCancel = false diff --git a/src/crux/agent/sandbox_run.go b/src/crux/agent/sandbox_run.go new file mode 100644 index 0000000000..b7426a7af9 --- /dev/null +++ b/src/crux/agent/sandbox_run.go @@ -0,0 +1,234 @@ +package agent + +import ( + "encoding/json" + "errors" + "os" + "path/filepath" + "regexp" + "strconv" + "strings" + "time" + + "github.com/hivecommons/hive/crux" +) + +const ( + // sandboxRunSummaryMax bounds SandboxRun.Summary so a run record, which + // is persisted and shipped in every status snapshot, never carries a + // whole transcript. + sandboxRunSummaryMax = 2000 + // sandboxRunSummaryTailLines is how much of the transcript tail stands in + // for the summary when the agent left no report. + sandboxRunSummaryTailLines = 20 +) + +func sandboxRunPath(workDir, agent string) string { + return filepath.Join(crux.SandboxRunsDir(workDir), sanitizeBranchPart(agent)+".json") +} + +func cloneSandboxRun(r *crux.SandboxRun) *crux.SandboxRun { + if r == nil { + return nil + } + cp := *r + return &cp +} + +// buildSandboxRun turns an executor result into the run record the dashboard +// and the token collector read. +func buildSandboxRun(runtime string, started, ended time.Time, res SandboxKickResult, runErr error) crux.SandboxRun { + run := crux.SandboxRun{ + Runtime: runtime, + StartedAt: started, + EndedAt: ended, + Succeeded: runErr == nil, + TimedOut: res.TimedOut, + CommitCount: res.CommitCount, + Branch: res.Branch, + Usage: parseSandboxUsage(res.Sandbox.Stdout), + } + if runErr != nil { + run.Error = truncateRunes(runErr.Error(), sandboxRunSummaryMax) + } + if res.PR != nil { + run.PRURL = res.PR.URL + } + if res.Report != nil && strings.TrimSpace(res.Report.Summary) != "" { + run.Summary = truncateRunes(strings.TrimSpace(res.Report.Summary), sandboxRunSummaryMax) + } else { + run.Summary = sandboxTranscriptTail(res.Sandbox.Stdout, res.Sandbox.Stderr) + } + return run +} + +// sandboxTranscriptTail is the last non-empty lines of the transcript, stdout +// first and stderr only when stdout had nothing to say. +func sandboxTranscriptTail(stdout, stderr string) string { + src := stdout + if strings.TrimSpace(src) == "" { + src = stderr + } + lines := strings.Split(strings.ReplaceAll(src, "\r\n", "\n"), "\n") + kept := make([]string, 0, sandboxRunSummaryTailLines) + for i := len(lines) - 1; i >= 0 && len(kept) < sandboxRunSummaryTailLines; i-- { + if strings.TrimSpace(lines[i]) == "" { + continue + } + kept = append([]string{strings.TrimRight(lines[i], " \t")}, kept...) + } + return truncateRunes(strings.Join(kept, "\n"), sandboxRunSummaryMax) +} + +var ( + // copilotUsageLineRe matches one row of the Copilot CLI's non-interactive + // footer, e.g. " claude-sonnet-4.5 12.3k in, 456 out, 1.2k cached". + copilotUsageLineRe = regexp.MustCompile(`^\s*(\S+)\s+([\d.,]+[kKmM]?)\s+in,\s*([\d.,]+[kKmM]?)\s+out(?:,\s*([\d.,]+[kKmM]?)\s+cached)?`) + copilotUsageHeader = "Breakdown by AI model" +) + +// parseSandboxUsage recovers token usage from a sandbox transcript. It +// understands the Copilot CLI's "Breakdown by AI model" footer and the Claude +// Code JSON / stream-json result record ({"type":"result","usage":{...}}). +// A transcript with neither yields a zero usage, never an estimate. +func parseSandboxUsage(stdout string) crux.SandboxRunUsage { + if u, ok := parseClaudeJSONUsage(stdout); ok { + return u + } + if u, ok := parseCopilotFooterUsage(stdout); ok { + return u + } + return crux.SandboxRunUsage{} +} + +func parseClaudeJSONUsage(stdout string) (crux.SandboxRunUsage, bool) { + type usage struct { + Input int64 `json:"input_tokens"` + Output int64 `json:"output_tokens"` + CacheRead int64 `json:"cache_read_input_tokens"` + CacheCreate int64 `json:"cache_creation_input_tokens"` + } + type result struct { + Type string `json:"type"` + Usage *usage `json:"usage"` + ModelUsage map[string]json.RawMessage `json:"modelUsage"` + } + var found crux.SandboxRunUsage + ok := false + for _, line := range strings.Split(stdout, "\n") { + line = strings.TrimSpace(line) + if !strings.HasPrefix(line, "{") || !strings.Contains(line, `"usage"`) { + continue + } + var r result + if json.Unmarshal([]byte(line), &r) != nil || r.Type != "result" || r.Usage == nil { + continue + } + found = crux.SandboxRunUsage{Input: r.Usage.Input, Output: r.Usage.Output, CacheRead: r.Usage.CacheRead, CacheCreate: r.Usage.CacheCreate} + for model := range r.ModelUsage { + found.Model = model + break + } + ok = found.Total() > 0 + } + return found, ok +} + +func parseCopilotFooterUsage(stdout string) (crux.SandboxRunUsage, bool) { + idx := strings.LastIndex(stdout, copilotUsageHeader) + if idx < 0 { + return crux.SandboxRunUsage{}, false + } + var total crux.SandboxRunUsage + for _, line := range strings.Split(stdout[idx+len(copilotUsageHeader):], "\n") { + m := copilotUsageLineRe.FindStringSubmatch(line) + if m == nil { + continue + } + if total.Model == "" { + total.Model = m[1] + } + total.Input += parseHumanCount(m[2]) + total.Output += parseHumanCount(m[3]) + total.CacheRead += parseHumanCount(m[4]) + } + return total, total.Total() > 0 +} + +// parseHumanCount reads "12.3k", "1,204" or "2M" as an integer count. +func parseHumanCount(s string) int64 { + s = strings.ReplaceAll(strings.TrimSpace(s), ",", "") + if s == "" { + return 0 + } + mult := 1.0 + switch s[len(s)-1] { + case 'k', 'K': + mult, s = 1000, s[:len(s)-1] + case 'm', 'M': + mult, s = 1000000, s[:len(s)-1] + } + f, err := strconv.ParseFloat(s, 64) + if err != nil { + return 0 + } + return int64(f*mult + 0.5) +} + +// SandboxRunHistoryEntry is one line of runs.jsonl: the run plus the agent +// it belongs to and the backend/model it ran on, which the token collector +// needs and the run record itself does not carry. +type SandboxRunHistoryEntry struct { + Agent string `json:"agent"` + Backend string `json:"backend,omitempty"` + Model string `json:"model,omitempty"` + Run crux.SandboxRun `json:"run"` +} + +// persistSandboxRun writes the agent's last-run file and appends the run to +// the history. Failures are returned for logging only; the in-memory record +// is already set. +func persistSandboxRun(workDir string, entry SandboxRunHistoryEntry) error { + dir := crux.SandboxRunsDir(workDir) + if err := os.MkdirAll(dir, 0o770); err != nil { + return err + } + last, err := json.Marshal(entry) + if err != nil { + return err + } + path := sandboxRunPath(workDir, entry.Agent) + tmp := path + ".tmp" + if err := os.WriteFile(tmp, last, 0o660); err != nil { + return err + } + if err := os.Rename(tmp, path); err != nil { + return err + } + f, err := os.OpenFile(crux.SandboxRunHistoryPath(workDir), os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o660) + if err != nil { + return err + } + defer func() { _ = f.Close() }() + _, err = f.Write(append(last, '\n')) + return err +} + +// loadSandboxRun reads the agent's persisted last run; (nil, nil) when none. +func loadSandboxRun(workDir, agent string) (*crux.SandboxRun, error) { + data, err := os.ReadFile(sandboxRunPath(workDir, agent)) + if errors.Is(err, os.ErrNotExist) { + return nil, nil + } + if err != nil { + return nil, err + } + var entry SandboxRunHistoryEntry + if err := json.Unmarshal(data, &entry); err != nil { + return nil, err + } + if entry.Run.StartedAt.IsZero() { + return nil, nil + } + return &entry.Run, nil +} diff --git a/src/crux/agent/sandbox_run_test.go b/src/crux/agent/sandbox_run_test.go new file mode 100644 index 0000000000..b47e1f812b --- /dev/null +++ b/src/crux/agent/sandbox_run_test.go @@ -0,0 +1,128 @@ +package agent + +import ( + "errors" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/crux/outputschema" + "github.com/hivecommons/hive/crux/sandbox" +) + +func TestParseSandboxUsage(t *testing.T) { + copilot := strings.Join([]string{ + "doing work...", + "Total usage est: 1 Premium request", + "Breakdown by AI model:", + " claude-sonnet-4.5 12.3k in, 456 out, 1.2k cached", + " gpt-5 1,204 in, 10 out", + }, "\n") + claude := `{"type":"system","subtype":"init"} +{"type":"result","subtype":"success","usage":{"input_tokens":100,"output_tokens":20,"cache_read_input_tokens":300,"cache_creation_input_tokens":5},"modelUsage":{"claude-opus-4":{}}} +` + cases := []struct { + name string + stdout string + want crux.SandboxRunUsage + }{ + {"copilot footer", copilot, crux.SandboxRunUsage{Model: "claude-sonnet-4.5", Input: 13504, Output: 466, CacheRead: 1200}}, + {"claude json result", claude, crux.SandboxRunUsage{Model: "claude-opus-4", Input: 100, Output: 20, CacheRead: 300, CacheCreate: 5}}, + {"plain transcript", "Opened PR #1402\nDone.\n", crux.SandboxRunUsage{}}, + {"empty", "", crux.SandboxRunUsage{}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := parseSandboxUsage(tc.stdout); got != tc.want { + t.Fatalf("got %+v, want %+v", got, tc.want) + } + }) + } +} + +func TestParseHumanCount(t *testing.T) { + for in, want := range map[string]int64{"12.3k": 12300, "1,204": 1204, "2M": 2000000, "7": 7, "junk": 0} { + if got := parseHumanCount(in); got != want { + t.Errorf("parseHumanCount(%q) = %d, want %d", in, got, want) + } + } +} + +func TestBuildSandboxRun(t *testing.T) { + started := time.Date(2026, 10, 21, 10, 0, 0, 0, time.UTC) + ended := started.Add(29 * time.Minute) + res := SandboxKickResult{ + Branch: "hive/deps-1793130444", + CommitCount: 2, + PR: &crux.PRResult{URL: "https://github.com/torch-spyre/spyre-inference/pull/1402"}, + Report: &outputschema.AgentReport{Summary: " bumped torch to 2.9 "}, + Sandbox: sandbox.Result{Stdout: "line1\nline2\n"}, + } + run := buildSandboxRun("job", started, ended, res, nil) + if !run.Succeeded || run.Runtime != "job" || run.Duration() != 29*time.Minute || run.PRURL != res.PR.URL || run.CommitCount != 2 || run.Branch != res.Branch { + t.Fatalf("run = %+v", run) + } + if run.Summary != "bumped torch to 2.9" { + t.Errorf("summary must prefer the agent report, got %q", run.Summary) + } + + // No report: the transcript tail is the summary, stderr only as fallback. + res.Report = nil + res.Sandbox.Stdout = strings.Repeat("x\n", 30) + "last line\n" + run = buildSandboxRun("podman", started, ended, res, nil) + if !strings.HasSuffix(run.Summary, "last line") || strings.Count(run.Summary, "\n") >= sandboxRunSummaryTailLines { + t.Errorf("summary must be the bounded stdout tail, got %q", run.Summary) + } + res.Sandbox.Stdout = "" + res.Sandbox.Stderr = "boom" + run = buildSandboxRun("podman", started, ended, res, errors.New("sandbox exited 1")) + if run.Succeeded || run.Error != "sandbox exited 1" || run.Summary != "boom" { + t.Errorf("failed run = %+v", run) + } + + // Long summaries are bounded. + res.Report = &outputschema.AgentReport{Summary: strings.Repeat("s", sandboxRunSummaryMax*2)} + run = buildSandboxRun("job", started, ended, res, nil) + if len([]rune(run.Summary)) > sandboxRunSummaryMax { + t.Errorf("summary not truncated: %d runes", len([]rune(run.Summary))) + } +} + +func TestPersistAndLoadSandboxRun(t *testing.T) { + workDir := t.TempDir() + if got, err := loadSandboxRun(workDir, "deps"); err != nil || got != nil { + t.Fatalf("missing record must be (nil, nil), got %+v, %v", got, err) + } + started := time.Date(2026, 10, 21, 10, 0, 0, 0, time.UTC) + first := SandboxRunHistoryEntry{Agent: "deps", Backend: "claude", Model: "opus", Run: crux.SandboxRun{Runtime: "job", StartedAt: started, EndedAt: started.Add(time.Minute), Error: "exit 1"}} + second := first + second.Run = crux.SandboxRun{Runtime: "job", StartedAt: started.Add(6 * time.Hour), EndedAt: started.Add(6*time.Hour + 29*time.Minute), Succeeded: true, Summary: "ok", Usage: crux.SandboxRunUsage{Model: "opus", Input: 10, Output: 2}} + for _, e := range []SandboxRunHistoryEntry{first, second} { + if err := persistSandboxRun(workDir, e); err != nil { + t.Fatal(err) + } + } + got, err := loadSandboxRun(workDir, "deps") + if err != nil || got == nil { + t.Fatalf("load: %+v, %v", got, err) + } + if !got.Succeeded || !got.StartedAt.Equal(second.Run.StartedAt) || got.Usage != second.Run.Usage || got.Summary != "ok" { + t.Errorf("last run must be the latest persisted, got %+v", got) + } + if _, err := os.Stat(filepath.Join(workDir, "sandbox-runs", "deps.json.tmp")); !os.IsNotExist(err) { + t.Errorf("temp file must be renamed away, stat err = %v", err) + } + hist, err := os.ReadFile(crux.SandboxRunHistoryPath(workDir)) + if err != nil { + t.Fatal(err) + } + if lines := strings.Split(strings.TrimSpace(string(hist)), "\n"); len(lines) != 2 { + t.Errorf("history must hold one line per run, got %d: %q", len(lines), hist) + } + if got, err := loadSandboxRun(workDir, "quality"); err != nil || got != nil { + t.Errorf("other agent must have no record, got %+v, %v", got, err) + } +} diff --git a/src/crux/process.go b/src/crux/process.go index b1ea412bba..0f31958a1c 100644 --- a/src/crux/process.go +++ b/src/crux/process.go @@ -70,6 +70,7 @@ type Process struct { TurnLoss TurnLoss ActionNudges int KickOutcome KickOutcome + SandboxLastRun *SandboxRun StartFailureClass string StartFailureReason string StartFailureCount int diff --git a/src/crux/process_types.go b/src/crux/process_types.go index 63695a78d3..b169df3f56 100644 --- a/src/crux/process_types.go +++ b/src/crux/process_types.go @@ -119,3 +119,46 @@ type TurnLoss struct { // Recent is the newest records, oldest first; the runtime bounds the ring. Recent []TurnInterruption } + +// SandboxRunUsage is the token usage one sandboxed kick reported, when its +// transcript carried any. Zero when the backend printed no usage footer. +type SandboxRunUsage struct { + Model string `json:"model,omitempty"` + Input int64 `json:"input,omitempty"` + Output int64 `json:"output,omitempty"` + CacheRead int64 `json:"cacheRead,omitempty"` + CacheCreate int64 `json:"cacheCreate,omitempty"` +} + +// Total is the sum of the four counters. +func (u SandboxRunUsage) Total() int64 { + return u.Input + u.Output + u.CacheRead + u.CacheCreate +} + +// SandboxRun is the outcome of the most recent sandboxed kick of an agent +// (podman or Kubernetes Job runtime). A sandboxed agent has no tmux session +// or in-pod process between kicks, so this record — not the process state — +// is what its health derives from. +type SandboxRun struct { + Runtime string `json:"runtime,omitempty"` + StartedAt time.Time `json:"startedAt"` + EndedAt time.Time `json:"endedAt"` + Succeeded bool `json:"succeeded"` + TimedOut bool `json:"timedOut,omitempty"` + Error string `json:"error,omitempty"` + CommitCount int `json:"commitCount,omitempty"` + Branch string `json:"branch,omitempty"` + PRURL string `json:"prURL,omitempty"` + // Summary is the agent's own report summary when it left one, else the + // scrubbed tail of the transcript. Bounded; see sandboxRunSummaryMax. + Summary string `json:"summary,omitempty"` + Usage SandboxRunUsage `json:"usage,omitempty"` +} + +// Duration is how long the run took, zero when the record is incomplete. +func (r SandboxRun) Duration() time.Duration { + if r.StartedAt.IsZero() || r.EndedAt.Before(r.StartedAt) { + return 0 + } + return r.EndedAt.Sub(r.StartedAt) +} diff --git a/src/crux/sandbox_run.go b/src/crux/sandbox_run.go new file mode 100644 index 0000000000..b34e02294a --- /dev/null +++ b/src/crux/sandbox_run.go @@ -0,0 +1,37 @@ +package crux + +import ( + "os" + "path/filepath" +) + +const ( + // DefaultWorkDir is the agent manager's work dir when HIVE_WORK_DIR is + // unset. + DefaultWorkDir = "/data/agents" + // sandboxRunsDirName holds one .json per sandboxed agent (its last + // run) and runs.jsonl, the append-only history. + sandboxRunsDirName = "sandbox-runs" + sandboxRunHistoryName = "runs.jsonl" +) + +// WorkDir is the agent manager's work dir: $HIVE_WORK_DIR or DefaultWorkDir. +// The host and the runtime (in-process or sidecar) share it through /data. +func WorkDir() string { + if d := os.Getenv("HIVE_WORK_DIR"); d != "" { + return d + } + return DefaultWorkDir +} + +// SandboxRunsDir is where sandbox run records live under workDir. +func SandboxRunsDir(workDir string) string { + return filepath.Join(workDir, sandboxRunsDirName) +} + +// SandboxRunHistoryPath is the append-only sandbox run history under +// workDir, one SandboxRun history entry per line. The token collector reads +// it to attribute sandbox usage per agent. +func SandboxRunHistoryPath(workDir string) string { + return filepath.Join(SandboxRunsDir(workDir), sandboxRunHistoryName) +} diff --git a/src/docs/api-reference.md b/src/docs/api-reference.md index ce798b8162..dd06ec1418 100644 --- a/src/docs/api-reference.md +++ b/src/docs/api-reference.md @@ -13,17 +13,17 @@ History point limits, timestamp cutoffs, incremental cost reads and ETag/304 beh | `GET` | `/api/version` | Dashboard auth/session | Build/version metadata; includes `upgradeMarker` (`target`, `current`, `attempts`, `maxAttempts`, `failed`, `requestedAt`, `lastError`) while a self-upgrade is in flight or has failed ([#6765](https://github.com/hivecommons/hive/issues/6765)), the hub-delivered `upgradePolicy` (`hub_managed`, `spoke_managed`, `schedule`, `schedule_hour`, `schedule_timezone`, `schedule_weekday`, `paused`, `branch`, `channel`, `target_sha`, `target_resolved`, `armed_target`, and `next_update_at` — the hub's RFC3339 ETA for the next promotion into the `stable` channel, omitted when unknown ([#10256](https://github.com/hivecommons/hive/issues/10256))) when available, and an `autoUpdate` object (`enabled`, `state` — one of `disabled`/`up_to_date`/`behind`/`retrying`/`failed`/`unknown` — `healthy`, `period`, `targetBranch`, `targetCommit`, `currentCommit`, `commitsBehind`, `lastAttemptAt`, `lastError`, `detail`, and `nextUpdateAt` — the hub's RFC3339 ETA for the next update, omitted when unknown, disabled or paused ([#10257](https://github.com/hivecommons/hive/issues/10257))) that never reports a failed or unknown update as healthy ([#6962](https://github.com/hivecommons/hive/issues/6962), [#6963](https://github.com/hivecommons/hive/issues/6963)) | `pkg/dashboard/api.go:42` | | `GET` | `/api/version/release-notes` | Dashboard auth/session | Changelog gap between two revisions: `?from=&to=` returns `{from:{sha,version?}, to:{sha,ref,version?}, sections:[{version,date,categories:{added,changed,fixed,security,deprecated}}], unreleased?, truncated, source, error?}` — the `CHANGELOG.md` release sections present at `to` and absent at `from`, newest first, plus (when `to` is not a release tag) the `changelog.d/` fragments new at `to` under an `Unreleased (in this build)` section. Fetch failures return `source: "unavailable"` with `error` rather than an error status; results are cached per (from, to) for an hour ([#11065](https://github.com/hivecommons/hive/issues/11065)) | `pkg/dashboard/api.go:43` | | `POST` | `/api/release-channel` | Owner only | Hosted spoke self-service release-channel selector; relays `stable`/`candidate`/`edge` to the hub's existing switch-branch endpoint with the spoke dashboard-token proof, and reports the requested channel as pending until the Deployment image lands | `pkg/dashboard/api.go:64` | -| `GET` | `/api/health` | Public | Basic health probe | `pkg/dashboard/server.go:1506` | -| `GET` | `/api/health/deep` | Public | Deep health probe | `pkg/dashboard/server.go:1507` | -| `GET` | `/api/livez` | Public | Kubernetes liveness probe | `pkg/dashboard/server.go:1508` | -| `GET` | `/metrics` | Registered only when `HIVE_METRICS_ENABLED`; requires `Authorization: ****** (403 if the token is unset) | Prometheus metrics | `pkg/dashboard/server.go:1514` | -| `GET` | `/api/status` | Dashboard auth/session | Dashboard aggregate status; `labelPresets[]` (`{key,label,description,labels[],states[]}`) lists the built-in label-filter presets for the Projects cards, led by `needs-human` whose `labels[]` are the effective hard-suppress labels plus `needs-signal`, the hold labels, `stage/071-needs-author` and the reporter-trust awaiting label (#12172); accepts `?fields=a,b` for top-level selection and `?omit=repos` to drop heavy sections, and honors `Accept-Encoding: gzip`; `agents[]` includes busy-pane visibility fields (`kicksUndeliverable`, `busySince`, `lastTranscriptActivity`, `condition`, `conditionMessage`) when a Working pane cannot accept kicks; a working agent also carries `agents[].run` — the live per-run line (#14576): `{since, elapsedS, costUSD, tokens, priced, tests, filesChanged}`, the estimated spend/tokens since the kick was delivered (`lastKickAt`, falling back to `busySince`) (delta of the token collector's per-agent cumulative figure; `priced:false` when the agent has none yet), the last visible test-runner result (`pass`/`fail`/empty) and the last `N files changed` git summary (`-1` when none) parsed from the captured pane; top-level `fleetRun` `{running, hourUSD, hourPriced}` is the governor strip's `running N · $X this hour` figure; `sla` (fleet) and `repos[].sla` (per repo) carry the Delivery SLA block derived from the lifecycle timeline with no extra GitHub calls: `windows.{7d,30d}.{open_to_triage,triage_to_start,start_to_pr,pr_to_merge,open_to_merge}` as `{p50_h,p95_h,n}` (hours, nearest-rank), `history[]` of up to 30 daily buckets `{day,start_to_pr_p50,pr_to_merge_p50,open_to_merge_p50,n_started,n_pr,n_merged}` persisted under `/data/sla-history.json`, `stale_in_progress[]` (`{number,repo,started_at,age_h,agent}` — issues carrying `hive/in-progress` longer than the agent claim TTL with no linked PR), `stale_count`, `merged_7d`, `claim_ttl_s` and, on the fleet block, the effective `targets` (`start_to_pr_h`, `pr_to_merge_h`, `open_to_merge_h`); `staleSince` and `repos[].countsStale` / `countsStaleSince` mark totals served from last-good figures — the previous enumeration (#13382) or the ForgeState cache's last-good counts (#13376) | `pkg/dashboard/server.go:1519` | -| `GET` | `/api/status/summary` | Dashboard auth/session | Compact agent/governor/budget status summary for pollers; honors `Accept-Encoding: gzip` | `pkg/dashboard/server.go:1520` | -| `GET` | `/api/overview/issues.csv` | Dashboard auth/session | Cached Overview issue-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1521` | -| `GET` | `/api/overview/issues.json` | Dashboard auth/session | Cached Overview issue-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1522` | -| `GET` | `/api/overview/prs.csv` | Dashboard auth/session | Cached Overview PR-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1523` | -| `GET` | `/api/overview/prs.json` | Dashboard auth/session | Cached Overview PR-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1524` | -| `GET` | `/api/events` | Dashboard auth/session | Server-sent event stream | `pkg/dashboard/server.go:1525` | +| `GET` | `/api/health` | Public | Basic health probe | `pkg/dashboard/server.go:1511` | +| `GET` | `/api/health/deep` | Public | Deep health probe | `pkg/dashboard/server.go:1512` | +| `GET` | `/api/livez` | Public | Kubernetes liveness probe | `pkg/dashboard/server.go:1513` | +| `GET` | `/metrics` | Registered only when `HIVE_METRICS_ENABLED`; requires `Authorization: ****** (403 if the token is unset) | Prometheus metrics | `pkg/dashboard/server.go:1519` | +| `GET` | `/api/status` | Dashboard auth/session | Dashboard aggregate status; `labelPresets[]` (`{key,label,description,labels[],states[]}`) lists the built-in label-filter presets for the Projects cards, led by `needs-human` whose `labels[]` are the effective hard-suppress labels plus `needs-signal`, the hold labels, `stage/071-needs-author` and the reporter-trust awaiting label (#12172); accepts `?fields=a,b` for top-level selection and `?omit=repos` to drop heavy sections, and honors `Accept-Encoding: gzip`; `agents[]` includes busy-pane visibility fields (`kicksUndeliverable`, `busySince`, `lastTranscriptActivity`, `condition`, `conditionMessage`) when a Working pane cannot accept kicks; a working agent also carries `agents[].run` — the live per-run line (#14576): `{since, elapsedS, costUSD, tokens, priced, tests, filesChanged}`, the estimated spend/tokens since the kick was delivered (`lastKickAt`, falling back to `busySince`) (delta of the token collector's per-agent cumulative figure; `priced:false` when the agent has none yet), the last visible test-runner result (`pass`/`fail`/empty) and the last `N files changed` git summary (`-1` when none) parsed from the captured pane; top-level `fleetRun` `{running, hourUSD, hourPriced}` is the governor strip's `running N · $X this hour` figure; `sla` (fleet) and `repos[].sla` (per repo) carry the Delivery SLA block derived from the lifecycle timeline with no extra GitHub calls: `windows.{7d,30d}.{open_to_triage,triage_to_start,start_to_pr,pr_to_merge,open_to_merge}` as `{p50_h,p95_h,n}` (hours, nearest-rank), `history[]` of up to 30 daily buckets `{day,start_to_pr_p50,pr_to_merge_p50,open_to_merge_p50,n_started,n_pr,n_merged}` persisted under `/data/sla-history.json`, `stale_in_progress[]` (`{number,repo,started_at,age_h,agent}` — issues carrying `hive/in-progress` longer than the agent claim TTL with no linked PR), `stale_count`, `merged_7d`, `claim_ttl_s` and, on the fleet block, the effective `targets` (`start_to_pr_h`, `pr_to_merge_h`, `open_to_merge_h`); `staleSince` and `repos[].countsStale` / `countsStaleSince` mark totals served from last-good figures — the previous enumeration (#13382) or the ForgeState cache's last-good counts (#13376) | `pkg/dashboard/server.go:1524` | +| `GET` | `/api/status/summary` | Dashboard auth/session | Compact agent/governor/budget status summary for pollers; honors `Accept-Encoding: gzip` | `pkg/dashboard/server.go:1525` | +| `GET` | `/api/overview/issues.csv` | Dashboard auth/session | Cached Overview issue-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1526` | +| `GET` | `/api/overview/issues.json` | Dashboard auth/session | Cached Overview issue-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1527` | +| `GET` | `/api/overview/prs.csv` | Dashboard auth/session | Cached Overview PR-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1528` | +| `GET` | `/api/overview/prs.json` | Dashboard auth/session | Cached Overview PR-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1529` | +| `GET` | `/api/events` | Dashboard auth/session | Server-sent event stream | `pkg/dashboard/server.go:1530` | | `GET` | `/api/swarm` | Dashboard auth/session | Current swarm status: display name, duration, active repo, expiry, persisted prep metrics when available, and idle-unlock fields. | `pkg/dashboard/swarm.go:310` | | `POST` | `/api/swarm` | Owner only | Start a 24h repo swarm for a configured repo, returning 409 when another swarm is active and 423 when a previous swarm keeps the next swarm locked until enough agents are idle unless body `{"repo":"owner/name","force":true}` is used; saves the active record before repo prep, persists prep metrics, and announces the start to Discord when configured. | `pkg/dashboard/swarm.go:311` | | `DELETE` | `/api/swarm` | Owner only | End the active swarm, score issues closed and PRs merged in the window, append it to history, and announce the result to Discord when configured. | `pkg/dashboard/swarm.go:312` | @@ -66,7 +66,7 @@ History point limits, timestamp cutoffs, incremental cost reads and ETag/304 beh | Method | Path | Auth | Purpose | Source | |---|---|---|---|---| | `GET` | `/api/style` | Public | Sanitized custom dashboard CSS | `pkg/dashboard/api.go:44` | -| `GET` | `/branding/custom.css` | Dashboard auth/session | Operator branding stylesheet override, read per request (see [branding](branding.md)) | `pkg/dashboard/server.go:1586` | +| `GET` | `/branding/custom.css` | Dashboard auth/session | Operator branding stylesheet override, read per request (see [branding](branding.md)) | `pkg/dashboard/server.go:1591` | | `GET` | `/api/snapshot/frame-ancestors` | Public | Snapshot framing allowlist | `pkg/dashboard/api.go:77` | | `GET` | `/api/snapshot` | Public | Snapshot data | `pkg/dashboard/api.go:78` | | `GET` | `/snapshot` | Public | Public read-only snapshot page | `pkg/dashboard/api.go:79` | @@ -94,9 +94,9 @@ History point limits, timestamp cutoffs, incremental cost reads and ETag/304 beh | `GET` | `/api/openrouter/models` | Dashboard auth/session | Open Router Models | `pkg/dashboard/openrouter.go:47` | | `GET` | `/api/openrouter/credit` | Dashboard auth/session | Open Router Credit | `pkg/dashboard/openrouter.go:48` | | `GET` | `/openrouter/callback` | Public | Open Router Callback | `pkg/dashboard/openrouter.go:49` | -| `POST` | `/api/github-app/recheck` | Dashboard auth/session | GitHub App Recheck | `pkg/dashboard/server.go:1526` | -| `POST` | `/api/github-app/install-clicked` | Dashboard auth/session | GitHub App Install Clicked | `pkg/dashboard/server.go:1527` | -| `GET` | `/gh-setup` | Public | GitHub App Setup Callback | `pkg/dashboard/server.go:1528` | +| `POST` | `/api/github-app/recheck` | Dashboard auth/session | GitHub App Recheck | `pkg/dashboard/server.go:1531` | +| `POST` | `/api/github-app/install-clicked` | Dashboard auth/session | GitHub App Install Clicked | `pkg/dashboard/server.go:1532` | +| `GET` | `/gh-setup` | Public | GitHub App Setup Callback | `pkg/dashboard/server.go:1533` | ## Configuration @@ -593,7 +593,7 @@ always resolved server-side from the validated token. | `POST` | `/api/hives/{id}/heartbeat` | Dashboard auth/session | Hives Heartbeat | `pkg/dashboard/api_contribute.go:275` | | `DELETE` | `/api/hives/{id}` | Owner only | Hives Delete | `pkg/dashboard/api_contribute.go:276` | | `POST` | `/api/hives/onboard` | Dashboard auth/session | Hives Onboard | `pkg/dashboard/api_contribute.go:277` | -| `GET` | `/sso` | Public | SSO | `pkg/dashboard/server.go:1533` | +| `GET` | `/sso` | Public | SSO | `pkg/dashboard/server.go:1538` | ## Hub SaaS diff --git a/src/docs/sandbox-isolation.md b/src/docs/sandbox-isolation.md index 3fafa3db3c..2cd87a0456 100644 --- a/src/docs/sandbox-isolation.md +++ b/src/docs/sandbox-isolation.md @@ -172,6 +172,25 @@ agents: Hive warns at boot and on reload (`config.AgentSandboxGateWarnings`) when a job-runtime agent resolves no `workspace_claim` or names a runtime hive does not have, and refuses the kick with the same message rather than failing it after the agent is marked running. +## Health and metrics for sandboxed agents + +A sandboxed agent (Podman or Job runtime) has no tmux session, no in-pod process and no `pluk` log between kicks: the manager parks it in the `idle` state and each kick runs to completion in a container or Job. The dashboard therefore must not read "not running" as "session down" for these agents, and the token collector cannot find their CLI session files because the CLI's home directory lives in the sandbox, not in the hive pod. + +**Last-run record.** When a sandboxed kick ends, `crux/agent` builds a `SandboxRun` (runtime, start/end, succeeded, timed out, error, commit count, branch, PR URL, summary, usage) and keeps it on the agent process as `sandboxLastRun`. It is also written to `/sandbox-runs/.json` (restored on restart, so a pod roll does not blank the card) and appended to `/sandbox-runs/runs.jsonl` (`work_dir` is `HIVE_WORK_DIR`, default `/data/agents`). The summary is the agent's `.hive/agent-report.json` `summary` when it left one, otherwise the scrubbed tail of the transcript. + +**Health.** `/api/status` carries a `sandboxRun` object on every sandboxed agent with a `health` verdict the SPA uses instead of the session check: + +| `health` | Meaning | Dot | +|-----------|---------|-----| +| `ok` | last run succeeded (or no run yet) and the next cadence occurrence is still in the future | green | +| `running` | a kick is in flight (container or Job running/pending) | amber | +| `failed` | the last run failed or timed out — see the agent's log | red | +| `overdue` | the cadence had a due occurrence that never started, older than the grace (half the period, clamped to 15 min – 2 h); `dueAt` names it | red | + +On-demand agents, paused agents and `pause`/`continuous` cadences are never overdue. A never-kicked interval agent counts its first period from the time the agent came up. The card's blocker line reads `Job ok` / `Job running` / `Job ready` (or `sandbox …` for Podman) / `last run failed` / `last run timed out` / `run overdue` for sandboxed agents instead of `session down`, and the "Now:" line shows the last run's outcome, duration, PR link and summary. + +**Tokens and sessions.** The collector reads `sandbox-runs/runs.jsonl` and turns every run that reported usage into one session (`sandbox--`, backend `sandbox`), so TOKENS/SESSIONS, by-agent and by-model totals and cost include sandboxed work. Usage is parsed from the transcript: Claude's `--output-format json`/`stream-json` `result` line and Copilot's "Breakdown by AI model" footer are recognised. A backend that prints plain text only (Claude without `--output-format`) reports no usage; such runs stay healthy but contribute no tokens rather than an estimate. + ## Network trade-off `network_mode: none` remains available and maps to Podman's `--network=none`, but it only works for non-inference jobs or runtimes that already expose a local/socket model proxy inside the container. The default sandbox network mode is `restricted`: operators must provide a Podman network/proxy policy that allows only the inference endpoint and MITM proxy required by the selected backend. This is a compromise until every supported backend can run through a credential-free local socket without general egress. diff --git a/src/pkg/dashboard/agent_activity_section_10935_test.go b/src/pkg/dashboard/agent_activity_section_10935_test.go index 6ed8bea4c0..9b73c4531f 100644 --- a/src/pkg/dashboard/agent_activity_section_10935_test.go +++ b/src/pkg/dashboard/agent_activity_section_10935_test.go @@ -31,6 +31,11 @@ func TestAgentActivityRows(t *testing.T) { jsFunc(t, html, "agentNowItem"), jsFunc(t, html, "agentNowItemHtml"), jsFunc(t, html, "agentNowReservations"), + jsFunc(t, html, "agentSandboxRun"), + jsFunc(t, html, "agentSessionDown"), + jsFunc(t, html, "agentSandboxDownLabel"), + jsFunc(t, html, "agentSandboxRunText"), + jsFunc(t, html, "agentSandboxRunHtml"), jsFunc(t, html, "agentNowLineHtml"), jsFunc(t, html, "agentNowStateOf"), jsFunc(t, html, "agentStateText"), diff --git a/src/pkg/dashboard/disabled_agent_enable_test.go b/src/pkg/dashboard/disabled_agent_enable_test.go index 4e980e0ae5..c683a5ee35 100644 --- a/src/pkg/dashboard/disabled_agent_enable_test.go +++ b/src/pkg/dashboard/disabled_agent_enable_test.go @@ -232,6 +232,11 @@ func TestAgentNowLineStates(t *testing.T) { jsFunc(t, html, "agentNowItem"), jsFunc(t, html, "agentNowItemHtml"), jsFunc(t, html, "agentNowReservations"), + jsFunc(t, html, "agentSandboxRun"), + jsFunc(t, html, "agentSessionDown"), + jsFunc(t, html, "agentSandboxDownLabel"), + jsFunc(t, html, "agentSandboxRunText"), + jsFunc(t, html, "agentSandboxRunHtml"), jsFunc(t, html, "agentNowLineHtml"), }, "\n") + ` const assert = require('assert'); diff --git a/src/pkg/dashboard/sandbox_health.go b/src/pkg/dashboard/sandbox_health.go new file mode 100644 index 0000000000..f51cd1c629 --- /dev/null +++ b/src/pkg/dashboard/sandbox_health.go @@ -0,0 +1,149 @@ +package dashboard + +import ( + "time" + + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/pkg/config" +) + +// Sandbox run health verdicts. A sandboxed agent (podman or Kubernetes Job +// runtime) has no tmux session or in-pod process between kicks, so the +// process-state rule the SPA applies to every other agent — anything but +// "running" is down — reads a perfectly healthy sandbox agent as "Stopped — +// see the agent's log" with a red dot and "session down". These verdicts +// derive instead from the last run and the cadence, which is all a Job +// agent has. +const ( + // SandboxHealthOK: the last run succeeded (or none has run yet) and the + // next scheduled kick is still in the future. + SandboxHealthOK = "ok" + // SandboxHealthRunning: a kick is in flight (Job pending or running). + SandboxHealthRunning = "running" + // SandboxHealthFailed: the last run failed or timed out. + SandboxHealthFailed = "failed" + // SandboxHealthOverdue: a scheduled kick is due and the grace has passed + // without a run starting — the cadence was missed. + SandboxHealthOverdue = "overdue" +) + +const ( + // sandboxOverdueGraceMin / Max bound the grace after a due occurrence + // before a missed cadence is reported: half the period, clamped so a + // 15-minute interval is not flagged by normal governor jitter and a + // daily cron is not left red for twelve hours. + sandboxOverdueGraceMin = 15 * time.Minute + sandboxOverdueGraceMax = 2 * time.Hour + sandboxOverdueGraceDiv = 2 +) + +// FrontendSandboxRun is the sandbox run record and health verdict on a +// sandboxed agent's status payload. +type FrontendSandboxRun struct { + // Health is one of the SandboxHealth* verdicts. + Health string `json:"health"` + // Runtime is "podman" or "job" for the last run; empty before the first. + Runtime string `json:"runtime,omitempty"` + // LastRunAt / LastRunEndedAt bracket the last run (RFC 3339, UTC). + LastRunAt string `json:"lastRunAt,omitempty"` + LastRunEndedAt string `json:"lastRunEndedAt,omitempty"` + // DurationS is the last run's wall time in seconds. + DurationS int `json:"durationS,omitempty"` + Succeeded bool `json:"succeeded"` + TimedOut bool `json:"timedOut,omitempty"` + Error string `json:"error,omitempty"` + CommitCount int `json:"commitCount,omitempty"` + Branch string `json:"branch,omitempty"` + PRURL string `json:"prURL,omitempty"` + // Summary is the agent's report summary or the transcript tail. + Summary string `json:"summary,omitempty"` + // Tokens is the usage the run's transcript reported; 0 when it carried + // none (the collector then has no session for this run either). + Tokens int64 `json:"tokens,omitempty"` + Model string `json:"model,omitempty"` + // DueAt is the scheduled occurrence that has been missed when Health is + // overdue; empty otherwise. + DueAt string `json:"dueAt,omitempty"` +} + +// sandboxOverdueGrace is how long after a due occurrence a missed cadence is +// tolerated before the agent is reported overdue. +func sandboxOverdueGrace(cadence config.Cadence, now time.Time) time.Duration { + period, ok := cadence.Period(now) + if !ok || period <= 0 { + return sandboxOverdueGraceMax + } + grace := period / sandboxOverdueGraceDiv + if grace < sandboxOverdueGraceMin { + grace = sandboxOverdueGraceMin + } + if grace > sandboxOverdueGraceMax { + grace = sandboxOverdueGraceMax + } + return grace +} + +// sandboxRunHealth derives a sandboxed agent's health from its process state +// (running = kick in flight), its last run record and its cadence: +// +// - running while a kick is in flight; +// - failed when the last run failed or timed out; +// - overdue when the cadence has a due occurrence older than the grace and +// nothing is running — the kick never started; +// - ok otherwise, including before the first run on a hive whose cadence +// has not come due yet. +// +// cadence is the agent's cadence for the CURRENT governor mode; a paused/off +// cadence or an on-demand agent has nothing to be overdue against. +func sandboxRunHealth(proc *crux.Process, cadence config.Cadence, onDemand bool, now time.Time) FrontendSandboxRun { + out := FrontendSandboxRun{Health: SandboxHealthOK} + if proc == nil { + return out + } + if run := proc.SandboxLastRun; run != nil { + out.Runtime = run.Runtime + out.LastRunAt = formatOptionalTime(run.StartedAt) + out.LastRunEndedAt = formatOptionalTime(run.EndedAt) + out.DurationS = int(run.Duration() / time.Second) + out.Succeeded = run.Succeeded + out.TimedOut = run.TimedOut + out.Error = redactTokens(run.Error) + out.CommitCount = run.CommitCount + out.Branch = run.Branch + out.PRURL = run.PRURL + out.Summary = redactTokens(run.Summary) + out.Tokens = run.Usage.Total() + out.Model = run.Usage.Model + } + if proc.State == crux.StateRunning { + out.Health = SandboxHealthRunning + return out + } + if proc.SandboxLastRun != nil && !proc.SandboxLastRun.Succeeded { + out.Health = SandboxHealthFailed + return out + } + if onDemand || proc.Paused || proc.State == crux.StatePaused || cadence == "" || cadence.IsPaused() || cadence.IsContinuous() { + return out + } + var lastKick time.Time + if proc.LastKick != nil { + lastKick = *proc.LastKick + } + due, ok := cadence.DueOccurrence(lastKick, now, 0) + if !ok { + return out + } + // A never-kicked interval agent is "due now" by definition; it is not + // overdue until the first period has elapsed since the agent came up. + if lastKick.IsZero() && proc.StartedAt != nil { + if next, nok := cadence.NextAfter(*proc.StartedAt); nok { + due = next + } + } + if now.Sub(due) > sandboxOverdueGrace(cadence, now) { + out.Health = SandboxHealthOverdue + out.DueAt = formatOptionalTime(due) + } + return out +} diff --git a/src/pkg/dashboard/sandbox_health_test.go b/src/pkg/dashboard/sandbox_health_test.go new file mode 100644 index 0000000000..5c7ed90bcc --- /dev/null +++ b/src/pkg/dashboard/sandbox_health_test.go @@ -0,0 +1,110 @@ +package dashboard + +import ( + "strings" + "testing" + "time" + + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/crux/cruxtest" + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/governor" +) + +func TestSandboxRunHealth(t *testing.T) { + now := time.Date(2026, 10, 21, 12, 0, 0, 0, time.UTC) + ok := &crux.SandboxRun{Runtime: "job", StartedAt: now.Add(-40 * time.Minute), EndedAt: now.Add(-11 * time.Minute), Succeeded: true, PRURL: "https://github.com/o/r/pull/1402", Usage: crux.SandboxRunUsage{Model: "m", Input: 10, Output: 5}} + failed := &crux.SandboxRun{Runtime: "job", StartedAt: now.Add(-40 * time.Minute), EndedAt: now.Add(-39 * time.Minute), Error: "exit 1"} + kick := func(ago time.Duration) *time.Time { t := now.Add(-ago); return &t } + cases := []struct { + name string + proc *crux.Process + cadence config.Cadence + onDemand bool + want string + }{ + {"nil proc", nil, "30m", false, SandboxHealthOK}, + {"succeeded, next cron in the future", &crux.Process{State: crux.StateIdle, LastKick: kick(40 * time.Minute), SandboxLastRun: ok}, "0 */6 * * *", false, SandboxHealthOK}, + {"idle between interval kicks", &crux.Process{State: crux.StateIdle, LastKick: kick(40 * time.Minute), SandboxLastRun: ok}, "1h", false, SandboxHealthOK}, + {"kick in flight", &crux.Process{State: crux.StateRunning, LastKick: kick(time.Minute), SandboxLastRun: ok}, "1h", false, SandboxHealthRunning}, + {"last run failed", &crux.Process{State: crux.StateIdle, LastKick: kick(40 * time.Minute), SandboxLastRun: failed}, "1h", false, SandboxHealthFailed}, + {"failed state without a record", &crux.Process{State: crux.StateFailed, LastKick: kick(40 * time.Minute)}, "1h", false, SandboxHealthOK}, + {"missed interval past grace", &crux.Process{State: crux.StateIdle, LastKick: kick(3 * time.Hour), SandboxLastRun: ok}, "1h", false, SandboxHealthOverdue}, + {"missed interval within grace", &crux.Process{State: crux.StateIdle, LastKick: kick(75 * time.Minute), SandboxLastRun: ok}, "1h", false, SandboxHealthOK}, + {"on-demand never overdue", &crux.Process{State: crux.StateIdle, LastKick: kick(30 * 24 * time.Hour), SandboxLastRun: ok}, "1h", true, SandboxHealthOK}, + {"paused cadence never overdue", &crux.Process{State: crux.StateIdle, LastKick: kick(30 * 24 * time.Hour), SandboxLastRun: ok}, "pause", false, SandboxHealthOK}, + {"paused process never overdue", &crux.Process{State: crux.StatePaused, Paused: true, LastKick: kick(30 * 24 * time.Hour), SandboxLastRun: ok}, "1h", false, SandboxHealthOK}, + {"never kicked, fresh start", &crux.Process{State: crux.StateIdle, StartedAt: kick(10 * time.Minute)}, "1h", false, SandboxHealthOK}, + {"never kicked, first period long gone", &crux.Process{State: crux.StateIdle, StartedAt: kick(5 * time.Hour)}, "1h", false, SandboxHealthOverdue}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := sandboxRunHealth(tc.proc, tc.cadence, tc.onDemand, now) + if got.Health != tc.want { + t.Fatalf("health = %q, want %q (%+v)", got.Health, tc.want, got) + } + if tc.want == SandboxHealthOverdue && got.DueAt == "" { + t.Errorf("overdue verdict must name the missed occurrence") + } + if tc.want != SandboxHealthOverdue && got.DueAt != "" { + t.Errorf("dueAt must be empty unless overdue, got %q", got.DueAt) + } + }) + } + + got := sandboxRunHealth(&crux.Process{State: crux.StateIdle, LastKick: kick(40 * time.Minute), SandboxLastRun: ok}, "1h", false, now) + if got.Runtime != "job" || !got.Succeeded || got.DurationS != 29*60 || got.PRURL != ok.PRURL || got.Tokens != 15 || got.Model != "m" { + t.Errorf("run fields not carried over: %+v", got) + } +} + +func TestSandboxOverdueGraceClamped(t *testing.T) { + now := time.Date(2026, 10, 21, 12, 0, 0, 0, time.UTC) + for cadence, want := range map[config.Cadence]time.Duration{ + "10m": sandboxOverdueGraceMin, + "1h": 30 * time.Minute, + "0 0 * * *": sandboxOverdueGraceMax, + "continuous": sandboxOverdueGraceMax, + } { + if got := sandboxOverdueGrace(cadence, now); got != want { + t.Errorf("grace(%q) = %v, want %v", cadence, got, want) + } + } +} + +func TestBuildAgents_SandboxedAgentCarriesLastRun(t *testing.T) { + yes := true + agentCfg := config.AgentConfig{Backend: "claude", Enabled: true, Sandbox: &config.AgentSandboxOverride{Enabled: &yes}} + last := time.Now().Add(-30 * time.Minute) + run := &crux.SandboxRun{Runtime: "job", StartedAt: last, EndedAt: last.Add(29 * time.Minute), Succeeded: true, Summary: "opened spyre-inference#1402"} + statuses := map[string]*crux.Process{ + "deps": {Name: "deps", Config: agentCfg, State: crux.StateIdle, LastKick: &last, SandboxLastRun: run, OutputBuffer: cruxtest.NewRingBuffer(10)}, + "plain": {Name: "plain", Config: config.AgentConfig{Backend: "claude", Enabled: true}, State: crux.StateIdle, LastKick: &last, OutputBuffer: cruxtest.NewRingBuffer(10)}, + } + cfg := &config.Config{ + AgentRuntimeConfig: config.AgentRuntimeConfig{ + Agents: map[string]config.AgentConfig{"deps": agentCfg, "plain": statuses["plain"].Config}, + AgentSandbox: config.AgentSandboxConfig{Enabled: true}, + }, + Governor: config.GovernorConfig{Modes: map[string]config.ModeConfig{ + "idle": {Cadences: map[string]config.Cadence{"deps": "0 */6 * * *", "plain": "1h"}}, + }}, + } + byName := map[string]FrontendAgent{} + for _, a := range buildAgents(statuses, cfg, governor.State{Mode: governor.ModeIdle}) { + byName[a.Name] = a + } + deps := byName["deps"] + if !deps.Sandboxed || deps.SandboxRun == nil { + t.Fatalf("sandboxed agent must carry sandboxRun: %+v", deps) + } + if deps.SandboxRun.Health != SandboxHealthOK || deps.SandboxRun.Runtime != "job" || !deps.SandboxRun.Succeeded { + t.Errorf("sandboxRun = %+v", deps.SandboxRun) + } + if !strings.Contains(deps.DetailSummary, "spyre-inference#1402") { + t.Errorf("detailSummary must fall back to the run summary, got %q", deps.DetailSummary) + } + if byName["plain"].SandboxRun != nil { + t.Errorf("non-sandboxed agent must not carry sandboxRun") + } +} diff --git a/src/pkg/dashboard/server.go b/src/pkg/dashboard/server.go index 5d26805530..ef3376a527 100644 --- a/src/pkg/dashboard/server.go +++ b/src/pkg/dashboard/server.go @@ -829,7 +829,12 @@ type FrontendAgent struct { ProxyViolations int `json:"proxyViolations"` OnDemand bool `json:"onDemand,omitempty"` Sandboxed bool `json:"sandboxed,omitempty"` - LastError string `json:"lastError,omitempty"` + // SandboxRun is the last sandboxed run and the health verdict derived + // from it (sandbox_health.go). Present only for sandboxed agents; the + // SPA reads health from here instead of State, which is "idle" between + // kicks for an agent that has no session to be down. + SandboxRun *FrontendSandboxRun `json:"sandboxRun,omitempty"` + LastError string `json:"lastError,omitempty"` StallNudges int `json:"stallNudges,omitempty"` StallRecoveries int `json:"stallRecoveries,omitempty"` ActionNudges int `json:"actionNudges,omitempty"` diff --git a/src/pkg/dashboard/spoke_health_truth_test.go b/src/pkg/dashboard/spoke_health_truth_test.go index f573e6f882..85b9238a80 100644 --- a/src/pkg/dashboard/spoke_health_truth_test.go +++ b/src/pkg/dashboard/spoke_health_truth_test.go @@ -181,7 +181,10 @@ func TestSpokeHealthTruthUIWiring(t *testing.T) { why: "any failing deep check must flip the navbar chip to Degraded while preserving the shared chip slots", }, { - snippet: "const isDown = a.state !== 'running' && !isPaused && !isOff && !isOnDemand && !isStarting;", + // agentSessionDown is `a.state !== 'running'` for every agent + // with a session; only a sandboxed agent (no session between + // kicks) derives down from its last run instead. + snippet: "const isDown = agentSessionDown(a) && !isPaused && !isOff && !isOnDemand && !isStarting;", why: "the sidebar dot must treat every non-running, non-paused process state (stopped, failed, idle) as down", }, { diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html index 03f3c10996..6ab85b153f 100644 --- a/src/pkg/dashboard/static/index.html +++ b/src/pkg/dashboard/static/index.html @@ -10661,7 +10661,9 @@

Where to get help

// Post-restart boot stagger / launch in progress: the agent is not down, // it just hasn't reached its pane yet. Render as 'starting', not red. const isStarting = !!a.starting && a.state !== 'running' && !isPaused && !isOff; - const isStopped = a.state !== 'running' && !isPaused && !isOff && !isOnDemand && !isStarting; + // Sandboxed agents: down = last run failed / cadence missed, never + // "no session" (agentSessionDown). + const isStopped = agentSessionDown(a) && !isPaused && !isOff && !isOnDemand && !isStarting; // Same formula as the Auth section's Login button below: a down agent // with an unauthenticated CLI gets the amber needs-login state instead // of plain down-red (#2465). @@ -10688,7 +10690,7 @@

Where to get help

/* 'disabled' outranks 'down': both reach here with state!=='running', but a switched-off agent is not a fault, and labelling it "down" sent operators looking for a crash that never happened. */ - const stateLabel = agentSessionUnavailable(a) ? 'disabled' : needsLoginDown ? 'needs login' : isStopped ? (a.operatorStopped ? 'stopped by operator' : 'down') : isOnDemand ? 'on demand' : isPaused ? 'paused' : isOff ? agentCadenceOffLabel() : isRunning ? (a.awaitingCI ? 'Waiting on CI' : 'working') : 'idle'; + const stateLabel = agentSessionUnavailable(a) ? 'disabled' : needsLoginDown ? 'needs login' : isStopped ? (agentSandboxDownLabel(a) || (a.operatorStopped ? 'stopped by operator' : 'down')) : isOnDemand ? 'on demand' : isPaused ? 'paused' : isOff ? agentCadenceOffLabel() : isRunning ? (a.awaitingCI ? 'Waiting on CI' : 'working') : 'idle'; const kickId = `oc-kick-${a.name}`; const prevInput = document.getElementById(kickId); @@ -11057,7 +11059,7 @@

Where to get help

// down (#2465). Match the deep check: non-running + non-paused = down, // with paused/off/on-demand keeping their own styling. const isStarting = !!a.starting && a.state !== 'running' && !isPaused && !isOff; - const isDown = a.state !== 'running' && !isPaused && !isOff && !isOnDemand && !isStarting; + const isDown = agentSessionDown(a) && !isPaused && !isOff && !isOnDemand && !isStarting; // A down agent whose CLI is unauthenticated is not merely down — it can // never do work until someone logs it in. Same formula as the agent // panel's Login button; rendered amber with a "needs login" note. Only @@ -13476,7 +13478,13 @@

Where to get help

let showReservations = true; if (st.isPoweredOff || st.isOff) { lead = 'Off'; showReservations = false; } else if (st.isPaused) { lead = 'Paused'; showReservations = false; } - else if (st.isStopped) { lead = "Stopped — see the agent's log"; showReservations = false; } + else if (st.isStopped) { + const sbLabel = agentSandboxDownLabel(a); + lead = sbLabel ? sbLabel.charAt(0).toUpperCase() + sbLabel.slice(1) + " — see the agent's log" : "Stopped — see the agent's log"; + showReservations = false; + const sandboxHtml = (sbLabel && !outdated) ? agentSandboxRunHtml(a, ctx.hiveNowMs) : ''; + if (sandboxHtml) parts.push(sandboxHtml); + } else if (st.isStarting || (!a.lastKickAt && a.busy !== 'working')) { lead = 'Not started yet'; showReservations = false; } const la = a.lastAction || null; const item = (!lead && la) ? agentNowItem(la.repo, la.number, la.action, ctx) : null; @@ -13504,6 +13512,8 @@

Where to get help

const age = ago(la.at); parts.push('last: ' + agentNowItemHtml(item, false) + (age ? ', ' + age : '')); } + const sandboxHtml = outdated ? '' : agentSandboxRunHtml(a, ctx.hiveNowMs); + if (sandboxHtml) parts.push(sandboxHtml); } else if (!lead) { lead = esc(st.stateLabel || a.busy || ''); showReservations = false; @@ -13530,7 +13540,7 @@

Where to get help

const isPaused = a.paused === true && !isOnDemand; const isOff = agentHeldByMode(a) && !isPaused && !isOnDemand; const isStarting = !!a.starting && a.state !== 'running' && !isPaused && !isOff; - const isStopped = a.state !== 'running' && !isPaused && !isOff && !isOnDemand && !isStarting; + const isStopped = agentSessionDown(a) && !isPaused && !isOff && !isOnDemand && !isStarting; return { isPoweredOff, isOff, isPaused, isStopped, isStarting, stateLabel: isOnDemand ? 'on demand' : a.busy }; } @@ -17812,13 +17822,58 @@

All labels

const tip = `${who} is enabled but NO governor mode gives it a cadence, and it has never been kicked \u2014 it will sit idle forever. Click to set cadences.`; return ``; } + /* Sandboxed agents (podman or Kubernetes Job runtime) run each kick in a + throwaway container and sit in state 'idle' between kicks: there is no + tmux session or in-pod process to be "down". Their health comes from + the server-side sandboxRun verdict (last run + cadence), never from + state !== 'running'. */ + function agentSandboxRun(a) { + return (a && a.sandboxed && a.sandboxRun && a.sandboxRun.health) ? a.sandboxRun : null; + } + function agentSessionDown(a) { + const sr = agentSandboxRun(a); + if (sr) return sr.health === 'failed' || sr.health === 'overdue'; + return a.state !== 'running'; + } + function agentSandboxDownLabel(a) { + const sr = agentSandboxRun(a); + if (!sr) return ''; + if (sr.health === 'failed') return sr.timedOut ? 'last run timed out' : 'last run failed'; + if (sr.health === 'overdue') return 'run overdue'; + return ''; + } + function agentSandboxRunText(a, hiveNowMs) { + const sr = agentSandboxRun(a); + if (!sr || !sr.lastRunAt) return ''; + const ago = agentNowAgo((hiveNowMs || Date.now()) - Date.parse(sr.lastRunAt)); + const dur = sr.durationS >= 60 ? Math.round(sr.durationS / 60) + ' min' : (sr.durationS || 0) + ' s'; + let text = (sr.succeeded ? 'last run succeeded' : (sr.timedOut ? 'last run timed out' : 'last run failed')) + ' \u00b7 ' + ago + ' \u00b7 ' + dur; + if (sr.commitCount) text += ' \u00b7 ' + sr.commitCount + (sr.commitCount === 1 ? ' commit' : ' commits'); + return text; + } + function agentSandboxRunHtml(a, hiveNowMs) { + const sr = agentSandboxRun(a); + const text = agentSandboxRunText(a, hiveNowMs); + if (!text) return ''; + let html = esc(text); + if (sr.prURL) html += ' \u00b7 PR'; + if (sr.error && !sr.succeeded) html += ' \u2014 ' + esc(sr.error.length > 160 ? sr.error.slice(0, 160) + '\u2026' : sr.error); + return html; + } function agentBlockerLineHtml(a) { const onDemand = a.onDemand === true; let session = 'up', sessionCls = 'ab-ok'; let sessionTip = 'The tmux session is alive.'; + const sandboxRun = agentSandboxRun(a); if (agentIsDisabled(a)) { session = 'disabled'; sessionCls = 'ab-bad'; sessionTip = 'Disabled in config \u2014 the governor never starts this agent. Enable it with \u2699\uFE0F.'; + } else if (sandboxRun) { + const rt = sandboxRun.runtime === 'job' ? 'Job' : 'sandbox'; + if (sandboxRun.health === 'running') { session = rt + ' running'; sessionCls = 'ab-ok'; sessionTip = 'A sandboxed kick is in flight.'; } + else if (sandboxRun.health === 'failed') { session = agentSandboxDownLabel(a); sessionCls = 'ab-bad'; sessionTip = 'The last sandboxed run failed: ' + (sandboxRun.error || 'see the full log.'); } + else if (sandboxRun.health === 'overdue') { session = 'run overdue'; sessionCls = 'ab-bad'; sessionTip = 'A scheduled kick was due' + (sandboxRun.dueAt ? ' at ' + sandboxRun.dueAt : '') + ' and no run has started.'; } + else { session = sandboxRun.lastRunAt ? rt + ' ok' : rt + ' ready'; sessionCls = 'ab-ok'; sessionTip = 'Sandboxed agent: each kick runs in its own container, so there is no session between kicks. ' + (agentSandboxRunText(a) || 'No run yet.'); } } else if (a.state !== 'running') { session = 'down'; sessionCls = 'ab-bad'; sessionTip = 'No live tmux session \u2014 \u21bb restart asks the supervisor to respawn it.'; @@ -17889,7 +17944,7 @@

All labels

// Down = process not alive — anything but 'running'. 'stopped'-only // matching rendered crash-looping ('failed') agents green (#2465). const isStarting = !!a.starting && a.state !== 'running' && !isPaused && !isOff; - const isStopped = a.state !== 'running' && !isPaused && !isOff && !isOnDemand && !isStarting; + const isStopped = agentSessionDown(a) && !isPaused && !isOff && !isOnDemand && !isStarting; const needsLogin = _agentNeedsLogin(a, isStopped); const STALE_MS = 1200000; // 20 minutes const isStale = a.summaryUpdated && !isPaused && a.busy === 'working' && (Date.now() - new Date(a.summaryUpdated).getTime()) > STALE_MS; diff --git a/src/pkg/dashboard/status_builder.go b/src/pkg/dashboard/status_builder.go index fea23e62dd..2767eb61da 100644 --- a/src/pkg/dashboard/status_builder.go +++ b/src/pkg/dashboard/status_builder.go @@ -1120,6 +1120,16 @@ func buildAgentsWithHidden(statuses map[string]*crux.Process, cfg *config.Config } agentCfg := proc.Config + var sandboxRun *FrontendSandboxRun + if agentCfg.SandboxEnabled(cfg.AgentSandbox) { + sr := sandboxRunHealth(proc, cadenceValue, agentCfg.OnDemand || onDemandSet[name], time.Now()) + sandboxRun = &sr + // A sandboxed agent has no pane; after a pod roll its output ring + // is empty too, so the persisted run summary is the activity. + if detailSummary == "" && sr.Summary != "" { + detailSummary = sr.Summary + } + } continuousModes := cfg.ContinuousModes(name) continuousNow := agentCfg.ContinuousInMode(currentMode, modeCadence) continuousBlocked := continuousState.Blocked @@ -1143,6 +1153,7 @@ func buildAgentsWithHidden(statuses map[string]*crux.Process, cfg *config.Config ReplicaCount: agentCfg.ReplicaCount, OnDemand: agentCfg.OnDemand, Sandboxed: agentCfg.SandboxEnabled(cfg.AgentSandbox), + SandboxRun: sandboxRun, Session: name, State: string(proc.State), Starting: proc.Starting && proc.State != crux.StateRunning, diff --git a/src/pkg/tokens/collector.go b/src/pkg/tokens/collector.go index e9bb096e04..29b6f33012 100644 --- a/src/pkg/tokens/collector.go +++ b/src/pkg/tokens/collector.go @@ -189,6 +189,7 @@ type Diagnostics struct { LastClaudeScanError string `json:"last_claude_scan_error,omitempty"` LastCopilotScanError string `json:"last_copilot_scan_error,omitempty"` LastBobScanError string `json:"last_bob_scan_error,omitempty"` + LastSandboxScanError string `json:"last_sandbox_scan_error,omitempty"` LiveCaptureEnabled bool `json:"live_capture_enabled,omitempty"` } @@ -206,6 +207,7 @@ type Collector struct { claudeSessionsDir string copilotSessionsDir string bobSessionsDir string + sandboxRunHistoryPath string copilotLiveCaptureSinceMs int64 persistPath string detector func(string) string @@ -265,6 +267,13 @@ func (c *Collector) SetCopilotSessionsDir(dir string) { c.copilotSessionsDir = dir } +// SetSandboxRunHistoryPath points the collector at the sandbox run history +// (crux/agent sandbox-runs/runs.jsonl) so usage reported by podman and Job +// sandbox kicks is attributed to their agents like any other session. +func (c *Collector) SetSandboxRunHistoryPath(path string) { + c.sandboxRunHistoryPath = path +} + func (c *Collector) SetBobSessionsDir(dir string) { c.bobSessionsDir = dir } @@ -375,6 +384,16 @@ func (c *Collector) scan() { c.mu.Unlock() } + if c.sandboxRunHistoryPath != "" { + sandboxAgg, err := ScanSandboxRuns(c.sandboxRunHistoryPath) + if err != nil { + c.logger.Warn("sandbox run history scan failed", "error", err) + diag.LastSandboxScanError = err.Error() + } else if sandboxAgg != nil && sandboxAgg.SessionCount > 0 { + MergeAggregates(agg, sandboxAgg) + } + } + sessionDelta := agg.SessionCount - c.prevSessionCount tokenDelta := agg.TotalTokens - c.prevTotalTokens diff --git a/src/pkg/tokens/sandbox_scanner.go b/src/pkg/tokens/sandbox_scanner.go new file mode 100644 index 0000000000..216716bcb3 --- /dev/null +++ b/src/pkg/tokens/sandbox_scanner.go @@ -0,0 +1,127 @@ +package tokens + +import ( + "bufio" + "encoding/json" + "errors" + "fmt" + "os" + "strconv" + "time" +) + +// BackendSandbox names sessions reconstructed from sandbox run records +// (crux/agent sandbox-runs/runs.jsonl). A sandboxed kick runs in a podman +// container or a Kubernetes Job whose CLI home never reaches the pod, so the +// usage it reported in its transcript is the only record the hive has. +const BackendSandbox = "sandbox" + +// sandboxSessionIDPrefix keeps these session ids distinct from every scanner +// that keys on a file name. +const sandboxSessionIDPrefix = "sandbox-" + +// sandboxRunRecord is the subset of a crux/agent SandboxRunHistoryEntry this +// scanner reads. It is declared here rather than imported so pkg/tokens stays +// independent of the runtime module; the JSON keys are the contract. +type sandboxRunRecord struct { + Agent string `json:"agent"` + Backend string `json:"backend"` + Model string `json:"model"` + Run struct { + StartedAt time.Time `json:"startedAt"` + EndedAt time.Time `json:"endedAt"` + Usage struct { + Model string `json:"model"` + Input int64 `json:"input"` + Output int64 `json:"output"` + CacheRead int64 `json:"cacheRead"` + CacheCreate int64 `json:"cacheCreate"` + } `json:"usage"` + } `json:"run"` +} + +// ScanSandboxRuns reads a sandbox run history (one JSON record per line) and +// returns one session per run that reported token usage. Runs whose +// transcript carried no usage contribute nothing: the collector must not +// invent tokens, and a zero-token session would be dropped downstream anyway. +// A missing file is not an error — the hive simply has no sandboxed agents. +func ScanSandboxRuns(path string) (*AggregateSummary, error) { + agg := &AggregateSummary{ + ByAgent: make(map[string]int64), + ByModel: make(map[string]int64), + ByAgentDetail: make(map[string]*AgentModelBucket), + ByModelDetail: make(map[string]*AgentModelBucket), + } + f, err := os.Open(path) + if errors.Is(err, os.ErrNotExist) { + return agg, nil + } + if err != nil { + return nil, fmt.Errorf("opening sandbox run history: %w", err) + } + defer func() { _ = f.Close() }() + + scanner := bufio.NewScanner(f) + const maxLine = 1 << 20 + scanner.Buffer(make([]byte, 0, maxLine), maxLine) + var sessions []SessionSummary + for scanner.Scan() { + var rec sandboxRunRecord + if json.Unmarshal(scanner.Bytes(), &rec) != nil || rec.Agent == "" { + continue + } + if sess, ok := sandboxRunSession(rec); ok { + sessions = append(sessions, sess) + } + } + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("reading sandbox run history: %w", err) + } + MergeAggregates(agg, &AggregateSummary{Sessions: sessions}) + return agg, nil +} + +func sandboxRunSession(rec sandboxRunRecord) (SessionSummary, bool) { + u := rec.Run.Usage + total := u.Input + u.Output + u.CacheRead + u.CacheCreate + if total <= 0 || rec.Run.StartedAt.IsZero() { + return SessionSummary{}, false + } + model := u.Model + if model == "" { + model = rec.Model + } + if model == "" { + model = "unknown" + } + started := rec.Run.StartedAt.UnixMilli() + ended := rec.Run.EndedAt.UnixMilli() + if ended < started { + ended = started + } + // One kick is one user prompt and one assistant turn. + const messagesPerRun = 2 + return SessionSummary{ + SessionID: sandboxSessionIDPrefix + rec.Agent + "-" + strconv.FormatInt(started, 10), + Agent: rec.Agent, + Model: model, + InputTokens: u.Input, + OutputTokens: u.Output, + CacheRead: u.CacheRead, + CacheCreate: u.CacheCreate, + TotalTokens: total, + Messages: messagesPerRun, + FirstActive: started, + LastActive: ended, + Backend: BackendSandbox, + Usage: []UsageEvent{{ + TimestampMs: ended, + Model: model, + Coalesced: 1, + Input: u.Input, + Output: u.Output, + CacheRead: u.CacheRead, + CacheCreate: u.CacheCreate, + }}, + }, true +} diff --git a/src/pkg/tokens/sandbox_scanner_test.go b/src/pkg/tokens/sandbox_scanner_test.go new file mode 100644 index 0000000000..086d2c28d7 --- /dev/null +++ b/src/pkg/tokens/sandbox_scanner_test.go @@ -0,0 +1,79 @@ +package tokens + +import ( + "os" + "path/filepath" + "testing" +) + +const sandboxRunsFixture = `{"agent":"deps","backend":"claude","model":"opus","run":{"runtime":"job","startedAt":"2026-10-21T10:00:00Z","endedAt":"2026-10-21T10:29:00Z","succeeded":true,"usage":{"model":"claude-opus-4","input":100,"output":20,"cacheRead":300,"cacheCreate":5}}} +{"agent":"quality","backend":"copilot","run":{"runtime":"job","startedAt":"2026-10-21T11:00:00Z","endedAt":"2026-10-21T11:05:00Z","succeeded":true}} +not json at all +{"agent":"deps","backend":"claude","run":{"runtime":"job","startedAt":"2026-10-21T16:00:00Z","endedAt":"2026-10-21T16:20:00Z","succeeded":false,"usage":{"input":10,"output":1}}} +` + +func TestScanSandboxRuns(t *testing.T) { + path := filepath.Join(t.TempDir(), "runs.jsonl") + if err := os.WriteFile(path, []byte(sandboxRunsFixture), 0o644); err != nil { + t.Fatal(err) + } + agg, err := ScanSandboxRuns(path) + if err != nil { + t.Fatal(err) + } + // quality reported no usage and the garbage line is skipped: two sessions. + if agg.SessionCount != 2 || len(agg.Sessions) != 2 { + t.Fatalf("sessions = %d (%+v)", agg.SessionCount, agg.Sessions) + } + if agg.TotalTokens != 436 || agg.ByAgent["deps"] != 436 || agg.ByAgent["quality"] != 0 { + t.Errorf("totals = %d, byAgent = %v", agg.TotalTokens, agg.ByAgent) + } + if agg.ByModel["claude-opus-4"] != 425 || agg.ByModel["unknown"] != 11 { + t.Errorf("byModel = %v", agg.ByModel) + } + for _, s := range agg.Sessions { + if s.Backend != BackendSandbox || s.Agent != "deps" || s.SessionID == "" || s.TotalTokens == 0 { + t.Errorf("session = %+v", s) + } + if len(s.Usage) != 1 || s.Usage[0].Input+s.Usage[0].Output+s.Usage[0].CacheRead+s.Usage[0].CacheCreate != s.TotalTokens { + t.Errorf("one timeline event per run reconciling to the total, got %+v", s.Usage) + } + } + if agg.Sessions[0].SessionID == agg.Sessions[1].SessionID { + t.Errorf("runs of the same agent must not collide: %q", agg.Sessions[0].SessionID) + } +} + +func TestScanSandboxRuns_Missing(t *testing.T) { + agg, err := ScanSandboxRuns(filepath.Join(t.TempDir(), "nope.jsonl")) + if err != nil || agg == nil || agg.SessionCount != 0 { + t.Fatalf("missing history must be empty and quiet, got %+v, %v", agg, err) + } + if agg, err := ScanSandboxRuns(""); err != nil || agg.SessionCount != 0 { + t.Fatalf("unset path must be empty and quiet, got %+v, %v", agg, err) + } +} + +func TestCollector_MergesSandboxRuns(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "session1.jsonl"), []byte(`{"type":"usage","model":"sonnet","input_tokens":100,"output_tokens":50}`+"\n"), 0o644); err != nil { + t.Fatal(err) + } + runs := filepath.Join(dir, "runs.jsonl") + if err := os.WriteFile(runs, []byte(sandboxRunsFixture), 0o644); err != nil { + t.Fatal(err) + } + c := NewCollector(dir, testLogger()) + c.SetSandboxRunHistoryPath(runs) + c.scan() + sum := c.Summary() + if sum == nil || sum.ByAgent["deps"] != 436 { + t.Fatalf("sandbox runs must be merged into the aggregate: %+v", sum) + } + if sum.TotalTokens != 150+436 { + t.Errorf("total = %d", sum.TotalTokens) + } + if d := c.Diagnostics(); d.LastSandboxScanError != "" { + t.Errorf("unexpected scan error %q", d.LastSandboxScanError) + } +} From 857df8a3a1364b125c76ee5935f4bd0c25721ef3 Mon Sep 17 00:00:00 2001 From: Andrew Anderson Date: Wed, 28 Oct 2026 01:06:47 -0400 Subject: [PATCH 2/2] changelog: move fragment to the root changelog.d Signed-off-by: Andrew Anderson --- .../fixed-14844-sandbox-agent-health.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename {src/changelog.d => changelog.d}/fixed-14844-sandbox-agent-health.md (100%) diff --git a/src/changelog.d/fixed-14844-sandbox-agent-health.md b/changelog.d/fixed-14844-sandbox-agent-health.md similarity index 100% rename from src/changelog.d/fixed-14844-sandbox-agent-health.md rename to changelog.d/fixed-14844-sandbox-agent-health.md