From 02e8979b1d03639b5cd06fbea6a03cfe18779a3a Mon Sep 17 00:00:00 2001 From: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Wed, 28 Oct 2026 02:06:08 -0400 Subject: [PATCH] fix(agent): keep sandbox run persistence off the manager lock and bounded Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- changelog.d/fixed-sandbox-run-persistence.md | 1 + src/crux/agent/manager.go | 2 + src/crux/agent/manager_sandbox.go | 20 ++++++- src/crux/agent/sandbox_run.go | 46 +++++++++++++++- src/crux/agent/sandbox_run_test.go | 58 ++++++++++++++++++++ src/docs/sandbox-isolation.md | 4 +- 6 files changed, 125 insertions(+), 6 deletions(-) create mode 100644 changelog.d/fixed-sandbox-run-persistence.md diff --git a/changelog.d/fixed-sandbox-run-persistence.md b/changelog.d/fixed-sandbox-run-persistence.md new file mode 100644 index 0000000000..781717f3cb --- /dev/null +++ b/changelog.d/fixed-sandbox-run-persistence.md @@ -0,0 +1 @@ +- Sandboxed agent run records are now written without holding the agent manager lock, so a slow data volume no longer stalls status and kick calls, and `sandbox-runs/runs.jsonl` is compacted to its newest 1000 records so the token collector's scan cost stays bounded. diff --git a/src/crux/agent/manager.go b/src/crux/agent/manager.go index 89207f2ebc..bb317bf684 100644 --- a/src/crux/agent/manager.go +++ b/src/crux/agent/manager.go @@ -739,6 +739,8 @@ type Manager struct { // launcher; tests inject a fake through setSandboxJobLauncherFactoryForTest. sandboxJobLauncherFactory func(config.SandboxJobConfig) sandbox.Launcher sandboxRunner sandboxCommandRunner + // sandboxRunPersister overrides persistSandboxRun in tests. + sandboxRunPersister func(workDir string, entry SandboxRunHistoryEntry) error sandboxPushMinter pushbroker.TokenMinter sandboxPRClient PRCreator sandboxMutation effects.Boundary diff --git a/src/crux/agent/manager_sandbox.go b/src/crux/agent/manager_sandbox.go index c85f261555..80a392f5d8 100644 --- a/src/crux/agent/manager_sandbox.go +++ b/src/crux/agent/manager_sandbox.go @@ -203,6 +203,19 @@ func (m *Manager) runSandboxKick(ctx context.Context, name, runtime string, spec } res, err := exec.Run(ctx, spec) ended := time.Now() + // The run record is written to disk after m.mu is released (defers run + // LIFO), so a slow volume never stalls every status reader. + var pending *SandboxRunHistoryEntry + var persist func(string, SandboxRunHistoryEntry) error + workDir := "" + defer func() { + if pending == nil { + return + } + if perr := persist(workDir, *pending); perr != nil { + m.logger.Warn("sandbox run record not persisted", "name", name, "error", perr) + } + }() m.mu.Lock() defer m.mu.Unlock() agent, ok := m.agents[name] @@ -222,8 +235,11 @@ func (m *Manager) runSandboxKick(ctx context.Context, name, runtime string, spec agent.KickOutcome.KickAt = *agent.LastKick } entry := SandboxRunHistoryEntry{Agent: name, Backend: effectiveBackend(agent), Model: agent.Config.Model, Run: run} - if perr := persistSandboxRun(m.workDir, entry); perr != nil { - m.logger.Warn("sandbox run record not persisted", "name", name, "error", perr) + pending = &entry + workDir = m.workDir + persist = m.sandboxRunPersister + if persist == nil { + persist = persistSandboxRun } } if err != nil { diff --git a/src/crux/agent/sandbox_run.go b/src/crux/agent/sandbox_run.go index b7426a7af9..369371841a 100644 --- a/src/crux/agent/sandbox_run.go +++ b/src/crux/agent/sandbox_run.go @@ -8,6 +8,7 @@ import ( "regexp" "strconv" "strings" + "sync" "time" "github.com/hivecommons/hive/crux" @@ -21,8 +22,17 @@ const ( // sandboxRunSummaryTailLines is how much of the transcript tail stands in // for the summary when the agent left no report. sandboxRunSummaryTailLines = 20 + // sandboxRunHistoryMaxBytes is the size at which runs.jsonl is compacted + // to its newest sandboxRunHistoryKeep records, so the file the token + // collector re-reads on every scan stays bounded. + sandboxRunHistoryMaxBytes = 2 << 20 + sandboxRunHistoryKeep = 1000 ) +// sandboxRunPersistMu serializes persistSandboxRun so compaction never races +// an append from another agent's run, now that callers no longer hold m.mu. +var sandboxRunPersistMu sync.Mutex + func sandboxRunPath(workDir, agent string) string { return filepath.Join(crux.SandboxRunsDir(workDir), sanitizeBranchPart(agent)+".json") } @@ -189,6 +199,8 @@ type SandboxRunHistoryEntry struct { // the history. Failures are returned for logging only; the in-memory record // is already set. func persistSandboxRun(workDir string, entry SandboxRunHistoryEntry) error { + sandboxRunPersistMu.Lock() + defer sandboxRunPersistMu.Unlock() dir := crux.SandboxRunsDir(workDir) if err := os.MkdirAll(dir, 0o770); err != nil { return err @@ -209,9 +221,39 @@ func persistSandboxRun(workDir string, entry SandboxRunHistoryEntry) error { if err != nil { return err } - defer func() { _ = f.Close() }() _, err = f.Write(append(last, '\n')) - return err + if cerr := f.Close(); err == nil { + err = cerr + } + if err != nil { + return err + } + return compactSandboxRunHistory(crux.SandboxRunHistoryPath(workDir), sandboxRunHistoryMaxBytes, sandboxRunHistoryKeep) +} + +// compactSandboxRunHistory rewrites the history to its last keep lines once +// it exceeds maxBytes. The file format is unchanged. +func compactSandboxRunHistory(path string, maxBytes int64, keep int) error { + info, err := os.Stat(path) + if err != nil || info.Size() <= maxBytes { + return err + } + data, err := os.ReadFile(path) + if err != nil { + return err + } + lines := strings.SplitAfter(string(data), "\n") + if n := len(lines); n > 0 && lines[n-1] == "" { + lines = lines[:n-1] + } + if len(lines) <= keep { + return nil + } + tmp := path + ".tmp" + if err := os.WriteFile(tmp, []byte(strings.Join(lines[len(lines)-keep:], "")), 0o660); err != nil { + return err + } + return os.Rename(tmp, path) } // loadSandboxRun reads the agent's persisted last run; (nil, nil) when none. diff --git a/src/crux/agent/sandbox_run_test.go b/src/crux/agent/sandbox_run_test.go index b47e1f812b..ff3dfb4e2a 100644 --- a/src/crux/agent/sandbox_run_test.go +++ b/src/crux/agent/sandbox_run_test.go @@ -4,11 +4,13 @@ import ( "errors" "os" "path/filepath" + "strconv" "strings" "testing" "time" "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/crux/config" "github.com/hivecommons/hive/crux/outputschema" "github.com/hivecommons/hive/crux/sandbox" ) @@ -126,3 +128,59 @@ func TestPersistAndLoadSandboxRun(t *testing.T) { t.Errorf("other agent must have no record, got %+v, %v", got, err) } } + +func TestCompactSandboxRunHistoryKeepsNewestLines(t *testing.T) { + path := filepath.Join(t.TempDir(), "runs.jsonl") + var b strings.Builder + for i := 0; i < 10; i++ { + b.WriteString(`{"agent":"deps","n":` + strconv.Itoa(i) + "}\n") + } + if err := os.WriteFile(path, []byte(b.String()), 0o660); err != nil { + t.Fatal(err) + } + if err := compactSandboxRunHistory(path, 1<<20, 3); err != nil { + t.Fatal(err) + } + if got, _ := os.ReadFile(path); string(got) != b.String() { + t.Fatalf("a file under the cap must be left alone, got %q", got) + } + if err := compactSandboxRunHistory(path, 10, 3); err != nil { + t.Fatal(err) + } + got, _ := os.ReadFile(path) + want := "{\"agent\":\"deps\",\"n\":7}\n{\"agent\":\"deps\",\"n\":8}\n{\"agent\":\"deps\",\"n\":9}\n" + if string(got) != want { + t.Errorf("compacted history = %q, want %q", got, want) + } +} + +func TestSandboxRunPersistedWithoutManagerLock(t *testing.T) { + yes := true + m, _, _ := jobRuntimeManager(t, + &config.AgentSandboxOverride{Enabled: &yes}, + config.AgentSandboxConfig{Enabled: true, Image: "agent-image", + Job: &config.SandboxJobConfig{WorkspaceClaim: "hive-data"}}) + held := make(chan bool, 1) + m.mu.Lock() + m.sandboxRunPersister = func(string, SandboxRunHistoryEntry) error { + if m.mu.TryLock() { + m.mu.Unlock() + held <- false + } else { + held <- true + } + return nil + } + m.mu.Unlock() + if err := m.SendKick("deps", "fix"); err != nil { + t.Fatalf("SendKick: %v", err) + } + select { + case wasHeld := <-held: + if wasHeld { + t.Error("sandbox run was persisted while m.mu was held") + } + case <-time.After(3 * time.Second): + t.Fatal("sandbox run was never persisted") + } +} diff --git a/src/docs/sandbox-isolation.md b/src/docs/sandbox-isolation.md index 2cd87a0456..384cb5ebe2 100644 --- a/src/docs/sandbox-isolation.md +++ b/src/docs/sandbox-isolation.md @@ -126,7 +126,7 @@ For sandboxed agents, a kick now follows this path: 1. Hive prepares a per-kick host workspace under the sandbox workspace root by cloning/fetching the primary repo with hive-owned credentials before the sandbox starts. The clone URL and sandbox environment are sanitized so GitHub/token variables are not carried into the workspace or container. 2. Hive writes the kick prompt to `.hive/kick-prompt.txt` in the workspace and launches `pkg/sandbox.Launcher` with a rootless Podman `LaunchSpec`, workspace mount, explicit env allowlist, and configured network mode. -3. The manager marks the agent busy while the sandbox runs and returns it to idle or failed when the timeout/completion path finishes. Dashboard status uses the existing agent status structures. +3. The manager marks the agent busy while the sandbox runs and returns it to idle or failed when the timeout/completion path finishes. Because the agent sits in `idle` between kicks, dashboard health for sandboxed agents comes from the last-run record rather than the session check; see [Health and metrics](#health-and-metrics-for-sandboxed-agents). 4. Hive collects the transcript at `.hive/sandbox-transcript.log` and any `agent-report*.json` artifact following `pkg/outputschema` conventions. 5. If the sandbox produced commits, `pkg/pushbroker.Broker` scans the committed diff for token-like secrets and protected-path edits, mints a short-lived scoped GitHub App token outside the sandbox, pushes the branch, and opens a PR through the existing App-authored GitHub client. Broker rejection records audit detail and nothing is pushed. The PR title and body come from `.hive/pr-title.txt` (first line) and `.hive/pr-body.md` if the agent wrote them in the workspace, otherwise from the HEAD commit's subject and body; a generic `[] sandbox changes` placeholder is used only when neither exists. Kick templates for sandboxed agents should ask the agent to write those two files. @@ -176,7 +176,7 @@ Hive warns at boot and on reload (`config.AgentSandboxGateWarnings`) when a job- A sandboxed agent (Podman or Job runtime) has no tmux session, no in-pod process and no `pluk` log between kicks: the manager parks it in the `idle` state and each kick runs to completion in a container or Job. The dashboard therefore must not read "not running" as "session down" for these agents, and the token collector cannot find their CLI session files because the CLI's home directory lives in the sandbox, not in the hive pod. -**Last-run record.** When a sandboxed kick ends, `crux/agent` builds a `SandboxRun` (runtime, start/end, succeeded, timed out, error, commit count, branch, PR URL, summary, usage) and keeps it on the agent process as `sandboxLastRun`. It is also written to `/sandbox-runs/.json` (restored on restart, so a pod roll does not blank the card) and appended to `/sandbox-runs/runs.jsonl` (`work_dir` is `HIVE_WORK_DIR`, default `/data/agents`). The summary is the agent's `.hive/agent-report.json` `summary` when it left one, otherwise the scrubbed tail of the transcript. +**Last-run record.** When a sandboxed kick ends, `crux/agent` builds a `SandboxRun` (runtime, start/end, succeeded, timed out, error, commit count, branch, PR URL, summary, usage) and keeps it on the agent process as `sandboxLastRun`. It is also written to `/sandbox-runs/.json` (restored on restart, so a pod roll does not blank the card) and appended to `/sandbox-runs/runs.jsonl`, which is compacted to its newest 1000 records once it passes 2 MiB (`work_dir` is `HIVE_WORK_DIR`, default `/data/agents`). The summary is the agent's `.hive/agent-report.json` `summary` when it left one, otherwise the scrubbed tail of the transcript. **Health.** `/api/status` carries a `sandboxRun` object on every sandboxed agent with a `health` verdict the SPA uses instead of the session check: