Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/fixed-sandbox-run-persistence.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- Sandboxed agent run records are now written without holding the agent manager lock, so a slow data volume no longer stalls status and kick calls, and `sandbox-runs/runs.jsonl` is compacted to its newest 1000 records so the token collector's scan cost stays bounded.
2 changes: 2 additions & 0 deletions src/crux/agent/manager.go
Original file line number Diff line number Diff line change
Expand Up @@ -739,6 +739,8 @@ type Manager struct {
// launcher; tests inject a fake through setSandboxJobLauncherFactoryForTest.
sandboxJobLauncherFactory func(config.SandboxJobConfig) sandbox.Launcher
sandboxRunner sandboxCommandRunner
// sandboxRunPersister overrides persistSandboxRun in tests.
sandboxRunPersister func(workDir string, entry SandboxRunHistoryEntry) error
sandboxPushMinter pushbroker.TokenMinter
sandboxPRClient PRCreator
sandboxMutation effects.Boundary
Expand Down
20 changes: 18 additions & 2 deletions src/crux/agent/manager_sandbox.go
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
package agent

import (
Expand Down Expand Up @@ -203,6 +203,19 @@
}
res, err := exec.Run(ctx, spec)
ended := time.Now()
// The run record is written to disk after m.mu is released (defers run
// LIFO), so a slow volume never stalls every status reader.
var pending *SandboxRunHistoryEntry
var persist func(string, SandboxRunHistoryEntry) error
workDir := ""
defer func() {
if pending == nil {
return
}
if perr := persist(workDir, *pending); perr != nil {
m.logger.Warn("sandbox run record not persisted", "name", name, "error", perr)
}
}()
m.mu.Lock()
defer m.mu.Unlock()
agent, ok := m.agents[name]
Expand All @@ -222,8 +235,11 @@
agent.KickOutcome.KickAt = *agent.LastKick
}
entry := SandboxRunHistoryEntry{Agent: name, Backend: effectiveBackend(agent), Model: agent.Config.Model, Run: run}
if perr := persistSandboxRun(m.workDir, entry); perr != nil {
m.logger.Warn("sandbox run record not persisted", "name", name, "error", perr)
pending = &entry
workDir = m.workDir
persist = m.sandboxRunPersister
if persist == nil {
persist = persistSandboxRun
}
}
if err != nil {
Expand Down
46 changes: 44 additions & 2 deletions src/crux/agent/sandbox_run.go
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
package agent

import (
Expand All @@ -8,6 +8,7 @@
"regexp"
"strconv"
"strings"
"sync"
"time"

"github.com/hivecommons/hive/crux"
Expand All @@ -21,8 +22,17 @@
// sandboxRunSummaryTailLines is how much of the transcript tail stands in
// for the summary when the agent left no report.
sandboxRunSummaryTailLines = 20
// sandboxRunHistoryMaxBytes is the size at which runs.jsonl is compacted
// to its newest sandboxRunHistoryKeep records, so the file the token
// collector re-reads on every scan stays bounded.
sandboxRunHistoryMaxBytes = 2 << 20
sandboxRunHistoryKeep = 1000
)

// sandboxRunPersistMu serializes persistSandboxRun so compaction never races
// an append from another agent's run, now that callers no longer hold m.mu.
var sandboxRunPersistMu sync.Mutex

func sandboxRunPath(workDir, agent string) string {
return filepath.Join(crux.SandboxRunsDir(workDir), sanitizeBranchPart(agent)+".json")
}
Expand Down Expand Up @@ -189,6 +199,8 @@
// the history. Failures are returned for logging only; the in-memory record
// is already set.
func persistSandboxRun(workDir string, entry SandboxRunHistoryEntry) error {
sandboxRunPersistMu.Lock()
defer sandboxRunPersistMu.Unlock()
dir := crux.SandboxRunsDir(workDir)
if err := os.MkdirAll(dir, 0o770); err != nil {
return err
Expand All @@ -209,9 +221,39 @@
if err != nil {
return err
}
defer func() { _ = f.Close() }()
_, err = f.Write(append(last, '\n'))
return err
if cerr := f.Close(); err == nil {
err = cerr
}
if err != nil {
return err
}
return compactSandboxRunHistory(crux.SandboxRunHistoryPath(workDir), sandboxRunHistoryMaxBytes, sandboxRunHistoryKeep)
}

// compactSandboxRunHistory rewrites the history to its last keep lines once
// it exceeds maxBytes. The file format is unchanged.
func compactSandboxRunHistory(path string, maxBytes int64, keep int) error {
info, err := os.Stat(path)
if err != nil || info.Size() <= maxBytes {
return err
}
data, err := os.ReadFile(path)
if err != nil {
return err
}
lines := strings.SplitAfter(string(data), "\n")
if n := len(lines); n > 0 && lines[n-1] == "" {
lines = lines[:n-1]
}
if len(lines) <= keep {
return nil
}
tmp := path + ".tmp"
if err := os.WriteFile(tmp, []byte(strings.Join(lines[len(lines)-keep:], "")), 0o660); err != nil {
return err
}
return os.Rename(tmp, path)
}

// loadSandboxRun reads the agent's persisted last run; (nil, nil) when none.
Expand Down
58 changes: 58 additions & 0 deletions src/crux/agent/sandbox_run_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,13 @@ import (
"errors"
"os"
"path/filepath"
"strconv"
"strings"
"testing"
"time"

"github.com/hivecommons/hive/crux"
"github.com/hivecommons/hive/crux/config"
"github.com/hivecommons/hive/crux/outputschema"
"github.com/hivecommons/hive/crux/sandbox"
)
Expand Down Expand Up @@ -126,3 +128,59 @@ func TestPersistAndLoadSandboxRun(t *testing.T) {
t.Errorf("other agent must have no record, got %+v, %v", got, err)
}
}

func TestCompactSandboxRunHistoryKeepsNewestLines(t *testing.T) {
path := filepath.Join(t.TempDir(), "runs.jsonl")
var b strings.Builder
for i := 0; i < 10; i++ {
b.WriteString(`{"agent":"deps","n":` + strconv.Itoa(i) + "}\n")
}
if err := os.WriteFile(path, []byte(b.String()), 0o660); err != nil {
t.Fatal(err)
}
if err := compactSandboxRunHistory(path, 1<<20, 3); err != nil {
t.Fatal(err)
}
if got, _ := os.ReadFile(path); string(got) != b.String() {
t.Fatalf("a file under the cap must be left alone, got %q", got)
}
if err := compactSandboxRunHistory(path, 10, 3); err != nil {
t.Fatal(err)
}
got, _ := os.ReadFile(path)
want := "{\"agent\":\"deps\",\"n\":7}\n{\"agent\":\"deps\",\"n\":8}\n{\"agent\":\"deps\",\"n\":9}\n"
if string(got) != want {
t.Errorf("compacted history = %q, want %q", got, want)
}
}

func TestSandboxRunPersistedWithoutManagerLock(t *testing.T) {
yes := true
m, _, _ := jobRuntimeManager(t,
&config.AgentSandboxOverride{Enabled: &yes},
config.AgentSandboxConfig{Enabled: true, Image: "agent-image",
Job: &config.SandboxJobConfig{WorkspaceClaim: "hive-data"}})
held := make(chan bool, 1)
m.mu.Lock()
m.sandboxRunPersister = func(string, SandboxRunHistoryEntry) error {
if m.mu.TryLock() {
m.mu.Unlock()
held <- false
} else {
held <- true
}
return nil
}
m.mu.Unlock()
if err := m.SendKick("deps", "fix"); err != nil {
t.Fatalf("SendKick: %v", err)
}
select {
case wasHeld := <-held:
if wasHeld {
t.Error("sandbox run was persisted while m.mu was held")
}
case <-time.After(3 * time.Second):
t.Fatal("sandbox run was never persisted")
}
}
4 changes: 2 additions & 2 deletions src/docs/sandbox-isolation.md
Original file line number Diff line number Diff line change
Expand Up @@ -126,7 +126,7 @@ For sandboxed agents, a kick now follows this path:

1. Hive prepares a per-kick host workspace under the sandbox workspace root by cloning/fetching the primary repo with hive-owned credentials before the sandbox starts. The clone URL and sandbox environment are sanitized so GitHub/token variables are not carried into the workspace or container.
2. Hive writes the kick prompt to `.hive/kick-prompt.txt` in the workspace and launches `pkg/sandbox.Launcher` with a rootless Podman `LaunchSpec`, workspace mount, explicit env allowlist, and configured network mode.
3. The manager marks the agent busy while the sandbox runs and returns it to idle or failed when the timeout/completion path finishes. Dashboard status uses the existing agent status structures.
3. The manager marks the agent busy while the sandbox runs and returns it to idle or failed when the timeout/completion path finishes. Because the agent sits in `idle` between kicks, dashboard health for sandboxed agents comes from the last-run record rather than the session check; see [Health and metrics](#health-and-metrics-for-sandboxed-agents).
4. Hive collects the transcript at `.hive/sandbox-transcript.log` and any `agent-report*.json` artifact following `pkg/outputschema` conventions.
5. If the sandbox produced commits, `pkg/pushbroker.Broker` scans the committed diff for token-like secrets and protected-path edits, mints a short-lived scoped GitHub App token outside the sandbox, pushes the branch, and opens a PR through the existing App-authored GitHub client. Broker rejection records audit detail and nothing is pushed.
The PR title and body come from `.hive/pr-title.txt` (first line) and `.hive/pr-body.md` if the agent wrote them in the workspace, otherwise from the HEAD commit's subject and body; a generic `[<agent>] sandbox changes` placeholder is used only when neither exists. Kick templates for sandboxed agents should ask the agent to write those two files.
Expand Down Expand Up @@ -176,7 +176,7 @@ Hive warns at boot and on reload (`config.AgentSandboxGateWarnings`) when a job-

A sandboxed agent (Podman or Job runtime) has no tmux session, no in-pod process and no `pluk` log between kicks: the manager parks it in the `idle` state and each kick runs to completion in a container or Job. The dashboard therefore must not read "not running" as "session down" for these agents, and the token collector cannot find their CLI session files because the CLI's home directory lives in the sandbox, not in the hive pod.

**Last-run record.** When a sandboxed kick ends, `crux/agent` builds a `SandboxRun` (runtime, start/end, succeeded, timed out, error, commit count, branch, PR URL, summary, usage) and keeps it on the agent process as `sandboxLastRun`. It is also written to `<work_dir>/sandbox-runs/<agent>.json` (restored on restart, so a pod roll does not blank the card) and appended to `<work_dir>/sandbox-runs/runs.jsonl` (`work_dir` is `HIVE_WORK_DIR`, default `/data/agents`). The summary is the agent's `.hive/agent-report.json` `summary` when it left one, otherwise the scrubbed tail of the transcript.
**Last-run record.** When a sandboxed kick ends, `crux/agent` builds a `SandboxRun` (runtime, start/end, succeeded, timed out, error, commit count, branch, PR URL, summary, usage) and keeps it on the agent process as `sandboxLastRun`. It is also written to `<work_dir>/sandbox-runs/<agent>.json` (restored on restart, so a pod roll does not blank the card) and appended to `<work_dir>/sandbox-runs/runs.jsonl`, which is compacted to its newest 1000 records once it passes 2 MiB (`work_dir` is `HIVE_WORK_DIR`, default `/data/agents`). The summary is the agent's `.hive/agent-report.json` `summary` when it left one, otherwise the scrubbed tail of the transcript.

**Health.** `/api/status` carries a `sandboxRun` object on every sandboxed agent with a `health` verdict the SPA uses instead of the session check:

Expand Down
Loading