From 7377efa9a3a33b8981794d67027e3da246cd253c Mon Sep 17 00:00:00 2001 From: Andrew Anderson Date: Wed, 28 Oct 2026 12:34:17 -0400 Subject: [PATCH 1/2] =?UTF-8?q?=E2=9C=A8=20feat(sandbox):=20deterministic?= =?UTF-8?q?=20publish.verify=20preflight=20before=20push/PR?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The report-status gate (#14984) trusts the agent. Add a hive-side preflight: sandbox.publish.verify (per agent), agent_sandbox.publish.verify (global) or an inline block in the kick template lists shell commands hive runs in the same sandbox image and workspace after the run, with HIVE_CHANGED_FILES / HIVE_CHANGED_TESTS / HIVE_BASE_SHA exported. The first non-zero exit, launcher error or timeout withholds the push and the PR; the reason and output tail land in prWithheld (card: 'no PR opened: verify failed …'), the kick log archive gets a '---- verify: ----' section per command, and the existing sandbox_pr_withheld audit line fires. Motivated by torch-spyre/spyre-inference#1412, where a new tests/custom_ops/test_utils.py clashed with tests/multimodal/test_utils.py and broke pytest collection for every CI shard; a whole-suite --collect-only in the preflight would have stopped the PR. Fixes #14999 Reported-by: @joerunde Signed-off-by: Andrew Anderson --- changelog.d/added-sandbox-publish-verify.md | 1 + src/crux/agent/manager_sandbox.go | 23 +- src/crux/agent/sandbox_executor.go | 154 ++++++++- src/crux/agent/sandbox_kick_log.go | 16 + src/crux/agent/sandbox_publish_verify.go | 39 +++ src/crux/agent/sandbox_publish_verify_test.go | 291 ++++++++++++++++++ src/crux/config/agent_config.go | 34 ++ src/crux/config/agent_runtime.go | 25 ++ src/crux/config/sandbox_publish_test.go | 35 +++ src/docs/agent-logging.md | 3 +- src/docs/dashboard-sections.md | 2 +- src/docs/sandbox-isolation.md | 18 ++ src/pkg/dashboard/static/index.html | 5 +- 13 files changed, 634 insertions(+), 12 deletions(-) create mode 100644 changelog.d/added-sandbox-publish-verify.md create mode 100644 src/crux/agent/sandbox_publish_verify.go create mode 100644 src/crux/agent/sandbox_publish_verify_test.go create mode 100644 src/crux/config/sandbox_publish_test.go diff --git a/changelog.d/added-sandbox-publish-verify.md b/changelog.d/added-sandbox-publish-verify.md new file mode 100644 index 0000000000..bfb9d33d4c --- /dev/null +++ b/changelog.d/added-sandbox-publish-verify.md @@ -0,0 +1 @@ +- Sandboxed kicks gained a deterministic publish preflight that does not trust the agent's report: `sandbox.publish.verify` (per agent), `agent_sandbox.publish.verify` (global) or a `` block in the kick template lists shell commands hive runs in the same sandbox image/workspace after the run (with `HIVE_CHANGED_FILES` / `HIVE_CHANGED_TESTS` exported), and the first non-zero exit withholds the push and the PR with the command's output tail on the card (`no PR opened: verify failed (1/2): uv run pytest --collect-only -q (exit 2) …`) and in the kick log. A whole-suite `pytest --collect-only` there would have stopped a test-gap PR whose new test file's basename clashed with an existing one and broke collection for every CI shard. Reported-by: @joerunde (a1-vllm-spyre spoke); hivecommons/hive#14999. diff --git a/src/crux/agent/manager_sandbox.go b/src/crux/agent/manager_sandbox.go index 3c2ff299c8..341174d845 100644 --- a/src/crux/agent/manager_sandbox.go +++ b/src/crux/agent/manager_sandbox.go @@ -140,14 +140,21 @@ func (m *Manager) startSandboxKickLocked(agent *AgentProcess, message string) er Model: agent.Config.Model, LaunchCmd: agent.Config.LaunchCmd, }, - Message: message, - Org: m.project.Org, - Repo: repo, - WorkspaceDir: m.sandboxWorkspaceDirLocked(), - Image: agent.Config.SandboxImage(m.sandboxConfig), - EnvAllowlist: agent.Config.SandboxEnvAllowlist(m.sandboxConfig), - NetworkMode: agent.Config.SandboxNetworkMode(m.sandboxConfig), - Timeout: time.Duration(agent.Config.SandboxTimeoutS(m.sandboxConfig)) * time.Second, + Message: message, + Org: m.project.Org, + Repo: repo, + WorkspaceDir: m.sandboxWorkspaceDirLocked(), + Image: agent.Config.SandboxImage(m.sandboxConfig), + EnvAllowlist: agent.Config.SandboxEnvAllowlist(m.sandboxConfig), + NetworkMode: agent.Config.SandboxNetworkMode(m.sandboxConfig), + Timeout: time.Duration(agent.Config.SandboxTimeoutS(m.sandboxConfig)) * time.Second, + PublishVerify: agent.Config.SandboxPublishVerify(m.sandboxConfig), + VerifyTimeout: time.Duration(agent.Config.SandboxPublishVerifyTimeoutS(m.sandboxConfig)) * time.Second, + } + // A kick template's own verify block wins over config so the policy + // that asks for a proven result also says how hive checks it. + if cmds, ok := parsePublishVerifyDirective(message); ok { + spec.PublishVerify = cmds } runCtx, cancel := context.WithCancel(context.Background()) agent.cancel = cancel diff --git a/src/crux/agent/sandbox_executor.go b/src/crux/agent/sandbox_executor.go index baac6017b0..aa018e197d 100644 --- a/src/crux/agent/sandbox_executor.go +++ b/src/crux/agent/sandbox_executor.go @@ -69,6 +69,13 @@ type SandboxKickSpec struct { EnvAllowlist []string NetworkMode string Timeout time.Duration + // PublishVerify are the deterministic preflight commands run in the + // sandbox before the run's commits are pushed (config + // sandbox.publish.verify or a `` block in the + // kick template). Any non-zero exit withholds the push and the PR. + PublishVerify []string + // VerifyTimeout bounds each PublishVerify command; zero uses the default. + VerifyTimeout time.Duration } type configSnapshot struct { @@ -92,7 +99,10 @@ type SandboxKickResult struct { // PRWithheld names why commits the run left were NOT pushed or turned // into a PR (the agent reported failure, or the sandbox exited non-zero). // Empty when nothing was withheld. - PRWithheld string + PRWithheld string + // Verify records each publish.verify command that ran, in order. The + // first failure stops the list and withholds the PR. + Verify []SandboxVerifyResult TranscriptPath string Sandbox sandbox.Result Broker *pushbroker.Result @@ -101,6 +111,18 @@ type SandboxKickResult struct { Error string } +// SandboxVerifyResult is one publish.verify command's outcome. +type SandboxVerifyResult struct { + Command string + ExitCode int + Output string + Err string + TimedOut bool +} + +// Passed reports whether the command ran to completion with exit 0. +func (v SandboxVerifyResult) Passed() bool { return v.Err == "" && v.ExitCode == 0 && !v.TimedOut } + type SandboxExecutor struct { Launcher sandbox.Launcher Runner sandboxCommandRunner @@ -214,6 +236,16 @@ func (e *SandboxExecutor) Run(ctx context.Context, spec SandboxKickSpec) (Sandbo if count == 0 { return res, nil } + // Deterministic preflight: the report gate above trusts the agent; the + // policy-declared verify commands do not. They run in the same sandbox + // image against the run's workspace, and a non-zero exit withholds the + // push and the PR with the command's tail on the card. + if reason := e.runPublishVerify(ctx, spec, workspace, baseSHA, &res); reason != "" { + res.PRWithheld = reason + err := fmt.Errorf("%w: %s; %d commit(s) not pushed, no PR opened", ErrSandboxRunReportedFailure, reason, count) + res.Error = err.Error() + return res, err + } if !e.PushEnabled { return res, nil } @@ -564,6 +596,126 @@ var ErrSandboxRunReportedFailure = errors.New("sandbox run reported failure") // sandboxPublishWithheld returns why the run's commits must not be published, // or "" when the gate is open. The order matters for the message an operator // reads: the agent's own words first, the exit code second. +const ( + // sandboxVerifyTailMax bounds the verify output quoted on the card and in + // the audit record; the full output lands in the kick log archive. + sandboxVerifyTailMax = 600 + // sandboxVerifyOutputMax bounds what one verify command's combined output + // keeps in memory and in the kick log. + sandboxVerifyOutputMax = 64 << 10 + defaultSandboxVerifyTimeout = 15 * time.Minute +) + +// runPublishVerify runs spec.PublishVerify in order and returns the +// withhold reason for the first failure, or "" when every command passed +// (or none were declared). Results are appended to res.Verify either way. +func (e *SandboxExecutor) runPublishVerify(ctx context.Context, spec SandboxKickSpec, workspace, baseSHA string, res *SandboxKickResult) string { + if len(spec.PublishVerify) == 0 { + return "" + } + changed := e.changedFiles(ctx, workspace, baseSHA) + env := map[string]string{ + "HIVE_CHANGED_FILES": strings.Join(changed, "\n"), + "HIVE_CHANGED_TESTS": strings.Join(changedTestFiles(changed), "\n"), + "HIVE_BASE_SHA": baseSHA, + } + timeout := spec.VerifyTimeout + if timeout <= 0 { + timeout = defaultSandboxVerifyTimeout + } + for i, cmd := range spec.PublishVerify { + cmd = strings.TrimSpace(cmd) + if cmd == "" { + continue + } + vres := SandboxVerifyResult{Command: cmd} + vctx, cancel := context.WithTimeout(ctx, timeout) + sres, err := e.launcher().Run(vctx, sandbox.LaunchSpec{ + Image: spec.Image, + Name: "hive-" + sanitizeBranchPart(spec.Agent) + "-verify", + Workspace: workspace, + WorkspaceMount: sandbox.DefaultWorkspaceMount, + WorkDir: sandbox.DefaultWorkspaceMount, + Command: []string{"sh", "-lc", cmd}, + Env: env, + EnvAllowlist: spec.EnvAllowlist, + NetworkMode: spec.NetworkMode, + }) + vres.TimedOut = errors.Is(vctx.Err(), context.DeadlineExceeded) + cancel() + out := logscrub.ScrubString(sres.Stdout, logscrub.WithMarkers()) + if sres.Stderr != "" { + out += "\n" + logscrub.ScrubString(sres.Stderr, logscrub.WithMarkers()) + } + vres.Output = tailRunes(strings.TrimSpace(out), sandboxVerifyOutputMax) + vres.ExitCode = sres.ExitCode + if err != nil { + vres.Err = err.Error() + } + res.Verify = append(res.Verify, vres) + if vres.Passed() { + continue + } + reason := fmt.Sprintf("verify failed (%d/%d): %s", i+1, len(spec.PublishVerify), cmd) + switch { + case vres.TimedOut: + reason += fmt.Sprintf(" timed out after %s", timeout) + case vres.Err != "": + reason += ": " + vres.Err + default: + reason += fmt.Sprintf(" (exit %d)", vres.ExitCode) + } + if tail := tailRunes(vres.Output, sandboxVerifyTailMax); tail != "" { + reason += "\n" + tail + } + return reason + } + return "" +} + +// changedFiles lists paths touched by the run's commits (base..HEAD). An +// error yields nil so the verify commands still run, with empty env. +func (e *SandboxExecutor) changedFiles(ctx context.Context, workspace, baseSHA string) []string { + out, err := e.runner().Run(ctx, workspace, pushbroker.PushEnv(os.Environ()), "git", "diff", "--name-only", "--diff-filter=AMR", baseSHA, "HEAD") + if err != nil { + return nil + } + var files []string + for _, line := range strings.Split(string(out), "\n") { + if line = strings.TrimSpace(line); line != "" { + files = append(files, line) + } + } + return files +} + +// changedTestFiles keeps the pytest/go/js-style test files out of a changed +// list so a verify command can run exactly the tests the run added. +func changedTestFiles(files []string) []string { + var tests []string + for _, f := range files { + base := filepath.Base(f) + switch { + case strings.HasPrefix(base, "test_") && strings.HasSuffix(base, ".py"), + strings.HasSuffix(base, "_test.py"), + strings.HasSuffix(base, "_test.go"), + strings.HasSuffix(base, ".test.js"), strings.HasSuffix(base, ".test.ts"), + strings.HasSuffix(base, ".spec.js"), strings.HasSuffix(base, ".spec.ts"): + tests = append(tests, f) + } + } + return tests +} + +// tailRunes keeps the last max runes of s. +func tailRunes(s string, max int) string { + r := []rune(s) + if len(r) <= max { + return s + } + return "…" + string(r[len(r)-max:]) +} + func sandboxPublishWithheld(res SandboxKickResult) string { if res.ReportStatus.PublishBlocked() { reason := "agent report status=" + string(res.ReportStatus) diff --git a/src/crux/agent/sandbox_kick_log.go b/src/crux/agent/sandbox_kick_log.go index 08926f279b..56aeede587 100644 --- a/src/crux/agent/sandbox_kick_log.go +++ b/src/crux/agent/sandbox_kick_log.go @@ -60,6 +60,22 @@ func sandboxKickLogBody(runtime string, res SandboxKickResult, runErr error) str if !strings.HasSuffix(res.Sandbox.Stderr, "\n") { b.WriteString("\n") } + for _, v := range res.Verify { + verdict := "passed" + switch { + case v.TimedOut: + verdict = "timed out" + case v.Err != "": + verdict = "error: " + v.Err + case v.ExitCode != 0: + verdict = fmt.Sprintf("exit %d", v.ExitCode) + } + fmt.Fprintf(&b, "\n---- verify: %s (%s) ----\n", v.Command, verdict) + b.WriteString(v.Output) + if !strings.HasSuffix(v.Output, "\n") { + b.WriteString("\n") + } + } return b.String() } diff --git a/src/crux/agent/sandbox_publish_verify.go b/src/crux/agent/sandbox_publish_verify.go new file mode 100644 index 0000000000..ebcacd8ec5 --- /dev/null +++ b/src/crux/agent/sandbox_publish_verify.go @@ -0,0 +1,39 @@ +package agent + +import ( + "regexp" + "strings" +) + +// publishVerifyDirective matches an HTML-comment block a kick template uses +// to declare its publish preflight inline: +// +// +// +// One shell command per line; blank lines and `#` comments are ignored. The +// comment is invisible to the agent-facing markdown yet travels with the +// policy, so the template that tells the agent "run on card first" is the +// same artifact that tells hive how to check it did. +var publishVerifyDirective = regexp.MustCompile(`(?s)`) + +// parsePublishVerifyDirective extracts the verify commands from a rendered +// kick message. ok is true when a directive is present, even if it lists no +// commands (an explicit empty block turns the configured preflight off for +// that kick). +func parsePublishVerifyDirective(message string) (cmds []string, ok bool) { + m := publishVerifyDirective.FindStringSubmatch(message) + if m == nil { + return nil, false + } + for _, line := range strings.Split(m[1], "\n") { + line = strings.TrimSpace(line) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + cmds = append(cmds, line) + } + return cmds, true +} diff --git a/src/crux/agent/sandbox_publish_verify_test.go b/src/crux/agent/sandbox_publish_verify_test.go new file mode 100644 index 0000000000..a8e0052689 --- /dev/null +++ b/src/crux/agent/sandbox_publish_verify_test.go @@ -0,0 +1,291 @@ +package agent + +import ( + "context" + "errors" + "reflect" + "strings" + "sync" + "testing" + + "github.com/hivecommons/hive/crux/config" + "github.com/hivecommons/hive/crux/sandbox" +) + +// sandboxVerifyLauncher answers the agent run with exit 0 and scripts the +// verify commands: every LaunchSpec is recorded so a test can prove the +// preflight ran in the same image/workspace with the changed-file env. +type sandboxVerifyLauncher struct { + mu sync.Mutex + specs []sandbox.LaunchSpec + results map[string]sandbox.Result // keyed by the verify command + errFor string + runCount int +} + +func (l *sandboxVerifyLauncher) Run(_ context.Context, spec sandbox.LaunchSpec) (sandbox.Result, error) { + l.mu.Lock() + defer l.mu.Unlock() + l.specs = append(l.specs, spec) + l.runCount++ + if len(spec.Command) == 3 && spec.Command[0] == "sh" && spec.Command[1] == "-lc" { + cmd := spec.Command[2] + if cmd == l.errFor { + return sandbox.Result{}, errors.New("launcher exploded") + } + if r, ok := l.results[cmd]; ok { + return r, nil + } + } + return sandbox.Result{Stdout: "agent done\n"}, nil +} + +func (l *sandboxVerifyLauncher) verifySpecs() []sandbox.LaunchSpec { + l.mu.Lock() + defer l.mu.Unlock() + var out []sandbox.LaunchSpec + for _, s := range l.specs { + if strings.HasSuffix(s.Name, "-verify") { + out = append(out, s) + } + } + return out +} + +// verifyAwareRunner adds the changed-file listing the preflight exports. +type verifyAwareRunner struct{ sandboxFakeRunner } + +func (r *verifyAwareRunner) Run(ctx context.Context, dir string, env []string, name string, args ...string) ([]byte, error) { + if strings.Join(stripGitConfigArgs(args), " ") == "diff --name-only --diff-filter=AMR basesha HEAD" { + return []byte("src/thing.py\ntests/custom_ops/test_utils.py\n"), nil + } + return r.sandboxFakeRunner.Run(ctx, dir, env, name, args...) +} + +func runVerifyKick(t *testing.T, launcher *sandboxVerifyLauncher, commits int, verify []string) (*verifyAwareRunner, *sandboxFakePR, SandboxKickResult, error) { + t.Helper() + runner := &verifyAwareRunner{sandboxFakeRunner{commits: commits}} + pr := &sandboxFakePR{} + exec := &SandboxExecutor{Runner: runner, Launcher: launcher, PRClient: pr, PushEnabled: true, Minter: sandboxFakeMinter{}} + res, err := exec.Run(context.Background(), SandboxKickSpec{ + Agent: "quality", AgentConfig: configSnapshot{Backend: "claude"}, Message: "add tests", + Org: "torch-spyre", Repo: "spyre-inference", WorkspaceDir: t.TempDir(), Image: "spyre-dev", + EnvAllowlist: []string{"HOME"}, NetworkMode: "restricted", PublishVerify: verify, + }) + return runner, pr, res, err +} + +func TestSandboxExecutorPublishVerifyPassesThenPublishes(t *testing.T) { + launcher := &sandboxVerifyLauncher{results: map[string]sandbox.Result{ + "uv run pytest --collect-only -q": {Stdout: "120 tests collected\n"}, + "run-changed": {Stdout: "17 passed\n"}, + }} + runner, pr, res, err := runVerifyKick(t, launcher, 1, []string{"uv run pytest --collect-only -q", "run-changed"}) + if err != nil { + t.Fatalf("Run: %v", err) + } + if !runner.pushed || !pr.called { + t.Fatalf("pushed=%v pr.called=%v, want both true after verify passed", runner.pushed, pr.called) + } + if res.PRWithheld != "" { + t.Fatalf("PRWithheld = %q", res.PRWithheld) + } + if len(res.Verify) != 2 || !res.Verify[0].Passed() || !res.Verify[1].Passed() { + t.Fatalf("Verify = %+v, want two passed results", res.Verify) + } + specs := launcher.verifySpecs() + if len(specs) != 2 { + t.Fatalf("verify launches = %d, want 2", len(specs)) + } + got := specs[0] + if got.Image != "spyre-dev" || got.Workspace != res.Workspace || got.NetworkMode != "restricted" || !reflect.DeepEqual(got.EnvAllowlist, []string{"HOME"}) { + t.Errorf("verify launch did not reuse the run's image/workspace/network/env: %+v", got) + } + if !reflect.DeepEqual(got.Command, []string{"sh", "-lc", "uv run pytest --collect-only -q"}) { + t.Errorf("Command = %v", got.Command) + } + if got.Env["HIVE_CHANGED_FILES"] != "src/thing.py\ntests/custom_ops/test_utils.py" { + t.Errorf("HIVE_CHANGED_FILES = %q", got.Env["HIVE_CHANGED_FILES"]) + } + if got.Env["HIVE_CHANGED_TESTS"] != "tests/custom_ops/test_utils.py" { + t.Errorf("HIVE_CHANGED_TESTS = %q", got.Env["HIVE_CHANGED_TESTS"]) + } + if got.Env["HIVE_BASE_SHA"] != "basesha" { + t.Errorf("HIVE_BASE_SHA = %q", got.Env["HIVE_BASE_SHA"]) + } +} + +func TestSandboxExecutorPublishVerifyFailureWithholdsPR(t *testing.T) { + const collect = "uv run pytest --collect-only -q" + launcher := &sandboxVerifyLauncher{results: map[string]sandbox.Result{ + collect: {Stdout: "ERROR collecting tests/multimodal/test_utils.py\nimport file mismatch\n", Stderr: "Interrupted: 1 error during collection\n", ExitCode: 2}, + }} + runner, pr, res, err := runVerifyKick(t, launcher, 1, []string{collect, "never-runs"}) + if !errors.Is(err, ErrSandboxRunReportedFailure) { + t.Fatalf("err = %v, want ErrSandboxRunReportedFailure", err) + } + if runner.pushed || pr.called { + t.Fatalf("pushed=%v pr.called=%v after verify failed", runner.pushed, pr.called) + } + for _, want := range []string{"verify failed (1/2): " + collect, "(exit 2)", "import file mismatch", "Interrupted: 1 error during collection"} { + if !strings.Contains(res.PRWithheld, want) { + t.Errorf("PRWithheld %q lacks %q", res.PRWithheld, want) + } + } + if !strings.Contains(err.Error(), "1 commit(s) not pushed, no PR opened") { + t.Errorf("err %q does not say commits were withheld", err) + } + if len(res.Verify) != 1 || res.Verify[0].Passed() { + t.Fatalf("Verify = %+v, want exactly one failed result (later commands skipped)", res.Verify) + } + if len(launcher.verifySpecs()) != 1 { + t.Errorf("verify launches = %d, want 1 (stop at first failure)", len(launcher.verifySpecs())) + } + body := sandboxKickLogBody("job", res, err) + if !strings.Contains(body, "---- verify: "+collect+" (exit 2) ----") || !strings.Contains(body, "import file mismatch") { + t.Errorf("kick log lacks verify section:\n%s", body) + } + if !strings.Contains(body, "pr_withheld: verify failed") { + t.Errorf("kick log lacks pr_withheld header:\n%s", body) + } +} + +func TestSandboxExecutorPublishVerifyLauncherErrorWithholdsPR(t *testing.T) { + launcher := &sandboxVerifyLauncher{errFor: "boom"} + runner, pr, res, err := runVerifyKick(t, launcher, 1, []string{"boom"}) + if !errors.Is(err, ErrSandboxRunReportedFailure) || runner.pushed || pr.called { + t.Fatalf("err=%v pushed=%v pr=%v", err, runner.pushed, pr.called) + } + if !strings.Contains(res.PRWithheld, "launcher exploded") { + t.Errorf("PRWithheld = %q", res.PRWithheld) + } +} + +func TestSandboxExecutorPublishVerifySkippedWithoutCommits(t *testing.T) { + launcher := &sandboxVerifyLauncher{results: map[string]sandbox.Result{"x": {ExitCode: 1}}} + runner, pr, res, err := runVerifyKick(t, launcher, 0, []string{"x"}) + if err != nil || runner.pushed || pr.called { + t.Fatalf("err=%v pushed=%v pr=%v, want clean no-op", err, runner.pushed, pr.called) + } + if len(res.Verify) != 0 || len(launcher.verifySpecs()) != 0 { + t.Fatalf("verify ran with nothing to publish: %+v", res.Verify) + } +} + +func TestSandboxExecutorPublishVerifyEmptyListPublishes(t *testing.T) { + launcher := &sandboxVerifyLauncher{} + runner, pr, _, err := runVerifyKick(t, launcher, 1, nil) + if err != nil || !runner.pushed || !pr.called { + t.Fatalf("err=%v pushed=%v pr=%v, want publish with no preflight configured", err, runner.pushed, pr.called) + } + if len(launcher.verifySpecs()) != 0 { + t.Fatalf("verify launched without commands") + } +} + +func TestParsePublishVerifyDirective(t *testing.T) { + msg := "# Quality lane\n\nRun tests on card first.\n\n\n\nThen open a PR." + cmds, ok := parsePublishVerifyDirective(msg) + if !ok { + t.Fatal("directive not found") + } + want := []string{ + "uv run --no-sync pytest --collect-only -q", + "sh -c '[ -z \"$HIVE_CHANGED_TESTS\" ] || uv run --no-sync pytest $HIVE_CHANGED_TESTS -q'", + } + if !reflect.DeepEqual(cmds, want) { + t.Fatalf("cmds = %q, want %q", cmds, want) + } + if cmds, ok := parsePublishVerifyDirective("no directive here"); ok || cmds != nil { + t.Fatalf("absent directive: %q %v", cmds, ok) + } + if cmds, ok := parsePublishVerifyDirective(""); !ok || len(cmds) != 0 { + t.Fatalf("empty directive should be present with no commands: %q %v", cmds, ok) + } +} + +func TestChangedTestFiles(t *testing.T) { + got := changedTestFiles([]string{"src/a.py", "tests/test_a.py", "pkg/b_test.go", "web/c.spec.ts", "tests/conftest.py", "tests/helpers_test.py"}) + want := []string{"tests/test_a.py", "pkg/b_test.go", "web/c.spec.ts", "tests/helpers_test.py"} + if !reflect.DeepEqual(got, want) { + t.Fatalf("got %q want %q", got, want) + } +} + +// recordingLauncher keeps every LaunchSpec so a manager-level test can see +// the verify launches the executor made after the agent run. +type recordingLauncher struct { + mu sync.Mutex + specs []sandbox.LaunchSpec +} + +func (l *recordingLauncher) Run(_ context.Context, spec sandbox.LaunchSpec) (sandbox.Result, error) { + l.mu.Lock() + defer l.mu.Unlock() + l.specs = append(l.specs, spec) + return sandbox.Result{Stdout: "ok"}, nil +} + +func (l *recordingLauncher) verifyCommands() []string { + l.mu.Lock() + defer l.mu.Unlock() + var out []string + for _, s := range l.specs { + if strings.HasSuffix(s.Name, "-verify") && len(s.Command) == 3 { + out = append(out, s.Command[2]) + } + } + return out +} + +func verifyManager(t *testing.T, override *config.AgentSandboxOverride, global config.AgentSandboxConfig) (*Manager, *recordingLauncher) { + t.Helper() + cfg := map[string]config.AgentConfig{"quality": {Backend: "claude", Sandbox: override}} + m := NewManager(cfg, quietTestLogger(), ProjectContext{Org: "torch-spyre", Repos: []string{"spyre-inference"}}) + global.WorkspaceDir = t.TempDir() + m.SetSandboxConfig(global) + rec := &recordingLauncher{} + m.setSandboxJobLauncherFactoryForTest(func(config.SandboxJobConfig) sandbox.Launcher { return rec }) + m.setSandboxRunnerForTest(&sandboxFakeRunner{commits: 1}) + if err := m.Start(context.Background(), "quality"); err != nil { + t.Fatalf("Start: %v", err) + } + return m, rec +} + +func TestManagerSandboxKickRunsConfiguredPublishVerify(t *testing.T) { + yes := true + m, rec := verifyManager(t, + &config.AgentSandboxOverride{Enabled: &yes, Runtime: config.SandboxRuntimeJob}, + config.AgentSandboxConfig{Enabled: true, Image: "dev-image", + Job: &config.SandboxJobConfig{WorkspaceClaim: "hive-data"}, + Publish: &config.SandboxPublishConfig{Verify: []string{"make check"}}}) + if err := m.SendKick("quality", "add tests"); err != nil { + t.Fatalf("SendKick: %v", err) + } + waitIdle(t, m, "quality") + if got := rec.verifyCommands(); !reflect.DeepEqual(got, []string{"make check"}) { + t.Fatalf("verify commands = %q, want [make check]", got) + } +} + +func TestManagerSandboxKickTemplateDirectiveOverridesConfiguredVerify(t *testing.T) { + yes := true + m, rec := verifyManager(t, + &config.AgentSandboxOverride{Enabled: &yes, Runtime: config.SandboxRuntimeJob, + Publish: &config.SandboxPublishConfig{Verify: []string{"make check"}}}, + config.AgentSandboxConfig{Enabled: true, Image: "dev-image", + Job: &config.SandboxJobConfig{WorkspaceClaim: "hive-data"}}) + msg := "add tests\n\n" + if err := m.SendKick("quality", msg); err != nil { + t.Fatalf("SendKick: %v", err) + } + waitIdle(t, m, "quality") + if got := rec.verifyCommands(); !reflect.DeepEqual(got, []string{"uv run pytest --collect-only -q"}) { + t.Fatalf("verify commands = %q, want the template's", got) + } +} diff --git a/src/crux/config/agent_config.go b/src/crux/config/agent_config.go index 5e2c11a94f..ccc6282fbe 100644 --- a/src/crux/config/agent_config.go +++ b/src/crux/config/agent_config.go @@ -488,8 +488,39 @@ type AgentSandboxConfig struct { WorkspaceDir string `yaml:"workspace_dir,omitempty" json:"workspace_dir,omitempty"` Runtime string `yaml:"runtime,omitempty" json:"runtime,omitempty"` Job *SandboxJobConfig `yaml:"job,omitempty" json:"job,omitempty"` + // Publish gates what a sandboxed run may publish; see SandboxPublishConfig. + Publish *SandboxPublishConfig `yaml:"publish,omitempty" json:"publish,omitempty"` } +// SandboxPublishConfig is the deterministic preflight hive runs before it +// pushes a sandboxed run's commits and opens a PR. The agent's own report is +// necessary but trusts the agent; these commands do not. Each entry is run +// with `sh -lc` in a fresh container/Job of the same image, workspace and +// env allowlist as the agent's run, with HIVE_CHANGED_FILES and +// HIVE_CHANGED_TESTS exported (newline-separated paths from the run's +// commits). A non-zero exit from any command withholds the push and the PR +// and puts the command's output tail on the agent card. +// +// A kick template can declare the same list inline so the policy that asks +// for a card-proven result also says how hive checks it: +// +// +// +// A template declaration replaces the configured list for that kick. +type SandboxPublishConfig struct { + Verify []string `yaml:"verify,omitempty" json:"verify,omitempty"` + // VerifyTimeoutS bounds each verify command. Non-positive uses the + // executor default (DefaultSandboxVerifyTimeoutS). + VerifyTimeoutS int `yaml:"verify_timeout_s,omitempty" json:"verify_timeout_s,omitempty"` +} + +// DefaultSandboxVerifyTimeoutS bounds one publish.verify command when no +// timeout is configured: long enough for a dependency-heavy collect-only or a +// single test file, short enough that a hung verify cannot pin the agent. +const DefaultSandboxVerifyTimeoutS = 900 + // AgentSandboxOverride is the per-agent sandbox opt-in block. type AgentSandboxOverride struct { Enabled *bool `yaml:"enabled,omitempty" json:"enabled,omitempty"` @@ -499,6 +530,9 @@ type AgentSandboxOverride struct { TimeoutS int `yaml:"timeout_s,omitempty" json:"timeout_s,omitempty"` Runtime string `yaml:"runtime,omitempty" json:"runtime,omitempty"` Job *SandboxJobConfig `yaml:"job,omitempty" json:"job,omitempty"` + // Publish, when set, replaces the global publish preflight for this + // agent; an empty verify list here turns the preflight off for it. + Publish *SandboxPublishConfig `yaml:"publish,omitempty" json:"publish,omitempty"` } // BobConfig configures the IBM bobshell ("bob") CLI backend. Only the diff --git a/src/crux/config/agent_runtime.go b/src/crux/config/agent_runtime.go index c40d59c927..fa003a8b84 100644 --- a/src/crux/config/agent_runtime.go +++ b/src/crux/config/agent_runtime.go @@ -561,6 +561,31 @@ func (a *AgentConfig) SandboxTimeoutS(global AgentSandboxConfig) int { return global.TimeoutS } +// SandboxPublishVerify returns the publish preflight commands for the agent: +// the per-agent publish block when present (even if its list is empty, which +// disables the preflight for that agent), else the global one. +func (a *AgentConfig) SandboxPublishVerify(global AgentSandboxConfig) []string { + if a != nil && a.Sandbox != nil && a.Sandbox.Publish != nil { + return append([]string(nil), a.Sandbox.Publish.Verify...) + } + if global.Publish != nil { + return append([]string(nil), global.Publish.Verify...) + } + return nil +} + +// SandboxPublishVerifyTimeoutS returns the per-command verify timeout: the +// per-agent value, then the global one, then DefaultSandboxVerifyTimeoutS. +func (a *AgentConfig) SandboxPublishVerifyTimeoutS(global AgentSandboxConfig) int { + if a != nil && a.Sandbox != nil && a.Sandbox.Publish != nil && a.Sandbox.Publish.VerifyTimeoutS > 0 { + return a.Sandbox.Publish.VerifyTimeoutS + } + if global.Publish != nil && global.Publish.VerifyTimeoutS > 0 { + return global.Publish.VerifyTimeoutS + } + return DefaultSandboxVerifyTimeoutS +} + const DefaultBusyNoActivityThreshold = 30 * time.Minute // EffectiveBusyNoActivityThreshold returns the silence threshold for surfacing diff --git a/src/crux/config/sandbox_publish_test.go b/src/crux/config/sandbox_publish_test.go new file mode 100644 index 0000000000..3c00b9900e --- /dev/null +++ b/src/crux/config/sandbox_publish_test.go @@ -0,0 +1,35 @@ +package config + +import ( + "reflect" + "testing" +) + +func TestSandboxPublishVerifyPrecedence(t *testing.T) { + global := AgentSandboxConfig{Publish: &SandboxPublishConfig{Verify: []string{"make check"}, VerifyTimeoutS: 120}} + var none *AgentConfig + if got := none.SandboxPublishVerify(global); !reflect.DeepEqual(got, []string{"make check"}) { + t.Fatalf("nil agent: %q", got) + } + if got := none.SandboxPublishVerifyTimeoutS(global); got != 120 { + t.Fatalf("nil agent timeout = %d", got) + } + plain := &AgentConfig{Sandbox: &AgentSandboxOverride{}} + if got := plain.SandboxPublishVerify(global); !reflect.DeepEqual(got, []string{"make check"}) { + t.Fatalf("no publish override falls through to global: %q", got) + } + own := &AgentConfig{Sandbox: &AgentSandboxOverride{Publish: &SandboxPublishConfig{Verify: []string{"pytest --collect-only"}, VerifyTimeoutS: 30}}} + if got := own.SandboxPublishVerify(global); !reflect.DeepEqual(got, []string{"pytest --collect-only"}) { + t.Fatalf("override: %q", got) + } + if got := own.SandboxPublishVerifyTimeoutS(global); got != 30 { + t.Fatalf("override timeout = %d", got) + } + off := &AgentConfig{Sandbox: &AgentSandboxOverride{Publish: &SandboxPublishConfig{}}} + if got := off.SandboxPublishVerify(global); len(got) != 0 { + t.Fatalf("empty override should disable the preflight: %q", got) + } + if got := off.SandboxPublishVerifyTimeoutS(AgentSandboxConfig{}); got != DefaultSandboxVerifyTimeoutS { + t.Fatalf("default timeout = %d", got) + } +} diff --git a/src/docs/agent-logging.md b/src/docs/agent-logging.md index 9bb572e92e..db0ecd99b7 100644 --- a/src/docs/agent-logging.md +++ b/src/docs/agent-logging.md @@ -285,7 +285,8 @@ read the logs of the last several runs — not just the latest — `-sandbox.log` with `source: sandbox-podman` or `source: sandbox-job` and an outcome block (exit code, commits, branch, PR or `pr_withheld` reason, agent-report path and status, error) ahead of - the output. For these agents the `📄 full log` / `📋 log text` controls and + the output, followed by one `---- verify: () ----` section + per `publish.verify` command that ran. For these agents the `📄 full log` / `📋 log text` controls and `GET /api/agents/{name}/log` serve the newest such archive — the only durable copy, since a completed Job is deleted at once and the sandbox workspace is swept after two hours — and an agent that has not run yet diff --git a/src/docs/dashboard-sections.md b/src/docs/dashboard-sections.md index 7ae4480db8..f8e2ce2bbb 100644 --- a/src/docs/dashboard-sections.md +++ b/src/docs/dashboard-sections.md @@ -628,7 +628,7 @@ More about the line: **The log buttons.** `📄 full log` opens the worker's whole log and `📋 log text` copies it; `🕘 past kicks` lists the logs of earlier rounds. For a normal worker this is the terminal it is typing in. For a worker that runs each round in a sandbox or a Kubernetes Job (its card says so), there is no terminal: the buttons show the output Hive captured from the sandbox when the last round ended, and `past kicks` holds the earlier rounds. Before such a worker's first round has ended the button says there is no archived run yet. -**Sandboxed worker's last run.** A sandboxed worker's card also carries a **last run** line: when it ran, how long it took, whether it succeeded, and the pull request it opened. If the worker's own report said the run failed, or the sandbox exited with an error, Hive keeps any commits the run made off GitHub and the line reads `no PR opened: ` — for example `no PR opened: agent report status=failed: RPM install returned 401`. The full reason is in the line's tooltip and in the full log's outcome block. +**Sandboxed worker's last run.** A sandboxed worker's card also carries a **last run** line: when it ran, how long it took, whether it succeeded, and the pull request it opened. If the worker's own report said the run failed, or the sandbox exited with an error, Hive keeps any commits the run made off GitHub and the line reads `no PR opened: ` — for example `no PR opened: agent report status=failed: RPM install returned 401`. The same line appears when a configured publish preflight (`publish.verify`, for example a whole-suite `pytest --collect-only`) failed after the run: `no PR opened: verify failed (1/2): uv run pytest --collect-only -q (exit 2)`, with the command's output tail in the tooltip. The full reason is in the line's tooltip and in the full log's outcome block, where each verify command has its own `---- verify: … ----` section. **Example.** A card says "needs login". You click **Login** and sign in again. Another card says `Now: working — no issue or pull request yet · started 50 min ago`. That worker has changed nothing through the hive for 50 minutes, so you open its log to see what it is doing. diff --git a/src/docs/sandbox-isolation.md b/src/docs/sandbox-isolation.md index 5e8083d26a..8bdb98faf7 100644 --- a/src/docs/sandbox-isolation.md +++ b/src/docs/sandbox-isolation.md @@ -110,6 +110,10 @@ agent_sandbox: # Use "none" for non-inference/test runs that require full network isolation. network_mode: restricted timeout_s: 2700 + # Optional deterministic preflight before any push/PR; see the publish + # preflight note under "kick path" below. + publish: + verify: [] agents: scanner: sandbox: @@ -129,6 +133,20 @@ For sandboxed agents, a kick now follows this path: 3. The manager marks the agent busy while the sandbox runs and returns it to idle or failed when the timeout/completion path finishes. Because the agent sits in `idle` between kicks, dashboard health for sandboxed agents comes from the last-run record rather than the session check; see [Health and metrics](#health-and-metrics-for-sandboxed-agents). 4. Hive collects the transcript at `.hive/sandbox-transcript.log` and any `agent-report*.json` artifact following `pkg/outputschema` conventions. The container's captured stdout/stderr is also archived as the kick's log under the kick-log directory (`//-sandbox.log`, same retention as tmux kick logs — see [Agent logging](agent-logging.md#per-kick-log-history)). That archive, not the workspace transcript, is what the card's "full log" / "log text" buttons and the 🕘 kick history serve for a sandboxed agent, because the workspace is swept after two hours and a Job's pod is gone as soon as it completes. **Publish gate.** Commits alone do not earn a PR. If the agent's report carries `"status": "failed"` or `"status": "blocked"`, or the sandbox exited non-zero, hive leaves the branch local, opens no PR, records the run as failed with the reason in `prWithheld` (shown on the card as "no PR opened: …"), and writes a `sandbox_pr_withheld` audit line naming the workspace, commit count and branch. The `status` field is optional; a report without one, or no report at all, does not block publishing, so agents that predate the field behave as before. Kick templates for sandboxed agents whose work must build or pass a smoke test before it is worth a PR should tell the agent to write `"status": "failed"` (with the failure in `summary`) instead of committing when that step fails. + **Publish preflight (`publish.verify`).** The report gate trusts the agent; the preflight does not. When the run left commits and the gate above passed, hive runs each configured verify command — `sandbox.publish.verify` per agent, `agent_sandbox.publish.verify` globally, or a `` HTML comment in the kick template (one shell command per line; a template block replaces the configured list for that kick) — as `sh -lc ` in a fresh container/Job of the same image, workspace, env allowlist and network mode, with `HIVE_CHANGED_FILES` and `HIVE_CHANGED_TESTS` (newline-separated paths from the run's commits, base..HEAD) and `HIVE_BASE_SHA` exported. The first non-zero exit, launcher error or timeout (`publish.verify_timeout_s`, default 900 s per command) withholds the push and the PR exactly like a failed report: `prWithheld` reads `verify failed (1/2): (exit N)` followed by the command's output tail, the card shows the first line with the tail in its tooltip, and the kick log archive gets a `---- verify: (exit N) ----` section with the full output. With no commits there is nothing to publish and the preflight is skipped; with no commands configured it is a no-op. Typical use is a whole-suite `pytest --collect-only -q` plus running the tests the agent added — the two checks that would have caught a test-file basename clash that broke collection for an entire repository ([torch-spyre/spyre-inference#1412](https://github.com/torch-spyre/spyre-inference/pull/1412)): + + ```yaml + agents: + quality: + sandbox: + enabled: true + runtime: job + publish: + verify: + - uv run --no-sync pytest --collect-only -q + - sh -c '[ -z "$HIVE_CHANGED_TESTS" ] || uv run --no-sync pytest $HIVE_CHANGED_TESTS -q' + verify_timeout_s: 900 + ``` 5. If the sandbox produced commits, `pkg/pushbroker.Broker` scans the committed diff for token-like secrets and protected-path edits, mints a short-lived scoped GitHub App token outside the sandbox, pushes the branch, and opens a PR through the existing App-authored GitHub client. Broker rejection records audit detail and nothing is pushed. The PR title and body come from `.hive/pr-title.txt` (first line) and `.hive/pr-body.md` if the agent wrote them in the workspace, otherwise from the HEAD commit's subject and body; a generic `[] sandbox changes` placeholder is used only when neither exists. Kick templates for sandboxed agents should ask the agent to write those two files. diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html index 0e3218465e..1a82a6a828 100644 --- a/src/pkg/dashboard/static/index.html +++ b/src/pkg/dashboard/static/index.html @@ -18029,7 +18029,10 @@

All labels

let html = esc(text); if (sr.prURL) html += ' \u00b7 PR'; if (sr.prWithheld) { - html += ' \u00b7 no PR opened: ' + esc(sr.prWithheld.length > 120 ? sr.prWithheld.slice(0, 120) + '\u2026' : sr.prWithheld) + ''; + // First line is the reason (verify failed / report status); the + // command's output tail follows on later lines and stays in the tooltip. + var reason = sr.prWithheld.split('\n')[0]; + html += ' \u00b7 no PR opened: ' + esc(reason.length > 120 ? reason.slice(0, 120) + '\u2026' : reason) + ''; } else if (sr.error && !sr.succeeded) { html += ' \u2014 ' + esc(sr.error.length > 160 ? sr.error.slice(0, 160) + '\u2026' : sr.error); } From 1bf7628dedb9f686e0a11f3534046ddce570079c Mon Sep 17 00:00:00 2001 From: Andrew Anderson Date: Wed, 28 Oct 2026 12:42:03 -0400 Subject: [PATCH 2/2] =?UTF-8?q?=F0=9F=8C=B1=20chore(dashboard):=20allowlis?= =?UTF-8?q?t=20sandbox=20publish.verify=20in=20settings=20parity?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The knob is a policy-file setting (template directive or hive.yaml) with no Settings control yet; track the control under #14999. Signed-off-by: Andrew Anderson --- src/pkg/dashboard/settings_parity_allowlist.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/pkg/dashboard/settings_parity_allowlist.txt b/src/pkg/dashboard/settings_parity_allowlist.txt index b8da760dd5..520aaa2fd5 100644 --- a/src/pkg/dashboard/settings_parity_allowlist.txt +++ b/src/pkg/dashboard/settings_parity_allowlist.txt @@ -13,6 +13,8 @@ agent_sandbox.job.volumes[].claim # TODO #11854 agent_sandbox.job.volumes[].mount_path # TODO #11854 agent_sandbox.job.volumes[].name # TODO #11854 agent_sandbox.job.volumes[].read_only # TODO #11854 +agent_sandbox.publish.verify[] # TODO #14999 (policy-file knob; Settings control pending) +agent_sandbox.publish.verify_timeout_s # TODO #14999 (policy-file knob; Settings control pending) agents..acmm_levels[] # TODO #11854 agents..agent_spec # TODO #11854 agents..aliases[] # TODO #11854 @@ -115,6 +117,8 @@ agents..sandbox.job.workspace_claim_mount # TODO #11854 agents..sandbox.network_mode # TODO #11854 agents..sandbox.runtime # TODO #11854 agents..sandbox.timeout_s # TODO #11854 +agents..sandbox.publish.verify[] # TODO #14999 (policy-file knob; Settings control pending) +agents..sandbox.publish.verify_timeout_s # TODO #14999 (policy-file knob; Settings control pending) agents..session_recycle.enabled # TODO #11854 agents..session_recycle.max_age # TODO #11854 agents..session_recycle.max_rss_mb # TODO #11854