Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/added-sandbox-publish-verify.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- Sandboxed kicks gained a deterministic publish preflight that does not trust the agent's report: `sandbox.publish.verify` (per agent), `agent_sandbox.publish.verify` (global) or a `<!-- hive:publish.verify … -->` block in the kick template lists shell commands hive runs in the same sandbox image/workspace after the run (with `HIVE_CHANGED_FILES` / `HIVE_CHANGED_TESTS` exported), and the first non-zero exit withholds the push and the PR with the command's output tail on the card (`no PR opened: verify failed (1/2): uv run pytest --collect-only -q (exit 2) …`) and in the kick log. A whole-suite `pytest --collect-only` there would have stopped a test-gap PR whose new test file's basename clashed with an existing one and broke collection for every CI shard. Reported-by: @joerunde (a1-vllm-spyre spoke); hivecommons/hive#14999.
23 changes: 15 additions & 8 deletions src/crux/agent/manager_sandbox.go
Original file line number Diff line number Diff line change
Expand Up @@ -140,14 +140,21 @@ func (m *Manager) startSandboxKickLocked(agent *AgentProcess, message string) er
Model: agent.Config.Model,
LaunchCmd: agent.Config.LaunchCmd,
},
Message: message,
Org: m.project.Org,
Repo: repo,
WorkspaceDir: m.sandboxWorkspaceDirLocked(),
Image: agent.Config.SandboxImage(m.sandboxConfig),
EnvAllowlist: agent.Config.SandboxEnvAllowlist(m.sandboxConfig),
NetworkMode: agent.Config.SandboxNetworkMode(m.sandboxConfig),
Timeout: time.Duration(agent.Config.SandboxTimeoutS(m.sandboxConfig)) * time.Second,
Message: message,
Org: m.project.Org,
Repo: repo,
WorkspaceDir: m.sandboxWorkspaceDirLocked(),
Image: agent.Config.SandboxImage(m.sandboxConfig),
EnvAllowlist: agent.Config.SandboxEnvAllowlist(m.sandboxConfig),
NetworkMode: agent.Config.SandboxNetworkMode(m.sandboxConfig),
Timeout: time.Duration(agent.Config.SandboxTimeoutS(m.sandboxConfig)) * time.Second,
PublishVerify: agent.Config.SandboxPublishVerify(m.sandboxConfig),
VerifyTimeout: time.Duration(agent.Config.SandboxPublishVerifyTimeoutS(m.sandboxConfig)) * time.Second,
}
// A kick template's own verify block wins over config so the policy
// that asks for a proven result also says how hive checks it.
if cmds, ok := parsePublishVerifyDirective(message); ok {
spec.PublishVerify = cmds
}
runCtx, cancel := context.WithCancel(context.Background())
agent.cancel = cancel
Expand Down
154 changes: 153 additions & 1 deletion src/crux/agent/sandbox_executor.go
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
package agent

import (
Expand Down Expand Up @@ -69,6 +69,13 @@
EnvAllowlist []string
NetworkMode string
Timeout time.Duration
// PublishVerify are the deterministic preflight commands run in the
// sandbox before the run's commits are pushed (config
// sandbox.publish.verify or a `<!-- hive:publish.verify -->` block in the
// kick template). Any non-zero exit withholds the push and the PR.
PublishVerify []string
// VerifyTimeout bounds each PublishVerify command; zero uses the default.
VerifyTimeout time.Duration
}

type configSnapshot struct {
Expand All @@ -92,7 +99,10 @@
// PRWithheld names why commits the run left were NOT pushed or turned
// into a PR (the agent reported failure, or the sandbox exited non-zero).
// Empty when nothing was withheld.
PRWithheld string
PRWithheld string
// Verify records each publish.verify command that ran, in order. The
// first failure stops the list and withholds the PR.
Verify []SandboxVerifyResult
TranscriptPath string
Sandbox sandbox.Result
Broker *pushbroker.Result
Expand All @@ -101,6 +111,18 @@
Error string
}

// SandboxVerifyResult is one publish.verify command's outcome.
type SandboxVerifyResult struct {
Command string
ExitCode int
Output string
Err string
TimedOut bool
}

// Passed reports whether the command ran to completion with exit 0.
func (v SandboxVerifyResult) Passed() bool { return v.Err == "" && v.ExitCode == 0 && !v.TimedOut }

type SandboxExecutor struct {
Launcher sandbox.Launcher
Runner sandboxCommandRunner
Expand Down Expand Up @@ -214,6 +236,16 @@
if count == 0 {
return res, nil
}
// Deterministic preflight: the report gate above trusts the agent; the
// policy-declared verify commands do not. They run in the same sandbox
// image against the run's workspace, and a non-zero exit withholds the
// push and the PR with the command's tail on the card.
if reason := e.runPublishVerify(ctx, spec, workspace, baseSHA, &res); reason != "" {
res.PRWithheld = reason
err := fmt.Errorf("%w: %s; %d commit(s) not pushed, no PR opened", ErrSandboxRunReportedFailure, reason, count)
res.Error = err.Error()
return res, err
}
if !e.PushEnabled {
return res, nil
}
Expand Down Expand Up @@ -564,6 +596,126 @@
// sandboxPublishWithheld returns why the run's commits must not be published,
// or "" when the gate is open. The order matters for the message an operator
// reads: the agent's own words first, the exit code second.
const (
// sandboxVerifyTailMax bounds the verify output quoted on the card and in
// the audit record; the full output lands in the kick log archive.
sandboxVerifyTailMax = 600
// sandboxVerifyOutputMax bounds what one verify command's combined output
// keeps in memory and in the kick log.
sandboxVerifyOutputMax = 64 << 10
defaultSandboxVerifyTimeout = 15 * time.Minute
)

// runPublishVerify runs spec.PublishVerify in order and returns the
// withhold reason for the first failure, or "" when every command passed
// (or none were declared). Results are appended to res.Verify either way.
func (e *SandboxExecutor) runPublishVerify(ctx context.Context, spec SandboxKickSpec, workspace, baseSHA string, res *SandboxKickResult) string {
if len(spec.PublishVerify) == 0 {
return ""
}
changed := e.changedFiles(ctx, workspace, baseSHA)
env := map[string]string{
"HIVE_CHANGED_FILES": strings.Join(changed, "\n"),
"HIVE_CHANGED_TESTS": strings.Join(changedTestFiles(changed), "\n"),
"HIVE_BASE_SHA": baseSHA,
}
timeout := spec.VerifyTimeout
if timeout <= 0 {
timeout = defaultSandboxVerifyTimeout
}
for i, cmd := range spec.PublishVerify {
cmd = strings.TrimSpace(cmd)
if cmd == "" {
continue
}
vres := SandboxVerifyResult{Command: cmd}
vctx, cancel := context.WithTimeout(ctx, timeout)
sres, err := e.launcher().Run(vctx, sandbox.LaunchSpec{
Image: spec.Image,
Name: "hive-" + sanitizeBranchPart(spec.Agent) + "-verify",
Workspace: workspace,
WorkspaceMount: sandbox.DefaultWorkspaceMount,
WorkDir: sandbox.DefaultWorkspaceMount,
Command: []string{"sh", "-lc", cmd},
Env: env,
EnvAllowlist: spec.EnvAllowlist,
NetworkMode: spec.NetworkMode,
})
vres.TimedOut = errors.Is(vctx.Err(), context.DeadlineExceeded)
cancel()
out := logscrub.ScrubString(sres.Stdout, logscrub.WithMarkers())
if sres.Stderr != "" {
out += "\n" + logscrub.ScrubString(sres.Stderr, logscrub.WithMarkers())
}
vres.Output = tailRunes(strings.TrimSpace(out), sandboxVerifyOutputMax)
vres.ExitCode = sres.ExitCode
if err != nil {
vres.Err = err.Error()
}
res.Verify = append(res.Verify, vres)
if vres.Passed() {
continue
}
reason := fmt.Sprintf("verify failed (%d/%d): %s", i+1, len(spec.PublishVerify), cmd)
switch {
case vres.TimedOut:
reason += fmt.Sprintf(" timed out after %s", timeout)
case vres.Err != "":
reason += ": " + vres.Err
default:
reason += fmt.Sprintf(" (exit %d)", vres.ExitCode)
}
if tail := tailRunes(vres.Output, sandboxVerifyTailMax); tail != "" {
reason += "\n" + tail
}
return reason
}
return ""
}

// changedFiles lists paths touched by the run's commits (base..HEAD). An
// error yields nil so the verify commands still run, with empty env.
func (e *SandboxExecutor) changedFiles(ctx context.Context, workspace, baseSHA string) []string {
out, err := e.runner().Run(ctx, workspace, pushbroker.PushEnv(os.Environ()), "git", "diff", "--name-only", "--diff-filter=AMR", baseSHA, "HEAD")
if err != nil {
return nil
}
var files []string
for _, line := range strings.Split(string(out), "\n") {
if line = strings.TrimSpace(line); line != "" {
files = append(files, line)
}
}
return files
}

// changedTestFiles keeps the pytest/go/js-style test files out of a changed
// list so a verify command can run exactly the tests the run added.
func changedTestFiles(files []string) []string {
var tests []string
for _, f := range files {
base := filepath.Base(f)
switch {
case strings.HasPrefix(base, "test_") && strings.HasSuffix(base, ".py"),
strings.HasSuffix(base, "_test.py"),
strings.HasSuffix(base, "_test.go"),
strings.HasSuffix(base, ".test.js"), strings.HasSuffix(base, ".test.ts"),
strings.HasSuffix(base, ".spec.js"), strings.HasSuffix(base, ".spec.ts"):
tests = append(tests, f)
}
}
return tests
}

// tailRunes keeps the last max runes of s.
func tailRunes(s string, max int) string {
r := []rune(s)
if len(r) <= max {
return s
}
return "…" + string(r[len(r)-max:])
}

func sandboxPublishWithheld(res SandboxKickResult) string {
if res.ReportStatus.PublishBlocked() {
reason := "agent report status=" + string(res.ReportStatus)
Expand Down
16 changes: 16 additions & 0 deletions src/crux/agent/sandbox_kick_log.go
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
package agent

import (
Expand Down Expand Up @@ -60,6 +60,22 @@
if !strings.HasSuffix(res.Sandbox.Stderr, "\n") {
b.WriteString("\n")
}
for _, v := range res.Verify {
verdict := "passed"
switch {
case v.TimedOut:
verdict = "timed out"
case v.Err != "":
verdict = "error: " + v.Err
case v.ExitCode != 0:
verdict = fmt.Sprintf("exit %d", v.ExitCode)
}
fmt.Fprintf(&b, "\n---- verify: %s (%s) ----\n", v.Command, verdict)
b.WriteString(v.Output)
if !strings.HasSuffix(v.Output, "\n") {
b.WriteString("\n")
}
}
return b.String()
}

Expand Down
39 changes: 39 additions & 0 deletions src/crux/agent/sandbox_publish_verify.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
package agent

import (
"regexp"
"strings"
)

// publishVerifyDirective matches an HTML-comment block a kick template uses
// to declare its publish preflight inline:
//
// <!-- hive:publish.verify
// uv run pytest --collect-only -q
// sh -c '[ -z "$HIVE_CHANGED_TESTS" ] || uv run pytest $HIVE_CHANGED_TESTS -q'
// -->
//
// One shell command per line; blank lines and `#` comments are ignored. The
// comment is invisible to the agent-facing markdown yet travels with the
// policy, so the template that tells the agent "run on card first" is the
// same artifact that tells hive how to check it did.
var publishVerifyDirective = regexp.MustCompile(`(?s)<!--\s*hive:publish\.verify\s*\n(.*?)-->`)

// parsePublishVerifyDirective extracts the verify commands from a rendered
// kick message. ok is true when a directive is present, even if it lists no
// commands (an explicit empty block turns the configured preflight off for
// that kick).
func parsePublishVerifyDirective(message string) (cmds []string, ok bool) {
m := publishVerifyDirective.FindStringSubmatch(message)
if m == nil {
return nil, false
}
for _, line := range strings.Split(m[1], "\n") {
line = strings.TrimSpace(line)
if line == "" || strings.HasPrefix(line, "#") {
continue
}
cmds = append(cmds, line)
}
return cmds, true
}
Loading
Loading