Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/fixed-sandbox-job-full-log.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- Sandboxed agents (`sandbox.enabled: true`, Podman or `runtime: job`) no longer 404 on the card's `πŸ“„ full log` / `πŸ“‹ log text` buttons: hive archives the container's captured stdout/stderr as the kick log (`<kick_log_dir>/<agent>/<kick-start>-sandbox.log`, `source: sandbox-job`/`sandbox-podman`) when the run ends, serves the newest archive from `GET /api/agents/{name}/log`, lists every run in the πŸ•˜ kick history, and names the archive in the last-run record (`logId`); before its first run the button explains that no archived run exists yet. The output used to live only in the Job pod (deleted on completion) and the sandbox workspace (swept after two hours). Reported-by: @joerunde (a1-vllm-spyre spoke).
1 change: 1 addition & 0 deletions changelog.d/fixed-sandbox-pr-gate.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- Sandboxed kicks no longer push a branch and open a PR just because commits exist: `.hive/agent-report.json` gained an optional `status` (`success`, `failed`, `blocked`), and when the report says `failed`/`blocked` or the sandbox exited non-zero hive keeps the commits local, records the run as failed with the reason (`prWithheld`, shown on the card as "no PR opened: …"), and writes a `sandbox_pr_withheld` audit line instead of a PR that can only fail CI. Reports without a `status` publish as before. Reported-by: @joerunde (a1-vllm-spyre spoke).
6 changes: 5 additions & 1 deletion src/crux/agent/kick_logs.go
Original file line number Diff line number Diff line change
Expand Up @@ -274,7 +274,11 @@ func (m *Manager) FullLogHeader(name string) string {
m.mu.RLock()
defer m.mu.RUnlock()
agent, ok := m.agents[name]
if !ok || agent.tmuxSession == "" || !m.tmuxSessionExistsForAgent(agent) {
if !ok || agent.tmuxSession == "" {
return ""
}
// A sandboxed agent's full log is an archive that carries its own header.
if m.agentSandboxEnabledLocked(agent) || !m.tmuxSessionExistsForAgent(agent) {
return ""
}
return m.kickLogHeaderLocked(agent, "captured", time.Now(), "live", m.kickLogSourceLocked(agent))
Expand Down
14 changes: 14 additions & 0 deletions src/crux/agent/manager_sandbox.go
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
package agent

import (
"context"
"errors"
"fmt"
"path/filepath"
"strings"
Expand Down Expand Up @@ -207,8 +208,14 @@
// LIFO), so a slow volume never stalls every status reader.
var pending *SandboxRunHistoryEntry
var persist func(string, SandboxRunHistoryEntry) error
var logArchive *sandboxKickLogArchive
workDir := ""
defer func() {
// The log first: it is what an operator opens to understand the run
// record, and the record names it by id.
if perr := m.writeSandboxKickLog(name, logArchive); perr != nil {
m.logger.Warn("sandbox kick log not archived", "name", name, "error", perr)
}
if pending == nil {
return
}
Expand All @@ -229,6 +236,10 @@
// turn yesterday's successful run into "never ran".
if err == nil || !agent.sandboxResumeAfterCancel {
run := buildSandboxRun(runtime, started, ended, res, err)
logArchive = m.prepareSandboxKickLogLocked(agent, runtime, started, res, err)
if logArchive != nil {
run.LogID = logArchive.id
}
agent.SandboxLastRun = &run
agent.KickOutcome = KickOutcome{Kind: KickOutcomeEnded, At: ended, KickAt: started}
if agent.LastKick != nil {
Expand Down Expand Up @@ -262,6 +273,9 @@
if agent.OutputBuffer != nil {
agent.OutputBuffer.Write("sandbox kick failed: " + err.Error())
}
if errors.Is(err, ErrSandboxRunReportedFailure) {
m.auditSandbox(name, "sandbox_pr_withheld", fmt.Sprintf("workspace=%s commits=%d branch=%s reason=%s", res.Workspace, res.CommitCount, res.Branch, res.PRWithheld))
}
m.auditSandbox(name, "sandbox_failed", err.Error())
if res.Broker != nil && res.Broker.Error != "" {
m.auditSandbox(name, "sandbox_broker_rejected", res.Broker.Error)
Expand Down
25 changes: 25 additions & 0 deletions src/crux/agent/manager_tmux.go
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
package agent

import (
Expand Down Expand Up @@ -544,13 +544,23 @@
m.mu.RLock()
agent, ok := m.agents[name]
var target kickTranscriptTarget
var sandboxed bool
var logDir string
if ok {
target = m.kickTranscriptTargetLocked(agent)
sandboxed = m.agentSandboxEnabledLocked(agent)
logDir = m.agentKickLogDir(name)
}
m.mu.RUnlock()
if !ok {
return "", fmt.Errorf("agent %s not found", name)
}
// A sandboxed agent carries a session NAME like every agent but never a
// session: capturing it fails with "exit status 1", which the dashboard
// turned into a 404 on "full log" for every runtime: job agent.
if sandboxed {
return m.sandboxFullLog(name, logDir)
}
if agent.tmuxSession == "" {
return "", fmt.Errorf("agent %s has no active session", name)
}
Expand All @@ -560,6 +570,21 @@
return content, err
}

// sandboxFullLog is CaptureFullLog for a sandboxed agent: there is no pane,
// so the "full log" is the newest archived sandbox kick log β€” the container
// or Job output hive collected when the last run ended. A sandboxed agent
// with no run yet gets a message that says so rather than a pane error.
func (m *Manager) sandboxFullLog(name, logDir string) (string, error) {
id, content, err := latestKickLog(logDir)
if err != nil {
return "", fmt.Errorf("agent %s: reading archived sandbox run log: %w", name, err)
}
if id == "" {
return "", fmt.Errorf("agent %s runs sandboxed and has no archived run yet; the log appears here when its first kick ends", name)
}
return content, nil
}

// captureVisiblePaneForAgent captures only the visible pane (no scrollback).
func (m *Manager) captureVisiblePaneForAgent(agent *AgentProcess) string {
return m.terminalSession().CaptureVisiblePane(agent)
Expand Down
80 changes: 67 additions & 13 deletions src/crux/agent/sandbox_executor.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,9 @@ const (
sandboxPRBodyRelPath = ".hive/pr-body.md"
sandboxPRTitleMaxLen = 200
sandboxPRBodyMaxLen = 60000
// sandboxWithheldReasonMax bounds the report summary quoted in a
// publish-gate refusal, which lands in the run record and the audit log.
sandboxWithheldReasonMax = 300
)

type sandboxCommandRunner interface {
Expand Down Expand Up @@ -75,13 +78,21 @@ type configSnapshot struct {
}

type SandboxKickResult struct {
Workspace string
Branch string
BaseSHA string
HeadSHA string
CommitCount int
ReportPath string
Report *outputschema.AgentReport
Workspace string
Branch string
BaseSHA string
HeadSHA string
CommitCount int
ReportPath string
Report *outputschema.AgentReport
// ReportStatus is the agent's own verdict from the report's status field,
// read leniently so a report that fails schema validation can still say
// "failed" and keep its commits from being published.
ReportStatus outputschema.RunStatus
// PRWithheld names why commits the run left were NOT pushed or turned
// into a PR (the agent reported failure, or the sandbox exited non-zero).
// Empty when nothing was withheld.
PRWithheld string
TranscriptPath string
Sandbox sandbox.Result
Broker *pushbroker.Result
Expand Down Expand Up @@ -175,7 +186,7 @@ func (e *SandboxExecutor) Run(ctx context.Context, spec SandboxKickSpec) (Sandbo
}
res.TranscriptPath = filepath.Join(workspace, sandboxTranscriptRelPath)
_ = writeTranscript(res.TranscriptPath, sres)
res.ReportPath, res.Report = collectAgentReport(workspace)
res.ReportPath, res.Report, res.ReportStatus = collectAgentReport(workspace)
headSHA, count, commitErr := e.commitsSince(ctx, workspace, baseSHA)
res.HeadSHA, res.CommitCount = headSHA, count
if runErr != nil {
Expand All @@ -186,6 +197,20 @@ func (e *SandboxExecutor) Run(ctx context.Context, spec SandboxKickSpec) (Sandbo
res.Error = commitErr.Error()
return res, commitErr
}
// Publish gate: commits alone do not earn a PR. The agent's own verdict
// (report status failed/blocked) or a non-zero sandbox exit means the
// work was not proven, and a PR that only fails CI is worse than none.
if reason := sandboxPublishWithheld(res); reason != "" {
if count > 0 {
res.PRWithheld = reason
err := fmt.Errorf("%w: %s; %d commit(s) not pushed, no PR opened", ErrSandboxRunReportedFailure, reason, count)
res.Error = err.Error()
return res, err
}
err := fmt.Errorf("%w: %s", ErrSandboxRunReportedFailure, reason)
res.Error = err.Error()
return res, err
}
if count == 0 {
return res, nil
}
Expand Down Expand Up @@ -532,7 +557,34 @@ func writeTranscript(path string, res sandbox.Result) error {
return os.WriteFile(path, []byte(b.String()), 0o660)
}

func collectAgentReport(workspace string) (string, *outputschema.AgentReport) {
// ErrSandboxRunReportedFailure wraps every publish-gate refusal so callers
// can tell "the agent said its run failed" from an infrastructure error.
var ErrSandboxRunReportedFailure = errors.New("sandbox run reported failure")

// sandboxPublishWithheld returns why the run's commits must not be published,
// or "" when the gate is open. The order matters for the message an operator
// reads: the agent's own words first, the exit code second.
func sandboxPublishWithheld(res SandboxKickResult) string {
if res.ReportStatus.PublishBlocked() {
reason := "agent report status=" + string(res.ReportStatus)
if res.Report != nil && strings.TrimSpace(res.Report.Summary) != "" {
reason += ": " + truncateRunes(strings.TrimSpace(res.Report.Summary), sandboxWithheldReasonMax)
}
return reason
}
if res.Sandbox.ExitCode != 0 {
return fmt.Sprintf("sandbox exited %d", res.Sandbox.ExitCode)
}
return ""
}

// collectAgentReport finds the agent's report in the workspace. The third
// value is the report's status: from the validated report when it parses,
// else read straight from the JSON so an agent that wrote
// {"status":"failed"} with an otherwise malformed report still stops the
// publish gate β€” the one field that must survive a sloppy report is the one
// that says "do not ship this".
func collectAgentReport(workspace string) (string, *outputschema.AgentReport, outputschema.RunStatus) {
candidates := []string{
filepath.Join(workspace, ".hive", "agent-report.json"),
filepath.Join(workspace, "agent-report.json"),
Expand All @@ -555,14 +607,16 @@ func collectAgentReport(workspace string) (string, *outputschema.AgentReport) {
}
report, err := outputschema.Validate(data)
if err == nil {
return path, report
return path, report, report.Status
}
var raw struct {
Status string `json:"status"`
}
var raw map[string]any
if json.Unmarshal(data, &raw) == nil {
return path, nil
return path, nil, outputschema.RunStatus(strings.ToLower(strings.TrimSpace(raw.Status)))
}
}
return "", nil
return "", nil, ""
}

func (e *SandboxExecutor) launcher() sandbox.Launcher {
Expand Down
6 changes: 3 additions & 3 deletions src/crux/agent/sandbox_executor_paths_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -344,7 +344,7 @@ func TestCollectAgentReport_FindsPrefixedReportFile(t *testing.T) {
if err := os.WriteFile(path, []byte(valid), 0o660); err != nil {
t.Fatal(err)
}
gotPath, report := collectAgentReport(ws)
gotPath, report, _ := collectAgentReport(ws)
if gotPath != path {
t.Errorf("report path = %q, want %q", gotPath, path)
}
Expand All @@ -364,7 +364,7 @@ func TestCollectAgentReport_InvalidSchemaStillReturnsPath(t *testing.T) {
if err := os.WriteFile(filepath.Join(dir, "agent-report.json"), []byte(`{"unexpected":"shape"}`), 0o660); err != nil {
t.Fatal(err)
}
gotPath, report := collectAgentReport(ws)
gotPath, report, _ := collectAgentReport(ws)
if gotPath == "" {
t.Error("path must be surfaced for a JSON report that fails schema validation")
}
Expand All @@ -374,7 +374,7 @@ func TestCollectAgentReport_InvalidSchemaStillReturnsPath(t *testing.T) {
}

func TestCollectAgentReport_NothingFound(t *testing.T) {
gotPath, report := collectAgentReport(t.TempDir())
gotPath, report, _ := collectAgentReport(t.TempDir())
if gotPath != "" || report != nil {
t.Errorf("empty workspace: got (%q, %+v), want (\"\", nil)", gotPath, report)
}
Expand Down
100 changes: 100 additions & 0 deletions src/crux/agent/sandbox_executor_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package agent
import (
"context"
"errors"
"fmt"
"log/slog"
"os"
"path/filepath"
Expand All @@ -13,6 +14,7 @@ import (

"github.com/hivecommons/hive/crux"
"github.com/hivecommons/hive/crux/config"
"github.com/hivecommons/hive/crux/outputschema"
"github.com/hivecommons/hive/crux/pushbroker"
"github.com/hivecommons/hive/crux/sandbox"
)
Expand Down Expand Up @@ -456,3 +458,101 @@ func quietTestLogger() *slog.Logger {
}

var _ pushbroker.TokenMinter = sandboxFakeMinter{}

// sandboxReportLauncher lets a test pick the agent-report.json the sandbox
// leaves behind and the exit code the launcher reports, so the publish gate
// can be exercised on every input it reads.
type sandboxReportLauncher struct {
report string
exitCode int
}

func (l sandboxReportLauncher) Run(_ context.Context, spec sandbox.LaunchSpec) (sandbox.Result, error) {
if l.report != "" {
dir := filepath.Join(spec.Workspace, ".hive")
if err := os.MkdirAll(dir, 0o770); err != nil {
return sandbox.Result{}, err
}
if err := os.WriteFile(filepath.Join(dir, "agent-report.json"), []byte(l.report), 0o660); err != nil {
return sandbox.Result{}, err
}
}
return sandbox.Result{Stdout: "done\n", ExitCode: l.exitCode}, nil
}

// A commit is not a success signal. When the agent's report says the run
// failed (or the sandbox exited non-zero) hive must keep the branch local
// and open no PR β€” the a1 deps run that opened a PR after its RPM install
// 401'd is the case this guards against.
func TestSandboxExecutorPublishGate(t *testing.T) {
const okReport = `{"lane":"deps","kind":"fix","findings":[],"prs_opened":[],"beads_filed":[],"summary":"bumped"%s}`
tests := []struct {
name string
report string
exitCode int
commits int
wantPR bool
wantReason string
wantStatus outputschema.RunStatus
}{
{name: "success status publishes", report: fmt.Sprintf(okReport, `,"status":"success"`), commits: 1, wantPR: true, wantStatus: outputschema.StatusSuccess},
{name: "unspecified status publishes", report: fmt.Sprintf(okReport, ""), commits: 1, wantPR: true},
{name: "no report publishes", commits: 1, wantPR: true},
{name: "failed status withholds", report: fmt.Sprintf(okReport, `,"status":"failed"`), commits: 1, wantReason: "agent report status=failed: bumped", wantStatus: outputschema.StatusFailed},
{name: "blocked status withholds", report: fmt.Sprintf(okReport, `,"status":"blocked"`), commits: 1, wantReason: "agent report status=blocked", wantStatus: outputschema.StatusBlocked},
{name: "malformed report still carries status", report: `{"status":"failed","summary":"rpm install 401"}`, commits: 1, wantReason: "agent report status=failed", wantStatus: outputschema.StatusFailed},
{name: "non-zero exit withholds", report: fmt.Sprintf(okReport, ""), exitCode: 3, commits: 1, wantReason: "sandbox exited 3"},
{name: "failed status without commits reports failure", report: fmt.Sprintf(okReport, `,"status":"failed"`), wantReason: "agent report status=failed", wantStatus: outputschema.StatusFailed},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
runner := &sandboxFakeRunner{commits: tt.commits}
pr := &sandboxFakePR{}
exec := &SandboxExecutor{
Runner: runner,
Launcher: sandboxReportLauncher{report: tt.report, exitCode: tt.exitCode},
PRClient: pr,
PushEnabled: true,
Minter: sandboxFakeMinter{},
}
res, err := exec.Run(context.Background(), SandboxKickSpec{
Agent: "deps", AgentConfig: configSnapshot{Backend: "claude"}, Message: "bump",
Org: "kubestellar", Repo: "hive", WorkspaceDir: t.TempDir(), Image: "agent-image",
})
if res.ReportStatus != tt.wantStatus {
t.Errorf("ReportStatus = %q, want %q", res.ReportStatus, tt.wantStatus)
}
if tt.wantReason == "" {
if err != nil {
t.Fatalf("Run: %v", err)
}
if !runner.pushed || !pr.called {
t.Fatalf("pushed=%v pr.called=%v, want both true", runner.pushed, pr.called)
}
if res.PRWithheld != "" {
t.Fatalf("PRWithheld = %q, want empty", res.PRWithheld)
}
return
}
if !errors.Is(err, ErrSandboxRunReportedFailure) {
t.Fatalf("Run error = %v, want ErrSandboxRunReportedFailure", err)
}
if !strings.Contains(err.Error(), tt.wantReason) {
t.Errorf("error %q does not name reason %q", err, tt.wantReason)
}
if runner.pushed || pr.called {
t.Fatalf("pushed=%v pr.called=%v after a reported failure", runner.pushed, pr.called)
}
if tt.commits > 0 {
if !strings.Contains(res.PRWithheld, tt.wantReason) {
t.Errorf("PRWithheld = %q, want it to contain %q", res.PRWithheld, tt.wantReason)
}
if !strings.Contains(err.Error(), "1 commit(s) not pushed, no PR opened") {
t.Errorf("error %q should say the commits were not pushed", err)
}
} else if res.PRWithheld != "" {
t.Errorf("PRWithheld = %q with no commits, want empty", res.PRWithheld)
}
})
}
}
Loading
Loading