diff --git a/changelog.d/added-crux-agent-steps.md b/changelog.d/added-crux-agent-steps.md new file mode 100644 index 0000000000..7a8609f047 --- /dev/null +++ b/changelog.d/added-crux-agent-steps.md @@ -0,0 +1 @@ +- crux: agent step service, the sixth work service of the Crux work-services RFC ([hivecommons/crux#114](https://github.com/hivecommons/crux/issues/114) S6). `POST /v1/runs` with `"profile": "agent"` now opens a step session — a contained run that provisions its checkout and waits — and `POST /v1/runs/{id}/steps` runs one agent step inside it: a `claude`, `codex` or `process` backend reads the prompt in the sandbox and its JSON answer is checked against the request's `output_schema`, with bounded re-asks on a mismatch, a per-attempt timeout, tool and turn limits where the backend can enforce them, and a recorded `unsupported` refusal where it cannot. Each step is logged in the S4 command log, announced as a `step` turn signal, and read back with `GET /v1/runs/{id}/steps[/{n}]` or `crux --server … step run|list|show`. diff --git a/src/crux/agentstep/agentstep.go b/src/crux/agentstep/agentstep.go new file mode 100644 index 0000000000..1a831e2da5 --- /dev/null +++ b/src/crux/agentstep/agentstep.go @@ -0,0 +1,334 @@ +// Package agentstep is the schema-checked agent step service of the Crux +// work services (S6 in hivecommons/crux#114): run one agent turn in a +// contained workspace — a prompt, a backend CLI, a model, an allowlist of +// tools, a turn limit — and hand back the answer already validated against +// the caller's JSON Schema, with the token usage, a transcript of every +// attempt and a typed status. The loop is agent.SandboxExecutor.Run +// generalised: write the prompt into the workspace, build the backend's +// command (Backend carries the per-CLI flags), run it through +// sandbox.Launcher, read the answer back, check it, and on a mismatch retry +// with a corrective prompt. +// +// Steps belong to an agent-profile contained run (runs.Service, S2), which +// supplies the workspace, the launcher, the command log (S4) and the signal +// book (S5); this package never looks those up itself. Records live under +// the run's directory so a caller recovers them by run id and step number +// after a disconnect (S7). +package agentstep + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "time" + + "github.com/hivecommons/hive/crux/api" + "github.com/hivecommons/hive/crux/cmdlog" + "github.com/hivecommons/hive/crux/logscrub" + "github.com/hivecommons/hive/crux/sandbox" +) + +// Limits. +const ( + // MaxRetries bounds retry_on_mismatch. + MaxRetries = 5 + // MaxPromptBytes bounds an inline prompt. + MaxPromptBytes = 1 << 20 + // workspaceDir is where prompts and backend answer files go inside the + // workspace, next to the agent sandbox's .hive. + workspaceDir = ".crux/steps" + // launchNamePrefix names the container after the run, step and attempt + // so a teardown can find it. + launchNamePrefix = "crux-step-" + // teardownTimeout bounds the container removal after a cancel or a + // timeout. + teardownTimeout = 30 * time.Second +) + +// Host is what the session hands the service for one step: where to run, +// how to launch, where to record. The runs service builds it from the +// agent-profile run the step is posted to. +type Host struct { + Run string + RunDir string + Workspace string + Image string + Network string + // Env and EnvAllowlist are the session's, already filtered of + // credentials by sandbox.SanitizedEnv. + Env map[string]string + EnvAllowlist []string + Launcher sandbox.Launcher + // Teardown, when set, removes the launch by name after a cancel or a + // timeout (runs.Backend.Teardown). + Teardown func(ctx context.Context, name string) error + // Timeout is the per-attempt default; MaxTimeout caps a request's. + Timeout, MaxTimeout time.Duration + // CmdLog, when set, records each attempt's command (S4). + CmdLog cmdlog.Recorder + // Signal, when set, appends a runner signal to the run (S5): the + // service emits one step signal per finished step. + Signal func(kind, message string, data json.RawMessage) + // Scrub redacts output before it is persisted; nil uses logscrub. + Scrub func(string) string + // Now is the clock; nil uses time.Now. + Now func() time.Time +} + +func (h Host) scrub(s string) string { + if h.Scrub != nil { + return h.Scrub(s) + } + return logscrub.ScrubString(s) +} + +func (h Host) now() time.Time { + if h.Now != nil { + return h.Now().UTC() + } + return time.Now().UTC() +} + +// plan is a validated request. +type plan struct { + backend Backend + schema *Schema + rawSch json.RawMessage + prompt string + spec Spec + timeout time.Duration + retries int +} + +// Run executes one step for host and returns its saved record. A request +// the backend cannot honour is saved with status unsupported and returned +// with an *UnsupportedError, so the refusal is on the record as the RFC +// asks; a malformed request is a *RequestError with no record. +func Run(ctx context.Context, host Host, req api.StepRequest) (api.Step, error) { + p, err := validate(host, req) + if err != nil { + return api.Step{}, err + } + store := Store{RunDir: host.RunDir} + // Build the command once with placeholder paths to learn whether the + // backend can honour the request before anything is recorded as run. + probe := p.spec + probe.PromptRel, probe.AnswerRel = "prompt", "answer" + if _, cmdErr := p.backend.Command(probe); cmdErr != nil { + var unsupported *UnsupportedError + if !errors.As(cmdErr, &unsupported) { + return api.Step{}, cmdErr + } + step, err := store.Create(api.Step{Run: host.Run, Backend: req.Backend, Model: req.Model, CreatedAt: host.now()}) + if err != nil { + return api.Step{}, err + } + step = finish(host, store, step, api.StepUnsupported, unsupported.What, nil) + return step, unsupported + } + step, err := store.Create(api.Step{Run: host.Run, Backend: req.Backend, Model: req.Model, Status: api.StepRunning, CreatedAt: host.now()}) + if err != nil { + return api.Step{}, err + } + step.TranscriptPath = store.TranscriptRel(step.N) + return execute(ctx, host, store, step, p), nil +} + +func validate(host Host, req api.StepRequest) (plan, error) { + var p plan + backend, err := Lookup(strings.TrimSpace(req.Backend)) + if err != nil { + return p, err + } + p.backend = backend + switch { + case req.Prompt != "" && req.PromptPath != "": + return p, &RequestError{"prompt and prompt_path are mutually exclusive"} + case req.Prompt != "": + if len(req.Prompt) > MaxPromptBytes { + return p, &RequestError{fmt.Sprintf("prompt is %d bytes; the bound is %d (use prompt_path)", len(req.Prompt), MaxPromptBytes)} + } + p.prompt = req.Prompt + case req.PromptPath != "": + rel := filepath.Clean(req.PromptPath) + if filepath.IsAbs(rel) || rel == ".." || strings.HasPrefix(rel, "../") { + return p, &RequestError{"prompt_path must be relative to the workspace and stay inside it"} + } + data, err := os.ReadFile(filepath.Join(host.Workspace, rel)) + if err != nil { + return p, &RequestError{fmt.Sprintf("prompt_path %q: %v", req.PromptPath, err)} + } + p.prompt = string(data) + default: + return p, &RequestError{"one of prompt or prompt_path is required"} + } + if strings.TrimSpace(p.prompt) == "" { + return p, &RequestError{"prompt is empty"} + } + p.rawSch = req.OutputSchema + if p.schema, err = ParseSchema(req.OutputSchema); err != nil { + return p, &RequestError{strings.TrimPrefix(err.Error(), "agentstep: invalid output_schema: ")} + } + if req.RetryOnMismatch < 0 || req.RetryOnMismatch > MaxRetries { + return p, &RequestError{fmt.Sprintf("retry_on_mismatch must be between 0 and %d", MaxRetries)} + } + p.retries = req.RetryOnMismatch + if req.TurnLimit < 0 { + return p, &RequestError{"turn_limit must not be negative"} + } + p.timeout = host.Timeout + if req.TimeoutMs > 0 { + d := time.Duration(req.TimeoutMs) * time.Millisecond + if host.MaxTimeout > 0 && d > host.MaxTimeout { + return p, &RequestError{fmt.Sprintf("timeout_ms %d exceeds this server's cap of %d", req.TimeoutMs, host.MaxTimeout.Milliseconds())} + } + p.timeout = d + } + for _, t := range req.Tools { + if strings.TrimSpace(t) == "" { + return p, &RequestError{"tools must not contain an empty name"} + } + } + p.spec = Spec{Model: req.Model, Tools: req.Tools, TurnLimit: req.TurnLimit, Command: req.Command} + return p, nil +} + +// attempt is one transcript line: what ran and what came back, scrubbed. +type attempt struct { + Attempt int `json:"attempt"` + At time.Time `json:"at"` + Command []string `json:"command"` + ExitCode *int `json:"exit_code,omitempty"` + Stdout string `json:"stdout,omitempty"` + Stderr string `json:"stderr,omitempty"` + Status string `json:"status,omitempty"` + Reason string `json:"reason,omitempty"` + Violations []api.SchemaViolation `json:"violations,omitempty"` + Usage api.StepUsage `json:"usage"` +} + +// execute runs the attempts to a terminal status. Every exit path writes +// the record and then the run's step signal. +func execute(ctx context.Context, host Host, store Store, step api.Step, p plan) api.Step { + prompt := p.prompt + var violations []api.SchemaViolation + for n := 1; n <= 1+p.retries; n++ { + step.Attempts = n + a, outcome := runAttempt(ctx, host, store, step, p, n, prompt) + step.Usage.InputTokens += outcome.Usage.InputTokens + step.Usage.OutputTokens += outcome.Usage.OutputTokens + if outcome.Status != "" { + a.Status, a.Reason = outcome.Status, outcome.Reason + _ = store.AppendTranscript(step.N, a) + return finish(host, store, step, outcome.Status, outcome.Reason, nil) + } + if outcome.Answer == nil { + violations = []api.SchemaViolation{{Path: "$", Message: ErrNoAnswer.Error()}} + } else { + violations = p.schema.Validate(outcome.Answer) + } + if len(violations) == 0 { + a.Status = api.StepSucceeded + _ = store.AppendTranscript(step.N, a) + step.Answer = outcome.Answer + return finish(host, store, step, api.StepSucceeded, "", nil) + } + a.Status, a.Violations = api.StepSchemaMismatch, violations + _ = store.AppendTranscript(step.N, a) + prompt = p.prompt + "\n\n" + CorrectivePrompt(p.rawSch, violations, p.retries-n) + } + return finish(host, store, step, api.StepSchemaMismatch, fmt.Sprintf("the answer did not match output_schema after %d attempt(s)", step.Attempts), violations) +} + +// runAttempt launches the backend once and reads its outcome. +func runAttempt(ctx context.Context, host Host, store Store, step api.Step, p plan, n int, prompt string) (attempt, Outcome) { + a := attempt{Attempt: n, At: host.now()} + rel := filepath.ToSlash(filepath.Join(workspaceDir, fmt.Sprint(step.N))) + spec := p.spec + spec.PromptRel = rel + fmt.Sprintf("/prompt-%d.md", n) + spec.AnswerRel = rel + fmt.Sprintf("/answer-%d.txt", n) + if err := os.MkdirAll(filepath.Join(host.Workspace, rel), 0o770); err != nil { + return a, Outcome{Status: api.StepBackendError, Reason: "writing the prompt: " + host.scrub(err.Error())} + } + if err := os.WriteFile(filepath.Join(host.Workspace, filepath.FromSlash(spec.PromptRel)), []byte(prompt), 0o660); err != nil { + return a, Outcome{Status: api.StepBackendError, Reason: "writing the prompt: " + host.scrub(err.Error())} + } + cmd, err := p.backend.Command(spec) + if err != nil { + return a, Outcome{Status: api.StepUnsupported, Reason: err.Error()} + } + a.Command = cmd + var seq int64 + if host.CmdLog != nil { + if seq, err = host.CmdLog.Started(cmdlog.Start{Command: cmd, Cwd: sandbox.DefaultWorkspaceMount, Capture: api.CommandCaptureRunner}); err != nil { + seq = 0 + } + } + runCtx, cancel := context.WithTimeout(ctx, p.timeout) + defer cancel() + name := fmt.Sprintf("%s%s-%d-%d", launchNamePrefix, host.Run, step.N, n) + res, runErr := host.Launcher.Run(runCtx, sandbox.LaunchSpec{ + Image: host.Image, Name: name, Workspace: host.Workspace, Command: cmd, + Env: host.Env, EnvAllowlist: host.EnvAllowlist, NetworkMode: host.Network, + }) + cancelled := runErr != nil && ctx.Err() != nil + timedOut := runErr != nil && !cancelled && errors.Is(runCtx.Err(), context.DeadlineExceeded) + if (cancelled || timedOut) && host.Teardown != nil { + tctx, tcancel := context.WithTimeout(context.Background(), teardownTimeout) + _ = host.Teardown(tctx, name) + tcancel() + } + a.Stdout, a.Stderr = host.scrub(res.Stdout), host.scrub(res.Stderr) + launched := runErr == nil || res != (sandbox.Result{}) + if launched && res.ExitCode != -1 { + code := res.ExitCode + a.ExitCode = &code + } + var out Outcome + switch { + case cancelled: + out = Outcome{Status: api.StepCancelled, Reason: "the session was cancelled"} + case timedOut: + out = Outcome{Status: api.StepTimeout, Reason: fmt.Sprintf("the backend did not finish within %s", p.timeout)} + case !launched: + out = Outcome{Status: api.StepBackendError, Reason: host.scrub(runErr.Error())} + default: + out = p.backend.Parse(spec, res, func(rel string) ([]byte, error) { + return os.ReadFile(filepath.Join(host.Workspace, filepath.FromSlash(rel))) + }) + out.Reason = host.scrub(out.Reason) + } + a.Usage = out.Usage + if host.CmdLog != nil && seq > 0 { + end := cmdlog.End{ExitCode: a.ExitCode, Stdout: res.Stdout, Stderr: res.Stderr, Reason: out.Reason} + switch out.Status { + case api.StepCancelled: + end.Status = api.CommandCancelled + case api.StepTimeout, api.StepBackendError: + end.Status = api.CommandFailed + } + _ = host.CmdLog.Ended(seq, end) + } + return a, out +} + +// finish writes the terminal record and announces it on the run. +func finish(host Host, store Store, step api.Step, status, reason string, violations []api.SchemaViolation) api.Step { + now := host.now() + step.Status, step.Reason, step.FinishedAt, step.Violations = status, reason, &now, violations + if status != api.StepSucceeded { + step.Answer = nil + } + _ = store.Write(step) + if host.Signal != nil { + data, err := json.Marshal(api.SignalStepData{Step: step.N, Status: status, Attempts: step.Attempts}) + if err == nil { + host.Signal(api.SignalStep, reason, data) + } + } + return step +} diff --git a/src/crux/agentstep/agentstep_test.go b/src/crux/agentstep/agentstep_test.go new file mode 100644 index 0000000000..e59bedd78f --- /dev/null +++ b/src/crux/agentstep/agentstep_test.go @@ -0,0 +1,364 @@ +package agentstep + +import ( + "context" + "encoding/json" + "errors" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/hivecommons/hive/crux/api" + "github.com/hivecommons/hive/crux/cmdlog" + "github.com/hivecommons/hive/crux/sandbox" +) + +// fakeLauncher answers each launch from a queue of results, recording the +// spec and the prompt the backend would have read. +type fakeLauncher struct { + mu sync.Mutex + specs []sandbox.LaunchSpec + prompts []string + results []func(ctx context.Context, spec sandbox.LaunchSpec) (sandbox.Result, error) +} + +func (f *fakeLauncher) Run(ctx context.Context, spec sandbox.LaunchSpec) (sandbox.Result, error) { + f.mu.Lock() + f.specs = append(f.specs, spec) + // The prompt path is the last word of the sh -lc line. + line := spec.Command[len(spec.Command)-1] + rel := strings.Trim(line[strings.LastIndex(line, " < ")+3:], "'") + data, _ := os.ReadFile(filepath.Join(spec.Workspace, rel)) + f.prompts = append(f.prompts, string(data)) + next := f.results[0] + if len(f.results) > 1 { + f.results = f.results[1:] + } + f.mu.Unlock() + return next(ctx, spec) +} + +func answers(outs ...string) *fakeLauncher { + f := &fakeLauncher{} + for _, o := range outs { + o := o + f.results = append(f.results, func(context.Context, sandbox.LaunchSpec) (sandbox.Result, error) { + return sandbox.Result{Stdout: o}, nil + }) + } + return f +} + +type recorded struct { + kind string + data api.SignalStepData +} + +type fixture struct { + t *testing.T + host Host + launcher *fakeLauncher + signals []recorded + torn []string + log cmdlog.Recorder +} + +func newFixture(t *testing.T, launcher *fakeLauncher) *fixture { + t.Helper() + f := &fixture{t: t, launcher: launcher} + runDir := t.TempDir() + log, err := cmdlog.FileOpener(cmdlog.Policy{})(runDir) + if err != nil { + t.Fatal(err) + } + f.log = log + f.host = Host{ + Run: "run_1", RunDir: runDir, Workspace: t.TempDir(), Image: "img:1", Network: "restricted", + Env: map[string]string{"KEEP": "yes"}, EnvAllowlist: []string{"KEEP"}, + Launcher: launcher, Timeout: time.Minute, MaxTimeout: time.Hour, CmdLog: log, + Teardown: func(_ context.Context, name string) error { f.torn = append(f.torn, name); return nil }, + Signal: func(kind, _ string, data json.RawMessage) { + var d api.SignalStepData + _ = json.Unmarshal(data, &d) + f.signals = append(f.signals, recorded{kind, d}) + }, + } + return f +} + +const okSchema = `{"type":"object","required":["verdict"],"properties":{"verdict":{"enum":["pass","fail"]}}}` + +func (f *fixture) process(prompt string, schema string) api.StepRequest { + return api.StepRequest{Backend: "process", Prompt: prompt, OutputSchema: json.RawMessage(schema), Command: []string{"./ask"}} +} + +func (f *fixture) run(req api.StepRequest) api.Step { + f.t.Helper() + step, err := Run(context.Background(), f.host, req) + if err != nil { + f.t.Fatalf("Run: %v", err) + } + return step +} + +func (f *fixture) commands() []api.CommandRecord { + f.t.Helper() + page, err := cmdlog.Read(f.host.RunDir, 0, 0) + if err != nil { + f.t.Fatal(err) + } + return page.Commands +} + +func TestSucceededStepIsRecordedLoggedAndSignalled(t *testing.T) { + f := newFixture(t, answers(`{"verdict":"pass"}`)) + step := f.run(f.process("Is it fine?", okSchema)) + if step.N != 1 || step.Status != api.StepSucceeded || step.Attempts != 1 || string(step.Answer) != `{"verdict":"pass"}` || step.FinishedAt == nil || step.Backend != "process" { + t.Fatalf("step = %+v", step) + } + spec := f.launcher.specs[0] + if spec.Name != "crux-step-run_1-1-1" || spec.Image != "img:1" || spec.NetworkMode != "restricted" || spec.Env["KEEP"] != "yes" || spec.Workspace != f.host.Workspace { + t.Fatalf("spec = %+v", spec) + } + if f.launcher.prompts[0] != "Is it fine?" { + t.Fatalf("first prompt = %q, want the caller's verbatim", f.launcher.prompts[0]) + } + got, err := Store{RunDir: f.host.RunDir}.Get(1) + if err != nil || got.Status != api.StepSucceeded || got.TranscriptPath != "steps/1/transcript.jsonl" { + t.Fatalf("Get = %+v, %v", got, err) + } + transcript, _ := os.ReadFile(filepath.Join(f.host.RunDir, got.TranscriptPath)) + if n := strings.Count(string(transcript), "\n"); n != 1 || !strings.Contains(string(transcript), `"status":"succeeded"`) { + t.Fatalf("transcript = %s", transcript) + } + if cmds := f.commands(); len(cmds) != 1 || cmds[0].Status != api.CommandCompleted || cmds[0].Command[0] != "sh" { + t.Fatalf("cmdlog = %+v", cmds) + } + if len(f.signals) != 1 || f.signals[0].kind != api.SignalStep || f.signals[0].data != (api.SignalStepData{Step: 1, Status: api.StepSucceeded, Attempts: 1}) { + t.Fatalf("signals = %+v", f.signals) + } + if len(f.torn) != 0 { + t.Fatal("a finished step was torn down") + } +} + +func TestMismatchRetriesWithTheViolationsThenGivesUp(t *testing.T) { + f := newFixture(t, answers(`{"verdict":"maybe"}`, `prose`, `{"verdict":"pass"}`)) + req := f.process("Judge it.", okSchema) + req.RetryOnMismatch = 2 + step := f.run(req) + if step.Status != api.StepSucceeded || step.Attempts != 3 || step.N != 1 { + t.Fatalf("step = %+v", step) + } + if p := f.launcher.prompts[1]; !strings.HasPrefix(p, "Judge it.\n\n") || !strings.Contains(p, "$.verdict") || !strings.Contains(p, "remaining after this one: 1") { + t.Fatalf("second prompt = %q", p) + } + if p := f.launcher.prompts[2]; !strings.Contains(p, ErrNoAnswer.Error()) || !strings.Contains(p, "remaining after this one: 0") { + t.Fatalf("third prompt = %q", p) + } + if len(f.commands()) != 3 || len(f.signals) != 1 { + t.Fatalf("cmdlog %d signals %d", len(f.commands()), len(f.signals)) + } + + f = newFixture(t, answers(`{"verdict":"maybe"}`)) + req = f.process("Judge it.", okSchema) + req.RetryOnMismatch = 1 + step = f.run(req) + if step.Status != api.StepSchemaMismatch || step.Attempts != 2 || step.Answer != nil || len(step.Violations) != 1 || step.Violations[0].Path != "$.verdict" || !strings.Contains(step.Reason, "after 2 attempt") { + t.Fatalf("step = %+v", step) + } + if f.signals[0].data.Status != api.StepSchemaMismatch { + t.Fatalf("signal = %+v", f.signals) + } +} + +func TestStepsNumberUpAndListInOrder(t *testing.T) { + f := newFixture(t, answers(`{"verdict":"pass"}`)) + for i := 1; i <= 3; i++ { + if step := f.run(f.process("p", okSchema)); step.N != i { + t.Fatalf("step %d numbered %d", i, step.N) + } + } + store := Store{RunDir: f.host.RunDir} + steps, err := store.List() + if err != nil || len(steps) != 3 || steps[0].N != 1 || steps[2].N != 3 { + t.Fatalf("List = %+v, %v", steps, err) + } + if _, err := store.Get(4); !errors.Is(err, ErrNotFound) { + t.Fatalf("Get(4) = %v", err) + } + if _, err := store.Get(0); !errors.Is(err, ErrNotFound) { + t.Fatalf("Get(0) = %v", err) + } + // A corrupt record is reported after the readable ones. + if err := os.WriteFile(filepath.Join(store.Dir(2), RecordName), []byte("{"), 0o600); err != nil { + t.Fatal(err) + } + steps, err = store.List() + if !errors.Is(err, ErrCorrupt) || len(steps) != 2 { + t.Fatalf("List with a corrupt record = %+v, %v", steps, err) + } + // Numbering still moves past it. + if step := f.run(f.process("p", okSchema)); step.N != 4 { + t.Fatalf("after corrupt: N = %d", step.N) + } + if steps, err := (Store{RunDir: t.TempDir()}).List(); err != nil || len(steps) != 0 { + t.Fatalf("empty List = %+v, %v", steps, err) + } +} + +func TestPromptPathReadsFromTheWorkspace(t *testing.T) { + f := newFixture(t, answers(`{"verdict":"pass"}`)) + if err := os.WriteFile(filepath.Join(f.host.Workspace, "task.md"), []byte("from file"), 0o600); err != nil { + t.Fatal(err) + } + req := f.process("", okSchema) + req.PromptPath = "task.md" + f.run(req) + if f.launcher.prompts[0] != "from file" { + t.Fatalf("prompt = %q", f.launcher.prompts[0]) + } +} + +func TestRequestRefusals(t *testing.T) { + f := newFixture(t, answers(`{}`)) + long := strings.Repeat("x", MaxPromptBytes+1) + for _, tc := range []struct { + name string + req api.StepRequest + msg string + }{ + {"backend", api.StepRequest{Backend: "gemini", Prompt: "p", OutputSchema: json.RawMessage(okSchema)}, "unknown backend"}, + {"both prompts", api.StepRequest{Backend: "process", Prompt: "p", PromptPath: "x", OutputSchema: json.RawMessage(okSchema)}, "mutually exclusive"}, + {"no prompt", api.StepRequest{Backend: "process", OutputSchema: json.RawMessage(okSchema)}, "one of prompt or prompt_path"}, + {"blank prompt", api.StepRequest{Backend: "process", Prompt: " ", OutputSchema: json.RawMessage(okSchema)}, "prompt is empty"}, + {"long prompt", api.StepRequest{Backend: "process", Prompt: long, OutputSchema: json.RawMessage(okSchema)}, "bound"}, + {"absolute prompt path", api.StepRequest{Backend: "process", PromptPath: "/etc/passwd", OutputSchema: json.RawMessage(okSchema)}, "relative"}, + {"escaping prompt path", api.StepRequest{Backend: "process", PromptPath: "../x", OutputSchema: json.RawMessage(okSchema)}, "relative"}, + {"missing prompt path", api.StepRequest{Backend: "process", PromptPath: "nope.md", OutputSchema: json.RawMessage(okSchema)}, "nope.md"}, + {"no schema", api.StepRequest{Backend: "process", Prompt: "p"}, "output_schema"}, + {"bad schema", api.StepRequest{Backend: "process", Prompt: "p", OutputSchema: json.RawMessage(`{"oneOf":[]}`)}, "oneOf"}, + {"retries", api.StepRequest{Backend: "process", Prompt: "p", OutputSchema: json.RawMessage(okSchema), RetryOnMismatch: MaxRetries + 1}, "retry_on_mismatch"}, + {"turn limit", api.StepRequest{Backend: "process", Prompt: "p", OutputSchema: json.RawMessage(okSchema), TurnLimit: -1}, "turn_limit"}, + {"timeout cap", api.StepRequest{Backend: "process", Prompt: "p", OutputSchema: json.RawMessage(okSchema), TimeoutMs: (2 * time.Hour).Milliseconds()}, "timeout_ms"}, + {"empty tool", api.StepRequest{Backend: "claude", Prompt: "p", OutputSchema: json.RawMessage(okSchema), Tools: []string{" "}}, "empty name"}, + {"process without command", api.StepRequest{Backend: "process", Prompt: "p", OutputSchema: json.RawMessage(okSchema)}, "needs command"}, + {"claude with command", api.StepRequest{Backend: "claude", Prompt: "p", OutputSchema: json.RawMessage(okSchema), Command: []string{"x"}}, "builds its own command"}, + } { + t.Run(tc.name, func(t *testing.T) { + var reqErr *RequestError + _, err := Run(context.Background(), f.host, tc.req) + if !errors.As(err, &reqErr) || !strings.Contains(err.Error(), tc.msg) { + t.Fatalf("err = %T %v, want RequestError containing %q", err, err, tc.msg) + } + }) + } + if len(f.launcher.specs) != 0 { + t.Fatal("a refused request was launched") + } + if steps, _ := (Store{RunDir: f.host.RunDir}).List(); len(steps) != 0 { + t.Fatalf("a refused request was recorded: %+v", steps) + } +} + +func TestUnsupportedRequestIsRecordedAndRefused(t *testing.T) { + f := newFixture(t, answers(`{}`)) + req := f.process("p", okSchema) + req.Tools = []string{"Bash"} + step, err := Run(context.Background(), f.host, req) + var unsupported *UnsupportedError + if !errors.As(err, &unsupported) || !strings.Contains(err.Error(), "cannot restrict tools") { + t.Fatalf("err = %v", err) + } + if step.N != 1 || step.Status != api.StepUnsupported || step.Reason != err.Error() || step.Attempts != 0 { + t.Fatalf("step = %+v", step) + } + if got, err := (Store{RunDir: f.host.RunDir}).Get(1); err != nil || got.Status != api.StepUnsupported { + t.Fatalf("Get = %+v, %v", got, err) + } + if len(f.launcher.specs) != 0 || len(f.signals) != 1 || f.signals[0].data.Status != api.StepUnsupported { + t.Fatalf("launched %d, signals %+v", len(f.launcher.specs), f.signals) + } +} + +func TestBackendFailureStatuses(t *testing.T) { + // A non-zero exit is a backend error, logged failed, no retry. + f := newFixture(t, &fakeLauncher{results: []func(context.Context, sandbox.LaunchSpec) (sandbox.Result, error){ + func(context.Context, sandbox.LaunchSpec) (sandbox.Result, error) { + return sandbox.Result{ExitCode: 2, Stderr: "ghp_0123456789abcdefXYZ boom"}, nil + }, + }}) + req := f.process("p", okSchema) + req.RetryOnMismatch = 3 + step := f.run(req) + if step.Status != api.StepBackendError || step.Attempts != 1 || !strings.Contains(step.Reason, "exited 2") { + t.Fatalf("step = %+v", step) + } + if cmds := f.commands(); len(cmds) != 1 || cmds[0].Status != api.CommandFailed || *cmds[0].ExitCode != 2 { + t.Fatalf("cmdlog = %+v", cmds) + } + transcript, _ := os.ReadFile(filepath.Join(f.host.RunDir, step.TranscriptPath)) + if strings.Contains(string(transcript), "ghp_0123456789abcdefXYZ") { + t.Fatalf("transcript kept a secret: %s", transcript) + } + + // A launcher that could not start anything is a backend error too. + f = newFixture(t, &fakeLauncher{results: []func(context.Context, sandbox.LaunchSpec) (sandbox.Result, error){ + func(context.Context, sandbox.LaunchSpec) (sandbox.Result, error) { + return sandbox.Result{}, errors.New("podman: image missing") + }, + }}) + step = f.run(f.process("p", okSchema)) + if step.Status != api.StepBackendError || !strings.Contains(step.Reason, "image missing") { + t.Fatalf("step = %+v", step) + } + if cmds := f.commands(); len(cmds) != 1 || cmds[0].Status != api.CommandFailed || cmds[0].ExitCode != nil { + t.Fatalf("cmdlog = %+v", cmds) + } +} + +func TestTimeoutAndCancelTearDownTheLaunch(t *testing.T) { + wait := func(ctx context.Context, _ sandbox.LaunchSpec) (sandbox.Result, error) { + <-ctx.Done() + return sandbox.Result{ExitCode: -1}, ctx.Err() + } + f := newFixture(t, &fakeLauncher{results: []func(context.Context, sandbox.LaunchSpec) (sandbox.Result, error){wait}}) + req := f.process("p", okSchema) + req.TimeoutMs = 1 + step := f.run(req) + if step.Status != api.StepTimeout || !strings.Contains(step.Reason, "did not finish within 1ms") { + t.Fatalf("step = %+v", step) + } + if len(f.torn) != 1 || f.torn[0] != "crux-step-run_1-1-1" { + t.Fatalf("torn = %v", f.torn) + } + if cmds := f.commands(); len(cmds) != 1 || cmds[0].Status != api.CommandFailed { + t.Fatalf("cmdlog = %+v", cmds) + } + + f = newFixture(t, &fakeLauncher{results: []func(context.Context, sandbox.LaunchSpec) (sandbox.Result, error){wait}}) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + step, err := Run(ctx, f.host, f.process("p", okSchema)) + if err != nil || step.Status != api.StepCancelled || len(f.torn) != 1 { + t.Fatalf("step = %+v, %v, torn %v", step, err, f.torn) + } + if cmds := f.commands(); len(cmds) != 1 || cmds[0].Status != api.CommandCancelled { + t.Fatalf("cmdlog = %+v", cmds) + } + if f.signals[0].data.Status != api.StepCancelled { + t.Fatalf("signals = %+v", f.signals) + } +} + +func TestHostWithoutLogOrSignalsStillRuns(t *testing.T) { + f := newFixture(t, answers(`{"verdict":"fail"}`)) + f.host.CmdLog, f.host.Signal, f.host.Teardown = nil, nil, nil + step := f.run(f.process("p", okSchema)) + if step.Status != api.StepSucceeded || len(f.signals) != 0 { + t.Fatalf("step = %+v", step) + } +} diff --git a/src/crux/agentstep/backend.go b/src/crux/agentstep/backend.go new file mode 100644 index 0000000000..c464572247 --- /dev/null +++ b/src/crux/agentstep/backend.go @@ -0,0 +1,304 @@ +package agentstep + +import ( + "encoding/json" + "errors" + "fmt" + "strings" + + "github.com/hivecommons/hive/crux/api" + "github.com/hivecommons/hive/crux/jsonextract" + "github.com/hivecommons/hive/crux/sandbox" +) + +// Spec is what a Backend turns into the command the sandbox runs. Paths +// are relative to the workspace, which is the sandbox's working directory. +type Spec struct { + // PromptRel is the file holding this attempt's prompt. + PromptRel string + // AnswerRel is where a backend that can write its final message to a + // file should put it; the adapter reads it back with Outcome. + AnswerRel string + Model string + Tools []string + TurnLimit int + // Command is the process backend's argv; other backends refuse it. + Command []string +} + +// Outcome is what a Backend read out of one attempt. Answer is the +// candidate the schema is checked against; Status is empty when the +// attempt produced one, otherwise a terminal api.Step* status with Reason. +type Outcome struct { + Answer json.RawMessage + Usage api.StepUsage + Turns int + Status string + Reason string +} + +// Backend is the per-CLI adapter: it knows the flags that make the CLI +// read a prompt, pick a model, restrict its tools and stop after N turns, +// and how to read its answer and usage back. What it cannot enforce it +// refuses in Command with an *UnsupportedError; it never runs weakened. +type Backend interface { + Name() string + Command(spec Spec) ([]string, error) + // Parse reads the attempt's result. readFile returns a workspace file + // by relative path, for adapters that had the CLI write its answer to + // spec.AnswerRel. + Parse(spec Spec, res sandbox.Result, readFile func(rel string) ([]byte, error)) Outcome +} + +// UnsupportedError names a request the backend cannot honour. +type UnsupportedError struct{ What string } + +func (e *UnsupportedError) Error() string { return e.What } + +// RequestError is a malformed step request. +type RequestError struct{ Msg string } + +func (e *RequestError) Error() string { return e.Msg } + +// Backends is the shipped adapter set by name. +var Backends = map[string]Backend{ + api.StepBackendClaude: claudeBackend{}, + api.StepBackendCodex: codexBackend{}, + api.StepBackendProcess: processBackend{}, +} + +// Lookup returns the adapter for name or a RequestError listing the known +// ones. +func Lookup(name string) (Backend, error) { + if b, ok := Backends[name]; ok { + return b, nil + } + return nil, &RequestError{fmt.Sprintf("unknown backend %q; one of %s", name, strings.Join(backendNames(), ", "))} +} + +func backendNames() []string { + return []string{api.StepBackendClaude, api.StepBackendCodex, api.StepBackendProcess} +} + +// shellQuote single-quotes s for sh. +func shellQuote(s string) string { return "'" + strings.ReplaceAll(s, "'", `'\''`) + "'" } + +// shell wraps a command line so the prompt file is its stdin. +func shell(cmd, promptRel string) []string { + return []string{"sh", "-lc", cmd + " < " + shellQuote(promptRel)} +} + +// extractJSON finds the JSON value in model output: the whole text when it +// parses, else the text inside a ```json fence, else the first '{' to the +// last '}' (jsonextract.Object). Empty when nothing parses. +func extractJSON(text string) json.RawMessage { + candidates := []string{strings.TrimSpace(text)} + if i := strings.Index(text, "```"); i >= 0 { + rest := text[i+3:] + if nl := strings.IndexByte(rest, '\n'); nl >= 0 { + rest = rest[nl+1:] + } + if j := strings.Index(rest, "```"); j >= 0 { + candidates = append(candidates, strings.TrimSpace(rest[:j])) + } + } + candidates = append(candidates, jsonextract.Object(text)) + for _, c := range candidates { + if c != "" && json.Valid([]byte(c)) { + return json.RawMessage(c) + } + } + return nil +} + +// --- process ----------------------------------------------------------------- + +// processBackend runs the caller's command with the prompt on stdin and +// takes stdout as the answer. It restricts nothing and enforces no turn +// limit, so a request for either is refused. +type processBackend struct{} + +func (processBackend) Name() string { return api.StepBackendProcess } + +func (processBackend) Command(spec Spec) ([]string, error) { + if len(spec.Command) == 0 || strings.TrimSpace(spec.Command[0]) == "" { + return nil, &RequestError{"backend process needs command"} + } + if len(spec.Tools) > 0 { + return nil, &UnsupportedError{"backend process cannot restrict tools; it offers whatever the command offers"} + } + if spec.TurnLimit > 0 { + return nil, &UnsupportedError{"backend process cannot enforce a turn limit"} + } + if spec.Model != "" { + return nil, &UnsupportedError{"backend process takes no model; put it in the command"} + } + quoted := make([]string, len(spec.Command)) + for i, a := range spec.Command { + quoted[i] = shellQuote(a) + } + return shell(strings.Join(quoted, " "), spec.PromptRel), nil +} + +func (processBackend) Parse(_ Spec, res sandbox.Result, _ func(string) ([]byte, error)) Outcome { + if res.ExitCode != 0 { + return Outcome{Status: api.StepBackendError, Reason: fmt.Sprintf("command exited %d", res.ExitCode)} + } + return Outcome{Answer: extractJSON(res.Stdout), Turns: 1} +} + +// --- claude ------------------------------------------------------------------ + +// claudeBackend is `claude -p --output-format json`: one result object on +// stdout with the final text in result, token usage in usage, the turn +// count in num_turns and, when a tool outside --allowedTools was asked +// for, permission_denials. An empty Tools list runs with +// --dangerously-skip-permissions (every tool, the existing sandbox-kick +// behaviour); a non-empty one passes --allowedTools and leaves permission +// prompting on, so anything else is denied in -p mode. +type claudeBackend struct{} + +func (claudeBackend) Name() string { return api.StepBackendClaude } + +func (claudeBackend) Command(spec Spec) ([]string, error) { + if len(spec.Command) > 0 { + return nil, &RequestError{"backend claude builds its own command; command is for backend process"} + } + cmd := "claude -p --output-format json" + if spec.Model != "" { + cmd += " --model " + shellQuote(spec.Model) + } + if len(spec.Tools) > 0 { + cmd += " --allowedTools " + shellQuote(strings.Join(spec.Tools, ",")) + } else { + cmd += " --dangerously-skip-permissions" + } + if spec.TurnLimit > 0 { + cmd += fmt.Sprintf(" --max-turns %d", spec.TurnLimit) + } + return shell(cmd, spec.PromptRel), nil +} + +// claudeResult is the subset of the -p JSON result crux reads. +type claudeResult struct { + Type string `json:"type"` + Subtype string `json:"subtype"` + IsError bool `json:"is_error"` + Result string `json:"result"` + NumTurns int `json:"num_turns"` + Usage struct { + InputTokens int64 `json:"input_tokens"` + OutputTokens int64 `json:"output_tokens"` + } `json:"usage"` + PermissionDenials []struct { + ToolName string `json:"tool_name"` + } `json:"permission_denials"` +} + +func (claudeBackend) Parse(_ Spec, res sandbox.Result, _ func(string) ([]byte, error)) Outcome { + raw := extractJSON(res.Stdout) + var r claudeResult + if raw == nil || json.Unmarshal(raw, &r) != nil || r.Type != "result" { + return Outcome{Status: api.StepBackendError, Reason: fmt.Sprintf("claude exited %d without a result object", res.ExitCode)} + } + out := Outcome{Turns: r.NumTurns} + out.Usage.InputTokens, out.Usage.OutputTokens = r.Usage.InputTokens, r.Usage.OutputTokens + switch { + case len(r.PermissionDenials) > 0: + names := make([]string, len(r.PermissionDenials)) + for i, d := range r.PermissionDenials { + names[i] = d.ToolName + } + out.Status, out.Reason = api.StepToolDenied, "tool outside the allowlist requested: "+strings.Join(names, ", ") + case r.Subtype == "error_max_turns": + out.Status, out.Reason = api.StepTurnLimitExceeded, fmt.Sprintf("stopped after %d turns without an answer", r.NumTurns) + case r.IsError || (r.Subtype != "" && r.Subtype != "success"): + out.Status, out.Reason = api.StepBackendError, "claude reported "+r.Subtype + default: + out.Answer = extractJSON(r.Result) + } + return out +} + +// --- codex ------------------------------------------------------------------- + +// codexBackend is `codex exec --json`: a JSONL event stream on stdout and, +// with --output-last-message, the final message in a file crux names +// under the workspace. Usage comes from the turn.completed event. codex +// exec is one turn by construction, so a TurnLimit above 1 is refused, and +// it has no per-tool allowlist, so Tools is refused. The CLI's own sandbox +// and approvals are bypassed: crux's sandbox is the containment. +type codexBackend struct{} + +func (codexBackend) Name() string { return api.StepBackendCodex } + +func (codexBackend) Command(spec Spec) ([]string, error) { + if len(spec.Command) > 0 { + return nil, &RequestError{"backend codex builds its own command; command is for backend process"} + } + if len(spec.Tools) > 0 { + return nil, &UnsupportedError{"backend codex cannot restrict tools to an allowlist"} + } + if spec.TurnLimit > 1 { + return nil, &UnsupportedError{"backend codex runs exactly one turn; turn_limit above 1 cannot be enforced"} + } + cmd := "codex exec --json --skip-git-repo-check --dangerously-bypass-approvals-and-sandbox" + if spec.Model != "" { + cmd += " --model " + shellQuote(spec.Model) + } + cmd += " --output-last-message " + shellQuote(spec.AnswerRel) + " -" + return shell(cmd, spec.PromptRel), nil +} + +// codexEvent is the subset of the exec --json events crux reads. +type codexEvent struct { + Type string `json:"type"` + Usage *struct { + InputTokens int64 `json:"input_tokens"` + OutputTokens int64 `json:"output_tokens"` + } `json:"usage"` + Item *struct { + Type string `json:"type"` + Text string `json:"text"` + } `json:"item"` + Message string `json:"message"` +} + +func (codexBackend) Parse(spec Spec, res sandbox.Result, readFile func(string) ([]byte, error)) Outcome { + out := Outcome{Turns: 1} + var last, failure string + for _, line := range strings.Split(res.Stdout, "\n") { + var ev codexEvent + if json.Unmarshal([]byte(line), &ev) != nil { + continue + } + if ev.Usage != nil { + out.Usage.InputTokens += ev.Usage.InputTokens + out.Usage.OutputTokens += ev.Usage.OutputTokens + } + if ev.Item != nil && ev.Item.Type == "agent_message" { + last = ev.Item.Text + } + if ev.Type == "error" || ev.Type == "turn.failed" { + failure = ev.Message + } + } + if res.ExitCode != 0 { + if failure == "" { + failure = fmt.Sprintf("codex exited %d", res.ExitCode) + } + out.Status, out.Reason = api.StepBackendError, failure + return out + } + if readFile != nil { + if data, err := readFile(spec.AnswerRel); err == nil && len(strings.TrimSpace(string(data))) > 0 { + last = string(data) + } + } + out.Answer = extractJSON(last) + return out +} + +// ErrNoAnswer is the reason an attempt that ran clean produced nothing a +// schema could be checked against. +var ErrNoAnswer = errors.New("the backend returned no JSON value") diff --git a/src/crux/agentstep/backend_test.go b/src/crux/agentstep/backend_test.go new file mode 100644 index 0000000000..cf25bb17d9 --- /dev/null +++ b/src/crux/agentstep/backend_test.go @@ -0,0 +1,145 @@ +package agentstep + +import ( + "errors" + "strings" + "testing" + + "github.com/hivecommons/hive/crux/api" + "github.com/hivecommons/hive/crux/sandbox" +) + +func TestLookupKnowsTheThreeBackends(t *testing.T) { + for _, name := range []string{api.StepBackendClaude, api.StepBackendCodex, api.StepBackendProcess} { + b, err := Lookup(name) + if err != nil || b.Name() != name { + t.Fatalf("Lookup(%s) = %v, %v", name, b, err) + } + } + var reqErr *RequestError + if _, err := Lookup("gemini"); !errors.As(err, &reqErr) || !strings.Contains(err.Error(), "claude") { + t.Fatalf("Lookup(gemini) = %v", err) + } +} + +func TestProcessBackendCommandAndParse(t *testing.T) { + spec := Spec{PromptRel: ".crux/steps/1/prompt-1.md", Command: []string{"python3", "ask.py", "it's"}} + cmd, err := Backends["process"].Command(spec) + if err != nil { + t.Fatal(err) + } + want := []string{"sh", "-lc", `'python3' 'ask.py' 'it'\''s' < '.crux/steps/1/prompt-1.md'`} + if strings.Join(cmd, "\x00") != strings.Join(want, "\x00") { + t.Fatalf("Command = %q", cmd) + } + var reqErr *RequestError + var unsupported *UnsupportedError + if _, err := Backends["process"].Command(Spec{}); !errors.As(err, &reqErr) { + t.Errorf("no command: %v", err) + } + for _, s := range []Spec{ + {Command: []string{"x"}, Tools: []string{"Bash"}}, + {Command: []string{"x"}, TurnLimit: 2}, + {Command: []string{"x"}, Model: "m"}, + } { + if _, err := Backends["process"].Command(s); !errors.As(err, &unsupported) { + t.Errorf("%+v: %v", s, err) + } + } + out := Backends["process"].Parse(spec, sandbox.Result{Stdout: "note\n```json\n{\"a\":1}\n```\n"}, nil) + if out.Status != "" || string(out.Answer) != `{"a":1}` { + t.Fatalf("Parse fenced = %+v", out) + } + out = Backends["process"].Parse(spec, sandbox.Result{ExitCode: 3}, nil) + if out.Status != api.StepBackendError || !strings.Contains(out.Reason, "exited 3") { + t.Fatalf("Parse failure = %+v", out) + } + if out := Backends["process"].Parse(spec, sandbox.Result{Stdout: "nothing here"}, nil); out.Answer != nil { + t.Fatalf("Parse prose = %+v", out) + } +} + +func TestClaudeBackendCommandAndParse(t *testing.T) { + cmd, err := Backends["claude"].Command(Spec{PromptRel: "p.md", Model: "opus", Tools: []string{"Read", "Grep"}, TurnLimit: 3}) + if err != nil { + t.Fatal(err) + } + if line := cmd[2]; !strings.Contains(line, "claude -p --output-format json --model 'opus' --allowedTools 'Read,Grep' --max-turns 3 < 'p.md'") { + t.Fatalf("Command = %q", line) + } + cmd, _ = Backends["claude"].Command(Spec{PromptRel: "p.md"}) + if !strings.Contains(cmd[2], "--dangerously-skip-permissions") || strings.Contains(cmd[2], "--max-turns") { + t.Fatalf("no-tools command = %q", cmd[2]) + } + var reqErr *RequestError + if _, err := Backends["claude"].Command(Spec{Command: []string{"x"}}); !errors.As(err, &reqErr) { + t.Errorf("command with claude: %v", err) + } + for _, tc := range []struct { + stdout string + status string + answer string + }{ + {`{"type":"result","subtype":"success","result":"Sure: {\"ok\":true}","num_turns":2,"usage":{"input_tokens":10,"output_tokens":5}}`, "", `{"ok":true}`}, + {`{"type":"result","subtype":"success","result":"no json","num_turns":1}`, "", ""}, + {`{"type":"result","subtype":"success","result":"{}","permission_denials":[{"tool_name":"Bash"}]}`, api.StepToolDenied, ""}, + {`{"type":"result","subtype":"error_max_turns","num_turns":3}`, api.StepTurnLimitExceeded, ""}, + {`{"type":"result","subtype":"error_during_execution","is_error":true}`, api.StepBackendError, ""}, + {`garbage`, api.StepBackendError, ""}, + } { + out := Backends["claude"].Parse(Spec{}, sandbox.Result{Stdout: tc.stdout}, nil) + if out.Status != tc.status || string(out.Answer) != tc.answer { + t.Errorf("Parse(%s) = %+v", tc.stdout, out) + } + if tc.answer != "" && (out.Usage.InputTokens != 10 || out.Usage.OutputTokens != 5 || out.Turns != 2) { + t.Errorf("usage = %+v", out) + } + } +} + +func TestCodexBackendCommandAndParse(t *testing.T) { + cmd, err := Backends["codex"].Command(Spec{PromptRel: "p.md", AnswerRel: "a.txt", Model: "o3", TurnLimit: 1}) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(cmd[2], "codex exec --json --skip-git-repo-check --dangerously-bypass-approvals-and-sandbox --model 'o3' --output-last-message 'a.txt' - < 'p.md'") { + t.Fatalf("Command = %q", cmd[2]) + } + var unsupported *UnsupportedError + if _, err := Backends["codex"].Command(Spec{Tools: []string{"x"}}); !errors.As(err, &unsupported) { + t.Errorf("tools: %v", err) + } + if _, err := Backends["codex"].Command(Spec{TurnLimit: 2}); !errors.As(err, &unsupported) { + t.Errorf("turn limit: %v", err) + } + stdout := strings.Join([]string{ + `{"type":"thread.started"}`, + `not json`, + `{"type":"item.completed","item":{"type":"agent_message","text":"{\"from\":\"stdout\"}"}}`, + `{"type":"turn.completed","usage":{"input_tokens":7,"output_tokens":3}}`, + }, "\n") + files := map[string]string{"a.txt": "{\"from\":\"file\"}\n"} + read := func(rel string) ([]byte, error) { + if v, ok := files[rel]; ok { + return []byte(v), nil + } + return nil, errors.New("missing") + } + spec := Spec{AnswerRel: "a.txt"} + out := Backends["codex"].Parse(spec, sandbox.Result{Stdout: stdout}, read) + if out.Status != "" || string(out.Answer) != `{"from":"file"}` || out.Usage.InputTokens != 7 || out.Usage.OutputTokens != 3 { + t.Fatalf("Parse = %+v", out) + } + out = Backends["codex"].Parse(Spec{AnswerRel: "none"}, sandbox.Result{Stdout: stdout}, read) + if string(out.Answer) != `{"from":"stdout"}` { + t.Fatalf("Parse without a file = %+v", out) + } + out = Backends["codex"].Parse(spec, sandbox.Result{Stdout: `{"type":"error","message":"quota"}`, ExitCode: 1}, read) + if out.Status != api.StepBackendError || out.Reason != "quota" { + t.Fatalf("Parse failure = %+v", out) + } + out = Backends["codex"].Parse(spec, sandbox.Result{ExitCode: 2}, read) + if out.Status != api.StepBackendError || !strings.Contains(out.Reason, "exited 2") { + t.Fatalf("Parse silent failure = %+v", out) + } +} diff --git a/src/crux/agentstep/schema.go b/src/crux/agentstep/schema.go new file mode 100644 index 0000000000..078b6e723d --- /dev/null +++ b/src/crux/agentstep/schema.go @@ -0,0 +1,406 @@ +package agentstep + +import ( + "encoding/json" + "errors" + "fmt" + "math" + "regexp" + "sort" + "strconv" + "strings" + + "github.com/hivecommons/hive/crux/api" +) + +// MaxSchemaBytes bounds the schema a request may carry. +const MaxSchemaBytes = 256 << 10 + +// ErrSchema is a schema that is not usable: not an object, over the size +// bound, or using a keyword this validator does not implement (a keyword +// it cannot enforce is refused rather than ignored, so a caller never +// believes a constraint was checked when it was not). +var ErrSchema = errors.New("agentstep: invalid output_schema") + +// Schema is the subset of JSON Schema (draft 2020-12 vocabulary) crux +// checks answers against: type, enum, const, properties, required, +// additionalProperties (boolean or schema), items, minItems, maxItems, +// minLength, maxLength, pattern, minimum, maximum, exclusiveMinimum, +// exclusiveMaximum, multipleOf and the annotations title, description, +// default, examples and $schema. Anything else is ErrSchema. +type Schema struct { + types []string + enum []json.RawMessage + constant json.RawMessage + hasConst bool + properties map[string]*Schema + required []string + additional *Schema + additionalForbidden bool + items *Schema + minItems, maxItems *int + minLength, maxLength *int + pattern *regexp.Regexp + minimum, maximum *float64 + exclusiveMin, exclusiveMax *float64 + multipleOf *float64 +} + +// annotationKeywords are accepted and ignored: they constrain nothing. +var annotationKeywords = map[string]bool{ + "title": true, "description": true, "default": true, "examples": true, "$schema": true, "$id": true, "$comment": true, +} + +// ParseSchema parses raw into a Schema or returns ErrSchema. +func ParseSchema(raw json.RawMessage) (*Schema, error) { + if len(raw) == 0 { + return nil, fmt.Errorf("%w: output_schema is required", ErrSchema) + } + if len(raw) > MaxSchemaBytes { + return nil, fmt.Errorf("%w: %d bytes exceeds the %d-byte bound", ErrSchema, len(raw), MaxSchemaBytes) + } + return parseSchema(raw, "$") +} + +func parseSchema(raw json.RawMessage, at string) (*Schema, error) { + var fields map[string]json.RawMessage + if err := json.Unmarshal(raw, &fields); err != nil || fields == nil { + if err == nil { + // JSON null decodes to a nil map without error. + return nil, fmt.Errorf("%w: %s must be an object", ErrSchema, at) + } + // A bare boolean schema: true accepts anything, false nothing. + var b bool + if json.Unmarshal(raw, &b) == nil { + if b { + return &Schema{}, nil + } + return &Schema{enum: []json.RawMessage{}}, nil + } + return nil, fmt.Errorf("%w: %s must be an object", ErrSchema, at) + } + s := &Schema{} + for key, val := range fields { + var err error + switch key { + case "type": + err = s.parseType(val, at) + case "enum": + err = json.Unmarshal(val, &s.enum) + case "const": + s.constant, s.hasConst = val, true + case "properties": + var props map[string]json.RawMessage + if err = json.Unmarshal(val, &props); err == nil { + s.properties = map[string]*Schema{} + for name, sub := range props { + if s.properties[name], err = parseSchema(sub, at+"."+name); err != nil { + return nil, err + } + } + } + case "required": + err = json.Unmarshal(val, &s.required) + case "additionalProperties": + var b bool + if json.Unmarshal(val, &b) == nil { + s.additionalForbidden = !b + } else if s.additional, err = parseSchema(val, at+".*"); err != nil { + return nil, err + } + case "items": + if s.items, err = parseSchema(val, at+"[]"); err != nil { + return nil, err + } + case "minItems": + s.minItems, err = parseInt(val) + case "maxItems": + s.maxItems, err = parseInt(val) + case "minLength": + s.minLength, err = parseInt(val) + case "maxLength": + s.maxLength, err = parseInt(val) + case "pattern": + var p string + if err = json.Unmarshal(val, &p); err == nil { + s.pattern, err = regexp.Compile(p) + } + case "minimum": + s.minimum, err = parseNumber(val) + case "maximum": + s.maximum, err = parseNumber(val) + case "exclusiveMinimum": + s.exclusiveMin, err = parseNumber(val) + case "exclusiveMaximum": + s.exclusiveMax, err = parseNumber(val) + case "multipleOf": + if s.multipleOf, err = parseNumber(val); err == nil && *s.multipleOf <= 0 { + err = errors.New("must be positive") + } + default: + if annotationKeywords[key] { + continue + } + return nil, fmt.Errorf("%w: %s uses keyword %q, which this validator does not enforce", ErrSchema, at, key) + } + if err != nil { + return nil, fmt.Errorf("%w: %s.%s: %v", ErrSchema, at, key, err) + } + } + return s, nil +} + +func (s *Schema) parseType(val json.RawMessage, at string) error { + var one string + if json.Unmarshal(val, &one) == nil { + s.types = []string{one} + } else if err := json.Unmarshal(val, &s.types); err != nil { + return errors.New("must be a string or an array of strings") + } + for _, t := range s.types { + switch t { + case "object", "array", "string", "number", "integer", "boolean", "null": + default: + return fmt.Errorf("unknown type %q", t) + } + } + return nil +} + +func parseInt(val json.RawMessage) (*int, error) { + var n int + if err := json.Unmarshal(val, &n); err != nil || n < 0 { + return nil, errors.New("must be a non-negative integer") + } + return &n, nil +} + +func parseNumber(val json.RawMessage) (*float64, error) { + var f float64 + if err := json.Unmarshal(val, &f); err != nil { + return nil, errors.New("must be a number") + } + return &f, nil +} + +// Validate checks answer against the schema and returns every violation +// found, in path order; none means the answer matches. answer must be one +// JSON value. +func (s *Schema) Validate(answer json.RawMessage) []api.SchemaViolation { + var v any + dec := json.NewDecoder(strings.NewReader(string(answer))) + dec.UseNumber() + if err := dec.Decode(&v); err != nil { + return []api.SchemaViolation{{Path: "$", Message: "not valid JSON: " + err.Error()}} + } + if dec.More() { + return []api.SchemaViolation{{Path: "$", Message: "must be exactly one JSON value"}} + } + var out []api.SchemaViolation + s.check(v, "$", &out) + sort.SliceStable(out, func(i, j int) bool { return out[i].Path < out[j].Path }) + return out +} + +func (s *Schema) check(v any, path string, out *[]api.SchemaViolation) { + add := func(msg string) { *out = append(*out, api.SchemaViolation{Path: path, Message: msg}) } + if len(s.types) > 0 && !typeMatches(s.types, v) { + add(fmt.Sprintf("expected %s, got %s", strings.Join(s.types, " or "), typeName(v))) + return + } + if s.hasConst && !jsonEqual(s.constant, v) { + add("must equal the const value") + } + if s.enum != nil { + found := false + for _, e := range s.enum { + if jsonEqual(e, v) { + found = true + break + } + } + if !found { + add("not one of the enum values") + } + } + switch x := v.(type) { + case map[string]any: + s.checkObject(x, path, out) + case []any: + s.checkArray(x, path, out) + case string: + s.checkString(x, add) + case json.Number: + s.checkNumber(x, add) + } +} + +func (s *Schema) checkObject(x map[string]any, path string, out *[]api.SchemaViolation) { + for _, name := range s.required { + if _, ok := x[name]; !ok { + *out = append(*out, api.SchemaViolation{Path: path + "." + name, Message: "required property is missing"}) + } + } + keys := make([]string, 0, len(x)) + for k := range x { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + sub, declared := s.properties[k] + switch { + case declared: + sub.check(x[k], path+"."+k, out) + case s.additionalForbidden: + *out = append(*out, api.SchemaViolation{Path: path + "." + k, Message: "property is not allowed"}) + case s.additional != nil: + s.additional.check(x[k], path+"."+k, out) + } + } +} + +func (s *Schema) checkArray(x []any, path string, out *[]api.SchemaViolation) { + if s.minItems != nil && len(x) < *s.minItems { + *out = append(*out, api.SchemaViolation{Path: path, Message: fmt.Sprintf("needs at least %d items, has %d", *s.minItems, len(x))}) + } + if s.maxItems != nil && len(x) > *s.maxItems { + *out = append(*out, api.SchemaViolation{Path: path, Message: fmt.Sprintf("allows at most %d items, has %d", *s.maxItems, len(x))}) + } + if s.items != nil { + for i, item := range x { + s.items.check(item, path+"["+strconv.Itoa(i)+"]", out) + } + } +} + +func (s *Schema) checkString(x string, add func(string)) { + n := len([]rune(x)) + if s.minLength != nil && n < *s.minLength { + add(fmt.Sprintf("shorter than %d characters", *s.minLength)) + } + if s.maxLength != nil && n > *s.maxLength { + add(fmt.Sprintf("longer than %d characters", *s.maxLength)) + } + if s.pattern != nil && !s.pattern.MatchString(x) { + add("does not match pattern " + s.pattern.String()) + } +} + +func (s *Schema) checkNumber(x json.Number, add func(string)) { + f, err := x.Float64() + if err != nil { + add("not a representable number") + return + } + if s.minimum != nil && f < *s.minimum { + add(fmt.Sprintf("below minimum %v", *s.minimum)) + } + if s.maximum != nil && f > *s.maximum { + add(fmt.Sprintf("above maximum %v", *s.maximum)) + } + if s.exclusiveMin != nil && f <= *s.exclusiveMin { + add(fmt.Sprintf("not above %v", *s.exclusiveMin)) + } + if s.exclusiveMax != nil && f >= *s.exclusiveMax { + add(fmt.Sprintf("not below %v", *s.exclusiveMax)) + } + if s.multipleOf != nil { + q := f / *s.multipleOf + if math.Abs(q-math.Round(q)) > 1e-9 { + add(fmt.Sprintf("not a multiple of %v", *s.multipleOf)) + } + } +} + +func typeMatches(types []string, v any) bool { + for _, t := range types { + switch t { + case "null": + if v == nil { + return true + } + case "boolean": + if _, ok := v.(bool); ok { + return true + } + case "string": + if _, ok := v.(string); ok { + return true + } + case "object": + if _, ok := v.(map[string]any); ok { + return true + } + case "array": + if _, ok := v.([]any); ok { + return true + } + case "number": + if _, ok := v.(json.Number); ok { + return true + } + case "integer": + if n, ok := v.(json.Number); ok { + if _, err := n.Int64(); err == nil { + return true + } + if f, err := n.Float64(); err == nil && f == math.Trunc(f) { + return true + } + } + } + } + return false +} + +func typeName(v any) string { + switch x := v.(type) { + case nil: + return "null" + case bool: + return "boolean" + case string: + return "string" + case json.Number: + if _, err := x.Int64(); err == nil { + return "integer" + } + return "number" + case []any: + return "array" + case map[string]any: + return "object" + } + return fmt.Sprintf("%T", v) +} + +// jsonEqual compares a raw schema value with a decoded answer value by +// re-encoding both canonically. +func jsonEqual(raw json.RawMessage, v any) bool { + var want any + dec := json.NewDecoder(strings.NewReader(string(raw))) + dec.UseNumber() + if err := dec.Decode(&want); err != nil { + return false + } + a, err1 := json.Marshal(want) + b, err2 := json.Marshal(v) + return err1 == nil && err2 == nil && string(a) == string(b) +} + +// CorrectivePrompt is the repair instruction appended to the prompt of the +// attempt after a schema mismatch: the violations, verbatim, and the shape +// asked for. The pattern is outputschema.CorrectivePrompt, generalised to +// a caller-supplied schema. +func CorrectivePrompt(schema json.RawMessage, violations []api.SchemaViolation, remaining int) string { + var b strings.Builder + b.WriteString("Your previous answer did not match the required JSON Schema and could not be used. ") + b.WriteString("Reply with exactly one JSON value matching this schema and nothing else:\n") + b.Write(schema) + b.WriteString("\nFix these validation errors:\n") + for _, v := range violations { + fmt.Fprintf(&b, "- %s: %s\n", v.Path, v.Message) + } + fmt.Fprintf(&b, "Attempts remaining after this one: %d.", remaining) + return b.String() +} diff --git a/src/crux/agentstep/schema_test.go b/src/crux/agentstep/schema_test.go new file mode 100644 index 0000000000..b4a88bf042 --- /dev/null +++ b/src/crux/agentstep/schema_test.go @@ -0,0 +1,103 @@ +package agentstep + +import ( + "encoding/json" + "errors" + "strings" + "testing" +) + +func mustSchema(t *testing.T, raw string) *Schema { + t.Helper() + s, err := ParseSchema(json.RawMessage(raw)) + if err != nil { + t.Fatalf("ParseSchema(%s): %v", raw, err) + } + return s +} + +func paths(s *Schema, answer string) string { + var out []string + for _, v := range s.Validate(json.RawMessage(answer)) { + out = append(out, v.Path) + } + return strings.Join(out, " ") +} + +func TestSchemaValidatesTheSupportedKeywords(t *testing.T) { + s := mustSchema(t, `{ + "type": "object", + "required": ["verdict", "items"], + "additionalProperties": false, + "properties": { + "verdict": {"enum": ["pass", "fail"]}, + "score": {"type": "number", "minimum": 0, "maximum": 1, "exclusiveMaximum": 1}, + "count": {"type": "integer", "multipleOf": 2}, + "name": {"type": "string", "minLength": 2, "maxLength": 4, "pattern": "^[a-z]+$"}, + "items": {"type": "array", "minItems": 1, "maxItems": 2, "items": {"type": "string"}}, + "flag": {"const": true}, + "any": true, + "never": false, + "nullable": {"type": ["string", "null"]}, + "nested": {"type": "object", "additionalProperties": {"type": "integer"}} + } + }`) + for _, tc := range []struct{ answer, want string }{ + {`{"verdict":"pass","items":["a"]}`, ""}, + {`{"verdict":"pass","items":["a"],"score":0.5,"count":4,"name":"ab","flag":true,"any":[1],"nullable":null,"nested":{"x":1}}`, ""}, + {`{"verdict":"maybe","items":[]}`, "$.items $.verdict"}, + {`{"items":["a"]}`, "$.verdict"}, + {`{"verdict":"pass","items":["a"],"extra":1}`, "$.extra"}, + {`{"verdict":"pass","items":["a"],"score":1}`, "$.score"}, + {`{"verdict":"pass","items":["a"],"score":-1}`, "$.score"}, + {`{"verdict":"pass","items":["a"],"count":3}`, "$.count"}, + {`{"verdict":"pass","items":["a"],"count":2.5}`, "$.count"}, + {`{"verdict":"pass","items":["a"],"name":"a"}`, "$.name"}, + {`{"verdict":"pass","items":["a"],"name":"abcde"}`, "$.name"}, + {`{"verdict":"pass","items":["a"],"name":"AB"}`, "$.name"}, + {`{"verdict":"pass","items":["a","b","c"]}`, "$.items"}, + {`{"verdict":"pass","items":[1]}`, "$.items[0]"}, + {`{"verdict":"pass","items":["a"],"flag":false}`, "$.flag"}, + {`{"verdict":"pass","items":["a"],"never":1}`, "$.never"}, + {`{"verdict":"pass","items":["a"],"nullable":1}`, "$.nullable"}, + {`{"verdict":"pass","items":["a"],"nested":{"x":"y"}}`, "$.nested.x"}, + {`[]`, "$"}, + {`not json`, "$"}, + } { + if got := paths(s, tc.answer); got != tc.want { + t.Errorf("Validate(%s) violated %q, want %q", tc.answer, got, tc.want) + } + } +} + +func TestSchemaParseRefusesWhatItCannotEnforce(t *testing.T) { + for _, raw := range []string{ + ``, `null`, `[]`, `{"type": "thing"}`, `{"type": 3}`, `{"$ref": "#/x"}`, `{"oneOf": []}`, + `{"minimum": "1"}`, `{"minItems": -1}`, `{"pattern": "("}`, `{"properties": {"a": 1}}`, + `{"required": "a"}`, `{"enum": 1}`, `{"additionalProperties": 1}`, `{"items": [1]}`, + `{"multipleOf": 0}`, + } { + if _, err := ParseSchema(json.RawMessage(raw)); !errors.Is(err, ErrSchema) { + t.Errorf("ParseSchema(%s) = %v, want ErrSchema", raw, err) + } + } + if _, err := ParseSchema(json.RawMessage(`{"type":"string","` + strings.Repeat("x", MaxSchemaBytes) + `":1}`)); !errors.Is(err, ErrSchema) { + t.Errorf("oversized schema: %v", err) + } + // Annotations are allowed and ignored. + s := mustSchema(t, `{"$schema":"https://json-schema.org/draft/2020-12/schema","title":"t","description":"d","examples":[1],"default":1,"$id":"x","$comment":"c","type":"integer"}`) + if got := paths(s, `1`); got != "" { + t.Errorf("annotated schema violated %q", got) + } +} + +func TestCorrectivePromptNamesTheViolations(t *testing.T) { + s := mustSchema(t, `{"type":"object","required":["a"]}`) + v := s.Validate(json.RawMessage(`{}`)) + got := CorrectivePrompt(json.RawMessage(`{"type":"object","required":["a"]}`), v, 2) + for _, want := range []string{`"required":["a"]`, "- $.a: ", "remaining after this one: 2"} { + if !strings.Contains(got, want) { + t.Errorf("CorrectivePrompt lacks %q:\n%s", want, got) + } + } +} diff --git a/src/crux/agentstep/store.go b/src/crux/agentstep/store.go new file mode 100644 index 0000000000..876fc23815 --- /dev/null +++ b/src/crux/agentstep/store.go @@ -0,0 +1,159 @@ +package agentstep + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "sort" + "strconv" + "sync" + + "github.com/hivecommons/hive/crux/api" +) + +// Storage under the run's directory: +// +// steps//record.json the step's saved record (api.Step) +// steps//transcript.jsonl one line per attempt: command, exit, +// scrubbed output, status, violations, usage +// +// Steps are numbered from 1 in the order they were created. +const ( + StepsDir = "steps" + RecordName = "record.json" + TranscriptName = "transcript.jsonl" + fileMode = 0o600 + dirMode = 0o700 +) + +// Typed errors the serve routes map onto api.Code* values. +var ( + // ErrNotFound: no such step number for this run. + ErrNotFound = errors.New("agentstep: step not found") + // ErrCorrupt: a saved record did not parse. + ErrCorrupt = errors.New("agentstep: corrupt record") +) + +// createMu serialises step numbering across the process: two sessions +// never share a run directory, but one session may receive two posts. +var createMu sync.Mutex + +// Store reads and writes the steps of one run directory. +type Store struct{ RunDir string } + +// Dir is the directory of step n. +func (s Store) Dir(n int) string { return filepath.Join(s.RunDir, StepsDir, strconv.Itoa(n)) } + +// TranscriptRel is the transcript path relative to the run directory, as +// the record reports it. +func (s Store) TranscriptRel(n int) string { + return filepath.ToSlash(filepath.Join(StepsDir, strconv.Itoa(n), TranscriptName)) +} + +// Create assigns the next step number, writes the record and returns it. +func (s Store) Create(step api.Step) (api.Step, error) { + createMu.Lock() + defer createMu.Unlock() + steps, err := s.List() + if err != nil && !errors.Is(err, ErrCorrupt) { + return api.Step{}, err + } + step.N = 1 + if len(steps) > 0 { + step.N = steps[len(steps)-1].N + 1 + } + if err := os.MkdirAll(s.Dir(step.N), dirMode); err != nil { + return api.Step{}, fmt.Errorf("agentstep: %w", err) + } + return step, s.Write(step) +} + +// Write saves step's record, tmp+rename. +func (s Store) Write(step api.Step) error { + data, err := json.Marshal(step) + if err != nil { + return fmt.Errorf("agentstep: %w", err) + } + path := filepath.Join(s.Dir(step.N), RecordName) + tmp := path + ".tmp" + if err := os.WriteFile(tmp, data, fileMode); err != nil { + return fmt.Errorf("agentstep: %w", err) + } + if err := os.Rename(tmp, path); err != nil { + return fmt.Errorf("agentstep: %w", err) + } + return nil +} + +// Get is step n's saved record. +func (s Store) Get(n int) (api.Step, error) { + if n <= 0 { + return api.Step{}, ErrNotFound + } + data, err := os.ReadFile(filepath.Join(s.Dir(n), RecordName)) + if errors.Is(err, os.ErrNotExist) { + return api.Step{}, ErrNotFound + } + if err != nil { + return api.Step{}, fmt.Errorf("agentstep: %w", err) + } + var step api.Step + if err := json.Unmarshal(data, &step); err != nil { + return api.Step{}, fmt.Errorf("%w: step %d: %v", ErrCorrupt, n, err) + } + return step, nil +} + +// List is every step of the run, by number. A run with no steps yet is an +// empty list. A record that does not parse is reported after the ones +// that do, as ErrCorrupt, so a caller still sees the rest. +func (s Store) List() ([]api.Step, error) { + entries, err := os.ReadDir(filepath.Join(s.RunDir, StepsDir)) + if errors.Is(err, os.ErrNotExist) { + return []api.Step{}, nil + } + if err != nil { + return nil, fmt.Errorf("agentstep: %w", err) + } + var nums []int + for _, e := range entries { + if n, err := strconv.Atoi(e.Name()); err == nil && n > 0 && e.IsDir() { + nums = append(nums, n) + } + } + sort.Ints(nums) + steps := make([]api.Step, 0, len(nums)) + var corrupt error + for _, n := range nums { + step, err := s.Get(n) + switch { + case errors.Is(err, ErrNotFound): + // A directory without a record: a create that did not finish. + continue + case err != nil: + corrupt = err + continue + } + steps = append(steps, step) + } + return steps, corrupt +} + +// AppendTranscript adds one attempt line to step n's transcript. +func (s Store) AppendTranscript(n int, a attempt) error { + line, err := json.Marshal(a) + if err != nil { + return err + } + f, err := os.OpenFile(filepath.Join(s.Dir(n), TranscriptName), os.O_CREATE|os.O_APPEND|os.O_WRONLY, fileMode) + if err != nil { + return err + } + _, werr := f.Write(append(line, '\n')) + if cerr := f.Close(); werr == nil { + werr = cerr + } + return werr +} diff --git a/src/crux/api/runs.go b/src/crux/api/runs.go index fb42c54f57..dd017cc5d1 100644 --- a/src/crux/api/runs.go +++ b/src/crux/api/runs.go @@ -44,8 +44,9 @@ const ( // RunProfileExperiment (the default) runs with network none, no // credentials and time and output limits. RunProfileExperiment = "experiment" - // RunProfileAgent is S6's execution half and is not yet accepted - // (CodeUnsupported). + // RunProfileAgent opens a step session (S6): no command, network + // restricted by default, held waiting until cancelled; steps are + // posted to RunStepsPath. RunProfileAgent = "agent" ) diff --git a/src/crux/api/signals.go b/src/crux/api/signals.go index 2c9505eadd..a325825d13 100644 --- a/src/crux/api/signals.go +++ b/src/crux/api/signals.go @@ -47,8 +47,8 @@ func RunSignalsQuery(id string, sinceSeq int64, limit int) string { // AgentKickOutcomePath is the route of an agent's saved kick outcome. func AgentKickOutcomePath(name string) string { return AgentPath(name) + "/kick-outcome" } -// Signal kinds, the closed set every signal reports. Started and done are -// the runner's; the rest may come from the run's owner, and needs-input, +// Signal kinds, the closed set every signal reports. Started, done and step +// (SignalStep, steps.go) are the runner's; the rest may come from the run's owner, and needs-input, // input and blocked also from a gate opened for the run. const ( // SignalStarted: the sandbox was launched. diff --git a/src/crux/api/steps.go b/src/crux/api/steps.go new file mode 100644 index 0000000000..d17499d446 --- /dev/null +++ b/src/crux/api/steps.go @@ -0,0 +1,178 @@ +package api + +import ( + "context" + "encoding/json" + "net/http" + "strconv" + "time" +) + +// Schema-checked agent steps (hivecommons/crux#114 S6): run one agent turn +// in the workspace of an agent-profile contained run and get the answer +// back validated against a JSON shape. +// +// POST /v1/runs/{id}/steps run a step in the session's workspace +// GET /v1/runs/{id}/steps every step of the session, oldest first +// GET /v1/runs/{id}/steps/{n} one step's saved record +// +// A session is a run created with profile "agent" and no command: it +// provisions its workspace, reports "waiting" and stays open until it is +// cancelled, which is how a driver closes it. The routes admit the run's +// owner and the operator, like the run itself. Each step is appended to the +// run's command log (S4) and announced with a "step" turn signal (S5). + +// RunStepsPath is the route of one run's steps. +func RunStepsPath(id string) string { return RunPath(id) + "/steps" } + +// RunStepPath is the route of one step's saved record. +func RunStepPath(id string, n int) string { return RunStepsPath(id) + "/" + strconv.Itoa(n) } + +// Step statuses, the closed set every record reports. Every terminal status +// carries the attempts made, their usage and the transcript path, so a +// driver budgets and debugs from a failure as well as a success. +const ( + // StepRunning: the step is in flight. + StepRunning = "running" + // StepSucceeded: the answer matched the schema. + StepSucceeded = "succeeded" + // StepSchemaMismatch: every attempt's answer failed validation; the + // violations of the last one are on the record, no partial answer is. + StepSchemaMismatch = "schema_mismatch" + // StepTimeout: the backend did not finish within timeout_ms. + StepTimeout = "timeout" + // StepBackendError: the backend exited non-zero or its output was not + // what the adapter expects. + StepBackendError = "backend_error" + // StepToolDenied: the backend reported a tool outside the allowlist + // was requested and refused. + StepToolDenied = "tool_denied" + // StepTurnLimitExceeded: the backend stopped at the turn limit before + // answering. + StepTurnLimitExceeded = "turn_limit_exceeded" + // StepCancelled: the session was cancelled while the step ran. + StepCancelled = "cancelled" + // StepUnsupported: the request asked for something the backend cannot + // enforce (a tool allowlist, a turn limit); the step never ran. + StepUnsupported = "unsupported" +) + +// Step backends shipped with crux. Process is the generic adapter: the +// caller names the command, crux feeds the prompt on stdin and reads the +// answer from stdout; it enforces neither tools nor turns and says so. +const ( + StepBackendClaude = "claude" + StepBackendCodex = "codex" + StepBackendProcess = "process" +) + +// CodeStepBusy is a step posted while the session already runs one (409). +const CodeStepBusy = "step_busy" + +// CodeNotSession is a step posted to a run that is not an agent-profile +// session, or whose session has ended (409). +const CodeNotSession = "not_session" + +// SignalStep is the runner's turn signal for a finished step; Data is +// SignalStepData and the message the step's reason. +const SignalStep = "step" + +// SignalStepData is the data of a step signal. +type SignalStepData struct { + Step int `json:"step"` + Status string `json:"status"` + Attempts int `json:"attempts"` +} + +// StepRequest is POST /v1/runs/{id}/steps. +type StepRequest struct { + // Prompt is the step's instruction; PromptPath names a file in the + // workspace holding it instead (for large prompts). Exactly one. + Prompt string `json:"prompt,omitempty"` + PromptPath string `json:"prompt_path,omitempty"` + // Backend is StepBackendClaude, StepBackendCodex or StepBackendProcess. + Backend string `json:"backend"` + // Model, when set, is passed to the backend; empty is its default. + Model string `json:"model,omitempty"` + // Command is the process backend's argv, run in the workspace with the + // prompt on stdin; other backends refuse it. + Command []string `json:"command,omitempty"` + // OutputSchema is the JSON Schema the answer must match. Required. + OutputSchema json.RawMessage `json:"output_schema"` + // Tools is the allowlist of tools offered to the agent; empty is the + // backend's default set. A backend that cannot restrict its tools + // refuses a non-empty list (StepUnsupported). + Tools []string `json:"tools,omitempty"` + // TurnLimit stops the backend after this many turns; zero is the + // backend default. A backend that cannot enforce it refuses it. + TurnLimit int `json:"turn_limit,omitempty"` + // TimeoutMs bounds each attempt; zero uses the session's timeout. + TimeoutMs int64 `json:"timeout_ms,omitempty"` + // RetryOnMismatch is how many corrective attempts follow an answer that + // fails the schema; zero means one attempt only. + RetryOnMismatch int `json:"retry_on_mismatch,omitempty"` +} + +// StepUsage is the token usage a backend reported, summed over attempts. +type StepUsage struct { + InputTokens int64 `json:"input_tokens"` + OutputTokens int64 `json:"output_tokens"` +} + +// SchemaViolation is one way an answer missed the schema. +type SchemaViolation struct { + // Path is the JSON pointer-ish location ("$", "$.items[2].name"). + Path string `json:"path"` + Message string `json:"message"` +} + +// Step is the saved record of one agent step: the response of every steps +// route. Answer is present only on StepSucceeded; Violations only on +// StepSchemaMismatch. +type Step struct { + Run string `json:"run"` + N int `json:"n"` + Status string `json:"status"` + // Reason explains any status other than succeeded. + Reason string `json:"reason,omitempty"` + Backend string `json:"backend"` + Model string `json:"model,omitempty"` + CreatedAt time.Time `json:"created_at"` + FinishedAt *time.Time `json:"finished_at,omitempty"` + // Attempts is how many times the backend ran; each is one line of + // the transcript. + Attempts int `json:"attempts"` + Answer json.RawMessage `json:"answer,omitempty"` + Usage StepUsage `json:"usage"` + // TranscriptPath is the attempts' raw output, relative to the run's + // directory. + TranscriptPath string `json:"transcript_path,omitempty"` + Violations []SchemaViolation `json:"violations,omitempty"` +} + +// StepList is GET /v1/runs/{id}/steps. +type StepList struct { + Run string `json:"run"` + Steps []Step `json:"steps"` +} + +// RunStep is POST /v1/runs/{id}/steps: the finished step. +func (c *Client) RunStep(ctx context.Context, run string, req StepRequest) (Step, error) { + var out Step + err := c.Do(ctx, http.MethodPost, RunStepsPath(run), req, &out) + return out, err +} + +// ListSteps is GET /v1/runs/{id}/steps. +func (c *Client) ListSteps(ctx context.Context, run string) (StepList, error) { + var out StepList + err := c.Do(ctx, http.MethodGet, RunStepsPath(run), nil, &out) + return out, err +} + +// GetStep is GET /v1/runs/{id}/steps/{n}. +func (c *Client) GetStep(ctx context.Context, run string, n int) (Step, error) { + var out Step + err := c.Do(ctx, http.MethodGet, RunStepPath(run, n), nil, &out) + return out, err +} diff --git a/src/crux/cli/cli.go b/src/crux/cli/cli.go index 19a5bb209e..61e8058737 100644 --- a/src/crux/cli/cli.go +++ b/src/crux/cli/cli.go @@ -229,6 +229,10 @@ func (e *Env) dispatch(ctx context.Context, opts globalOptions, name string, arg // gates and its API) and read through that API. fmt.Fprintln(e.Stderr, "crux: signal needs --server ADDR (or CRUX_SERVER); turn signals are kept by a crux serve") return ExitUsage + case "step": + // Agent steps run inside a session a `crux serve` holds open. + fmt.Fprintln(e.Stderr, "crux: step needs --server ADDR (or CRUX_SERVER); agent steps run on a crux serve") + return ExitUsage case "help", "-h", "--help": fs := flag.NewFlagSet("crux", flag.ContinueOnError) e.usage(fs) @@ -373,6 +377,9 @@ Work services (no runtime; hivecommons/crux#114): signal emit|list a contained run's turn signals: where the turn is and how it ended (--server only; 'crux signal help' for flags) + step run|list|show schema-checked agent steps inside a + 'runs start --profile agent' session + (--server only; 'crux step help' for flags) While 'crux run' is in the foreground it serves /crux.sock; ps, logs, kick and stop use that socket when present so they act on the live runtime. diff --git a/src/crux/cli/remote.go b/src/crux/cli/remote.go index c16a638928..5576bec557 100644 --- a/src/crux/cli/remote.go +++ b/src/crux/cli/remote.go @@ -61,6 +61,8 @@ func (e *Env) remote(ctx context.Context, opts globalOptions, name string, args return e.remoteCmdlog(ctx, c, args) case "signal": return e.remoteSignal(ctx, c, args) + case "step": + return e.remoteStep(ctx, c, args) default: fmt.Fprintf(e.Stderr, "crux: %q is not available with --server; it runs where the runtime is\n", name) return ExitUsage diff --git a/src/crux/cli/runs.go b/src/crux/cli/runs.go index 483a78f601..9f0825b7d1 100644 --- a/src/crux/cli/runs.go +++ b/src/crux/cli/runs.go @@ -24,6 +24,9 @@ start flags: --timeout DURATION wall-clock limit (server default when unset) --max-output BYTES bytes of stdout/stderr returned (server default) --profile experiment|agent + agent opens a step session: no command, answered + as accepted, steps via 'crux step run', closed by + 'runs cancel' --idempotency-key KEY return the existing run for a repeated request --detach return the accepted record instead of waiting --json print the record as JSON` @@ -78,7 +81,9 @@ func (e *Env) remoteRunsStart(ctx context.Context, c *api.Client, args []string) if code, ok := parseSub(fs, args); !ok { return code } - if fs.NArg() == 0 { + // An agent session takes no command: its steps are posted with + // `crux step run`. + if fs.NArg() == 0 && req.Profile != api.RunProfileAgent { fmt.Fprintln(e.Stderr, runsUsage) return ExitUsage } diff --git a/src/crux/cli/step.go b/src/crux/cli/step.go new file mode 100644 index 0000000000..97efe60e87 --- /dev/null +++ b/src/crux/cli/step.go @@ -0,0 +1,224 @@ +package cli + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "strconv" + "strings" + + "github.com/hivecommons/hive/crux/api" + "github.com/hivecommons/hive/crux/config" +) + +// stepUsage is the `crux step` synopsis. +const stepUsage = `usage: crux --server ADDR step run --backend B (--prompt P | --prompt-file F) --schema FILE + [--model M] [--tool T]... [--turn-limit N] [--retry N] + [--timeout-ms N] [--json] [-- CMD ARGS...] + crux --server ADDR step list [--json] + crux --server ADDR step show [--json] + +run posts one agent step to an open agent-profile session (crux runs +start --profile agent) and waits for it: the backend (claude, codex or +process) reads the prompt inside the session's sandbox and its answer is +checked against the JSON Schema in --schema. --retry N re-asks up to N +times with the violations when the answer does not match. --prompt-file +is a path relative to the session's workspace. The process backend runs +the command after -- with the prompt on stdin and its stdout as the +answer. list prints one line per step; show prints one step with its +answer.` + +// remoteStep is `crux --server … step`: the agent-step family (RFC +// crux#114 S6). Steps run on crux serve; there is no in-process form. +func (e *Env) remoteStep(ctx context.Context, c *api.Client, args []string) int { + if len(args) == 0 { + fmt.Fprintln(e.Stderr, stepUsage) + return ExitUsage + } + switch args[0] { + case "run": + return e.remoteStepRun(ctx, c, args[1:]) + case "list": + return e.remoteStepList(ctx, c, args[1:]) + case "show": + return e.remoteStepShow(ctx, c, args[1:]) + case "help", "-h", "--help": + fmt.Fprintln(e.Stderr, stepUsage) + return ExitOK + default: + fmt.Fprintf(e.Stderr, "crux: step: unknown subcommand %q\n%s\n", args[0], stepUsage) + return ExitUsage + } +} + +func (e *Env) remoteStepRun(ctx context.Context, c *api.Client, args []string) int { + fs := e.subFlags("step run") + var req api.StepRequest + var tools stringList + var schemaFile string + fs.StringVar(&req.Backend, "backend", "", "claude, codex or process") + fs.StringVar(&req.Model, "model", "", "model the backend should use") + fs.StringVar(&req.Prompt, "prompt", "", "the prompt text") + fs.StringVar(&req.PromptPath, "prompt-file", "", "prompt file, relative to the session's workspace") + fs.StringVar(&schemaFile, "schema", "", "JSON Schema file the answer must match (required)") + fs.Var(&tools, "tool", "tool the backend may use (repeatable; none means no tools)") + fs.IntVar(&req.TurnLimit, "turn-limit", 0, "most agent turns per attempt (0: backend default)") + fs.IntVar(&req.RetryOnMismatch, "retry", 0, "re-ask up to N times when the answer misses the schema") + fs.Int64Var(&req.TimeoutMs, "timeout-ms", 0, "per-attempt wall clock in ms (server default when unset)") + asJSON := fs.Bool("json", false, "print the step as JSON") + var run string + if len(args) > 0 && !strings.HasPrefix(args[0], "-") { + run, args = args[0], args[1:] + } + if code, ok := parseSub(fs, args); !ok { + return code + } + rest := fs.Args() + if run == "" && len(rest) > 0 { + run, rest = rest[0], rest[1:] + } + // The flag package only consumes "--" while it is still parsing flags; + // after a bare it is ours to drop. + if len(rest) > 0 && rest[0] == "--" { + rest = rest[1:] + } + req.Command = rest + req.Tools = tools + if run == "" || req.Backend == "" || schemaFile == "" { + fmt.Fprintln(e.Stderr, stepUsage) + return ExitUsage + } + schema, err := os.ReadFile(schemaFile) + if err != nil { + fmt.Fprintf(e.Stderr, "crux: step run: --schema: %v\n", err) + return ExitUsage + } + if !json.Valid(schema) { + fmt.Fprintf(e.Stderr, "crux: step run: --schema %s is not valid JSON\n", schemaFile) + return ExitUsage + } + req.OutputSchema = json.RawMessage(schema) + step, err := c.RunStep(ctx, run, req) + if err != nil { + return e.stepErr("run", run, err) + } + if *asJSON { + return e.printJSON(step) + } + e.printStep(step) + if step.Status != api.StepSucceeded { + return ExitError + } + return ExitOK +} + +func (e *Env) remoteStepList(ctx context.Context, c *api.Client, args []string) int { + fs := e.subFlags("step list") + asJSON := fs.Bool("json", false, "print the steps as JSON") + run, code, ok := parseWithID(fs, args) + if !ok { + return code + } + if run == "" { + fmt.Fprintln(e.Stderr, stepUsage) + return ExitUsage + } + list, err := c.ListSteps(ctx, run) + if err != nil { + return e.stepErr("list", run, err) + } + if *asJSON { + return e.printJSON(list.Steps) + } + for _, step := range list.Steps { + fmt.Fprintln(e.Stdout, stepLine(step)) + } + return ExitOK +} + +func (e *Env) remoteStepShow(ctx context.Context, c *api.Client, args []string) int { + fs := e.subFlags("step show") + asJSON := fs.Bool("json", false, "print the step as JSON") + if code, ok := parseSub(fs, args); !ok { + return code + } + if fs.NArg() != 2 { + fmt.Fprintln(e.Stderr, stepUsage) + return ExitUsage + } + run := fs.Arg(0) + n, err := strconv.Atoi(fs.Arg(1)) + if err != nil || n <= 0 { + fmt.Fprintln(e.Stderr, "crux: step show: must be a positive integer") + return ExitUsage + } + step, err := c.GetStep(ctx, run, n) + if err != nil { + return e.stepErr("show", run, err) + } + if *asJSON { + return e.printJSON(step) + } + e.printStep(step) + return ExitOK +} + +// stepLine is the one-line form list prints. +func stepLine(step api.Step) string { + line := fmt.Sprintf("%d %s %s %s", step.N, step.CreatedAt.UTC().Format(timeLayout), step.Backend, step.Status) + if step.Attempts > 1 { + line += fmt.Sprintf(" (%d attempts)", step.Attempts) + } + if step.Reason != "" { + line += " " + step.Reason + } + return line +} + +// printStep is the multi-line form run and show print. +func (e *Env) printStep(step api.Step) { + fmt.Fprintf(e.Stdout, "step: %d\n", step.N) + fmt.Fprintf(e.Stdout, "run: %s\n", step.Run) + fmt.Fprintf(e.Stdout, "status: %s\n", step.Status) + if step.Reason != "" { + fmt.Fprintf(e.Stdout, "reason: %s\n", step.Reason) + } + backend := step.Backend + if step.Model != "" { + backend += " (" + step.Model + ")" + } + fmt.Fprintf(e.Stdout, "backend: %s\n", backend) + fmt.Fprintf(e.Stdout, "attempts: %d\n", step.Attempts) + fmt.Fprintf(e.Stdout, "created: %s\n", step.CreatedAt.UTC().Format(timeLayout)) + if step.FinishedAt != nil { + fmt.Fprintf(e.Stdout, "finished: %s\n", step.FinishedAt.UTC().Format(timeLayout)) + } + if step.Usage.InputTokens > 0 || step.Usage.OutputTokens > 0 { + fmt.Fprintf(e.Stdout, "tokens: %d in, %d out\n", step.Usage.InputTokens, step.Usage.OutputTokens) + } + for _, v := range step.Violations { + fmt.Fprintf(e.Stdout, "violation: %s: %s\n", v.Path, v.Message) + } + if len(step.Answer) > 0 { + fmt.Fprintf(e.Stdout, "answer: %s\n", string(step.Answer)) + } +} + +// stepErr prints a failure; refusals the caller can fix are usage +// errors, everything else a runtime failure. +func (e *Env) stepErr(verb, run string, err error) int { + fmt.Fprintf(e.Stderr, "crux: step %s %s: %v\n", verb, run, err) + var se *api.StatusError + if errors.As(err, &se) { + switch se.Body.Code { + case api.CodeBadRequest, api.CodeForbidden, api.CodeNotSession, api.CodeStepBusy: + return ExitUsage + } + } + if api.IsUnauthorized(err) { + fmt.Fprintf(e.Stderr, "crux: the server wants a bearer token; pass --token or set %s\n", config.ServeTokenEnvVar) + } + return ExitError +} diff --git a/src/crux/cli/step_test.go b/src/crux/cli/step_test.go new file mode 100644 index 0000000000..b9d1995d37 --- /dev/null +++ b/src/crux/cli/step_test.go @@ -0,0 +1,213 @@ +package cli + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/hivecommons/hive/crux/api" + "github.com/hivecommons/hive/crux/config" +) + +// fakeStepsAPI serves run_ok's steps and records each posted request. +func fakeStepsAPI(t *testing.T) (string, func() []api.StepRequest) { + t.Helper() + at := time.Date(2026, 11, 8, 12, 0, 0, 0, time.UTC) + fin := at.Add(time.Minute) + steps := []api.Step{ + {Run: "run_ok", N: 1, Status: api.StepSucceeded, Backend: "claude", Model: "opus", CreatedAt: at, FinishedAt: &fin, Attempts: 2, + Answer: json.RawMessage(`{"verdict":"pass"}`), Usage: api.StepUsage{InputTokens: 10, OutputTokens: 4}}, + {Run: "run_ok", N: 2, Status: api.StepSchemaMismatch, Backend: "process", CreatedAt: at, FinishedAt: &fin, Attempts: 1, + Reason: "the answer did not match output_schema after 1 attempt(s)", Violations: []api.SchemaViolation{{Path: "$.verdict", Message: "required property is missing"}}}, + } + var mu sync.Mutex + var posted []api.StepRequest + fail := func(w http.ResponseWriter, status int, code, msg string) { + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(api.Error{Error: msg, Code: code}) + } + mux := http.NewServeMux() + mux.HandleFunc("GET "+api.PathRuns+"/{id}/steps", func(w http.ResponseWriter, r *http.Request) { + if r.PathValue("id") != "run_ok" { + fail(w, http.StatusNotFound, api.CodeNotFound, "run not found") + return + } + _ = json.NewEncoder(w).Encode(api.StepList{Run: "run_ok", Steps: steps}) + }) + mux.HandleFunc("GET "+api.PathRuns+"/{id}/steps/{n}", func(w http.ResponseWriter, r *http.Request) { + if r.PathValue("id") != "run_ok" || r.PathValue("n") != "1" { + fail(w, http.StatusNotFound, api.CodeNotFound, "step not found") + return + } + _ = json.NewEncoder(w).Encode(steps[0]) + }) + mux.HandleFunc("POST "+api.PathRuns+"/{id}/steps", func(w http.ResponseWriter, r *http.Request) { + var in api.StepRequest + _ = json.NewDecoder(r.Body).Decode(&in) + mu.Lock() + posted = append(posted, in) + mu.Unlock() + switch { + case r.PathValue("id") == "run_busy": + fail(w, http.StatusConflict, api.CodeStepBusy, "the session is already running a step") + case r.PathValue("id") != "run_ok": + fail(w, http.StatusConflict, api.CodeNotSession, "not an open agent step session") + case in.Backend == "process": + w.WriteHeader(http.StatusCreated) + _ = json.NewEncoder(w).Encode(steps[1]) + default: + w.WriteHeader(http.StatusCreated) + _ = json.NewEncoder(w).Encode(steps[0]) + } + }) + ts := httptest.NewServer(mux) + t.Cleanup(ts.Close) + return ts.URL, func() []api.StepRequest { + mu.Lock() + defer mu.Unlock() + return append([]api.StepRequest(nil), posted...) + } +} + +func TestStepIsRemoteOnly(t *testing.T) { + t.Setenv(config.ServerEnvVar, "") + h := newHarness(t) + if code := h.run("step", "list", "run_ok"); code != ExitUsage || !strings.Contains(h.stderr.String(), "needs --server") { + t.Fatalf("local step: %d %s", code, h.stderr) + } + if len(h.rt.Calls()) != 0 { + t.Fatalf("step touched the local runtime: %v", h.rt.Calls()) + } + h = newHarness(t) + h.run("help") + if !strings.Contains(h.stderr.String(), "step run|list|show ") { + t.Fatalf("usage does not list step:\n%s", h.stderr) + } +} + +func TestStepRunPostsTheRequest(t *testing.T) { + t.Setenv(config.ServerEnvVar, "") + t.Setenv(config.ServeTokenEnvVar, "") + server, posted := fakeStepsAPI(t) + schema := filepath.Join(t.TempDir(), "schema.json") + if err := os.WriteFile(schema, []byte(`{"type":"object"}`), 0o600); err != nil { + t.Fatal(err) + } + remote := func(args ...string) (*harness, int) { + h := newHarness(t) + return h, h.run(append([]string{"--server", server, "step"}, args...)...) + } + + h, code := remote("run", "run_ok", "--backend", "claude", "--model", "opus", "--prompt", "Judge it", "--schema", schema, + "--tool", "Read", "--tool", "Grep", "--turn-limit", "3", "--retry", "1", "--timeout-ms", "5000") + if code != ExitOK { + t.Fatalf("run: %d %s", code, h.stderr) + } + out := h.stdout.String() + for _, want := range []string{"step: 1", "status: succeeded", "backend: claude (opus)", "attempts: 2", "tokens: 10 in, 4 out", `answer: {"verdict":"pass"}`, "finished: 2026-11-08T12:01:00.000Z"} { + if !strings.Contains(out, want) { + t.Errorf("run output lacks %q:\n%s", want, out) + } + } + reqs := posted() + if len(reqs) != 1 { + t.Fatalf("posted %d", len(reqs)) + } + got := reqs[0] + if got.Backend != "claude" || got.Model != "opus" || got.Prompt != "Judge it" || string(got.OutputSchema) != `{"type":"object"}` || + strings.Join(got.Tools, ",") != "Read,Grep" || got.TurnLimit != 3 || got.RetryOnMismatch != 1 || got.TimeoutMs != 5000 || len(got.Command) != 0 { + t.Fatalf("posted = %+v", got) + } + + // The process backend takes its command after --; a step that did not + // succeed exits non-zero and prints its violations. + h, code = remote("run", "--backend", "process", "--prompt-file", "task.md", "--schema", schema, "run_ok", "--", "python3", "ask.py") + if code != ExitError || !strings.Contains(h.stdout.String(), "status: schema_mismatch") || !strings.Contains(h.stdout.String(), "violation: $.verdict: required property is missing") { + t.Fatalf("process run: %d %s %s", code, h.stdout, h.stderr) + } + if got := posted()[1]; got.PromptPath != "task.md" || strings.Join(got.Command, " ") != "python3 ask.py" { + t.Fatalf("process posted = %+v", got) + } + h, code = remote("run", "run_ok", "--backend", "claude", "--prompt", "p", "--schema", schema, "--json") + var step api.Step + if err := json.Unmarshal(h.stdout.Bytes(), &step); code != ExitOK || err != nil || step.N != 1 { + t.Fatalf("run --json: %d %v %s", code, err, h.stdout) + } + + // Refusals the caller can fix are usage errors. + if h, code = remote("run", "run_ok", "--backend", "claude", "--prompt", "p"); code != ExitUsage { + t.Fatalf("no schema: %d %s", code, h.stderr) + } + if h, code = remote("run", "run_ok", "--backend", "claude", "--prompt", "p", "--schema", filepath.Join(t.TempDir(), "nope.json")); code != ExitUsage || !strings.Contains(h.stderr.String(), "--schema") { + t.Fatalf("missing schema file: %d %s", code, h.stderr) + } + bad := filepath.Join(t.TempDir(), "bad.json") + _ = os.WriteFile(bad, []byte("{"), 0o600) + if h, code = remote("run", "run_ok", "--backend", "claude", "--prompt", "p", "--schema", bad); code != ExitUsage || !strings.Contains(h.stderr.String(), "not valid JSON") { + t.Fatalf("bad schema file: %d %s", code, h.stderr) + } + if h, code = remote("run", "run_busy", "--backend", "claude", "--prompt", "p", "--schema", schema); code != ExitUsage || !strings.Contains(h.stderr.String(), "already running") { + t.Fatalf("busy: %d %s", code, h.stderr) + } + if h, code = remote("run", "run_exp", "--backend", "claude", "--prompt", "p", "--schema", schema); code != ExitUsage || !strings.Contains(h.stderr.String(), "not an open") { + t.Fatalf("not a session: %d %s", code, h.stderr) + } +} + +func TestStepListAndShow(t *testing.T) { + t.Setenv(config.ServerEnvVar, "") + t.Setenv(config.ServeTokenEnvVar, "") + server, _ := fakeStepsAPI(t) + remote := func(args ...string) (*harness, int) { + h := newHarness(t) + return h, h.run(append([]string{"--server", server, "step"}, args...)...) + } + h, code := remote("list", "run_ok") + want := "1 2026-11-08T12:00:00.000Z claude succeeded (2 attempts)\n" + + "2 2026-11-08T12:00:00.000Z process schema_mismatch the answer did not match output_schema after 1 attempt(s)\n" + if code != ExitOK || h.stdout.String() != want { + t.Fatalf("list: %d stdout=%q stderr=%q", code, h.stdout, h.stderr) + } + h, code = remote("list", "--json", "run_ok") + var got []api.Step + if err := json.Unmarshal(h.stdout.Bytes(), &got); code != ExitOK || err != nil || len(got) != 2 { + t.Fatalf("list --json: %d %v %s", code, err, h.stdout) + } + if h, code = remote("list", "run_missing"); code != ExitError || !strings.Contains(h.stderr.String(), "run not found") { + t.Fatalf("missing run: %d %s", code, h.stderr) + } + if h, code = remote("list"); code != ExitUsage { + t.Fatalf("list without a run: %d %s", code, h.stderr) + } + + h, code = remote("show", "run_ok", "1") + if code != ExitOK || !strings.Contains(h.stdout.String(), "step: 1\nrun: run_ok\nstatus: succeeded\n") { + t.Fatalf("show: %d %q", code, h.stdout) + } + h, code = remote("show", "--json", "run_ok", "1") + var step api.Step + if err := json.Unmarshal(h.stdout.Bytes(), &step); code != ExitOK || err != nil || step.N != 1 { + t.Fatalf("show --json: %d %v %s", code, err, h.stdout) + } + if h, code = remote("show", "run_ok", "2"); code != ExitError || !strings.Contains(h.stderr.String(), "step not found") { + t.Fatalf("show missing: %d %s", code, h.stderr) + } + if h, code = remote("show", "run_ok", "x"); code != ExitUsage { + t.Fatalf("show non-number: %d %s", code, h.stderr) + } + if h, code = remote("show", "run_ok"); code != ExitUsage { + t.Fatalf("show without n: %d %s", code, h.stderr) + } + if h, code = remote("bogus"); code != ExitUsage || !strings.Contains(h.stderr.String(), "unknown subcommand") { + t.Fatalf("bogus: %d %s", code, h.stderr) + } + if h, code = remote("help"); code != ExitOK || !strings.Contains(h.stderr.String(), "usage: crux --server ADDR step run") { + t.Fatalf("help: %d %s", code, h.stderr) + } +} diff --git a/src/crux/cli/verbs_test.go b/src/crux/cli/verbs_test.go index 1d2aa805f8..313f31efe9 100644 --- a/src/crux/cli/verbs_test.go +++ b/src/crux/cli/verbs_test.go @@ -15,8 +15,8 @@ import ( // nonRuntimeVerbs are dispatch cases that do not operate on a runtime: serve // exposes one, help prints usage, cline-turn executes one pane-side worker, -// checkout, runs, gate, cmdlog and signal drive work services on a server. -var nonRuntimeVerbs = map[string]bool{"serve": true, "help": true, "-h": true, "--help": true, "cline-turn": true, "checkout": true, "runs": true, "gate": true, "cmdlog": true, "signal": true} +// checkout, runs, gate, cmdlog, signal and step drive work services on a server. +var nonRuntimeVerbs = map[string]bool{"serve": true, "help": true, "-h": true, "--help": true, "cline-turn": true, "checkout": true, "runs": true, "gate": true, "cmdlog": true, "signal": true, "step": true} func TestVerbsHaveRuntimeMethods(t *testing.T) { rt := reflect.TypeOf((*crux.Runtime)(nil)).Elem() diff --git a/src/crux/runs/service.go b/src/crux/runs/service.go index 18a7bb69a2..fb524c2496 100644 --- a/src/crux/runs/service.go +++ b/src/crux/runs/service.go @@ -43,6 +43,11 @@ var ( ErrNotFound = runstate.ErrNotFound // ErrNoSignals: the server keeps no turn signals (Service.Signals nil). ErrNoSignals = errors.New("runs: turn signals are not enabled on this server") + // ErrNotSession: a step was posted to a run that is not an open + // agent-profile session. + ErrNotSession = errors.New("runs: not an open agent step session") + // ErrStepBusy: the session is already running a step. + ErrStepBusy = errors.New("runs: the session is already running a step") ) // RequestError is a malformed request (400). @@ -71,6 +76,7 @@ type detail struct { Command []string `json:"command"` Checkout string `json:"checkout,omitempty"` Workspace string `json:"workspace,omitempty"` + EnvAllowlist []string `json:"env_allowlist,omitempty"` ExitCode *int `json:"exit_code,omitempty"` TimedOut bool `json:"timed_out,omitempty"` Stdout string `json:"stdout,omitempty"` @@ -98,6 +104,13 @@ type active struct { cancelled bool // checkout is the S1 id the run holds open, if any (see InUse). checkout string + // The rest is the agent-profile session's (S6): ctx is what its steps + // run under, ready closes once the workspace is provisioned (readyErr + // says if that failed), and busy is held while a step runs. + ctx context.Context + ready chan struct{} + readyErr error + busy chan struct{} } // Caller is who asked for a run: the authenticated principal's name and @@ -177,7 +190,7 @@ func (s *Service) Create(launchCtx context.Context, caller Caller, req api.Conta return api.ContainedRun{}, false, err } owner := caller.Name - d := detail{Profile: p.profile, Network: p.network, Backend: s.Backend.Name, Command: p.command, Checkout: p.checkout, Workspace: p.workspace} + d := detail{Profile: p.profile, Network: p.network, Backend: s.Backend.Name, Command: p.command, Checkout: p.checkout, Workspace: p.workspace, EnvAllowlist: p.allowlist} raw, err := json.Marshal(d) if err != nil { return api.ContainedRun{}, false, err @@ -199,7 +212,12 @@ func (s *Service) Create(launchCtx context.Context, caller Caller, req api.Conta s.mu.Lock() s.active[rec.ID] = a s.mu.Unlock() - go s.execute(ctx, rec.ID, p, a) + if p.profile == api.RunProfileAgent { + a.ctx, a.ready, a.busy = ctx, make(chan struct{}), make(chan struct{}, 1) + go s.session(ctx, rec.ID, p, a) + } else { + go s.execute(ctx, rec.ID, p, a) + } return toWire(rec), true, nil } @@ -319,22 +337,31 @@ func (s *Service) Cancel(ctx context.Context, id string) (api.ContainedRun, erro func (s *Service) validate(caller Caller, req api.ContainedRunRequest) (plan, error) { var p plan - if len(req.Command) == 0 || strings.TrimSpace(req.Command[0]) == "" { - return p, &RequestError{"command is required"} - } - p.command = req.Command switch p.profile = req.Profile; p.profile { case "": p.profile = api.RunProfileExperiment case api.RunProfileExperiment: case api.RunProfileAgent: - return p, &UnsupportedError{fmt.Sprintf("profile %q is not available yet (schema-checked agent steps, S6)", req.Profile)} + // A session runs no command of its own: its steps are posted to it. + if len(req.Command) > 0 { + return p, &RequestError{fmt.Sprintf("profile %q takes no command; post steps to the session instead", req.Profile)} + } default: return p, &RequestError{fmt.Sprintf("profile must be %q or %q", api.RunProfileExperiment, api.RunProfileAgent)} } + if p.profile != api.RunProfileAgent && (len(req.Command) == 0 || strings.TrimSpace(req.Command[0]) == "") { + return p, &RequestError{"command is required"} + } + p.command = req.Command switch p.network = req.Network; p.network { case "": + // An agent needs its model, so a session defaults to the restricted + // network and is refused when the backend has none; an experiment + // defaults to none. p.network = api.RunNetworkNone + if p.profile == api.RunProfileAgent { + p.network = api.RunNetworkRestricted + } case api.RunNetworkNone, api.RunNetworkRestricted: default: return p, &RequestError{fmt.Sprintf("network must be %q or %q", api.RunNetworkNone, api.RunNetworkRestricted)} @@ -344,7 +371,7 @@ func (s *Service) validate(caller Caller, req api.ContainedRunRequest) (plan, er } for _, name := range req.EnvAllowlist { if sandbox.IsCredentialName(name) { - return p, &RequestError{fmt.Sprintf("env_allowlist names credential %q; the experiment profile carries no credentials", name)} + return p, &RequestError{fmt.Sprintf("env_allowlist names credential %q; contained runs carry no repository credentials", name)} } } p.allowlist = req.EnvAllowlist diff --git a/src/crux/runs/service_test.go b/src/crux/runs/service_test.go index 6c7ed81714..73cdb96750 100644 --- a/src/crux/runs/service_test.go +++ b/src/crux/runs/service_test.go @@ -444,7 +444,8 @@ func TestValidationErrors(t *testing.T) { }{ {"no command", api.ContainedRunRequest{WorkspacePath: ws}, &RequestError{}, "command is required"}, {"blank command", api.ContainedRunRequest{WorkspacePath: ws, Command: []string{" "}}, &RequestError{}, "command is required"}, - {"agent profile", api.ContainedRunRequest{WorkspacePath: ws, Command: []string{"x"}, Profile: "agent"}, &UnsupportedError{}, "S6"}, + {"agent with command", api.ContainedRunRequest{WorkspacePath: ws, Command: []string{"x"}, Profile: "agent"}, &RequestError{}, "takes no command"}, + {"agent needs restricted", api.ContainedRunRequest{WorkspacePath: ws, Profile: "agent"}, &UnsupportedError{}, `cannot enforce network "restricted"`}, {"bad profile", api.ContainedRunRequest{WorkspacePath: ws, Command: []string{"x"}, Profile: "root"}, &RequestError{}, "profile must be"}, {"bad network", api.ContainedRunRequest{WorkspacePath: ws, Command: []string{"x"}, Network: "host"}, &RequestError{}, "network must be"}, {"restricted unsupported", api.ContainedRunRequest{WorkspacePath: ws, Command: []string{"x"}, Network: "restricted"}, &UnsupportedError{}, `cannot enforce network "restricted"`}, diff --git a/src/crux/runs/steps.go b/src/crux/runs/steps.go new file mode 100644 index 0000000000..17856464ac --- /dev/null +++ b/src/crux/runs/steps.go @@ -0,0 +1,167 @@ +package runs + +import ( + "context" + "encoding/json" + "fmt" + "time" + + "github.com/hivecommons/hive/crux/agentstep" + "github.com/hivecommons/hive/crux/api" + "github.com/hivecommons/hive/crux/runstate" +) + +// session runs an agent-profile run (S6): it provisions the workspace, +// marks the run waiting and holds it open for steps until it is cancelled. +// The steps themselves run through RunStep under this goroutine's ctx. +func (s *Service) session(ctx context.Context, id string, p plan, a *active) { + defer func() { + s.mu.Lock() + delete(s.active, id) + s.mu.Unlock() + close(a.done) + }() + finish := func(status, reason string) { + _, _ = s.Store.Update(id, func(rec *runstate.Record) error { + rec.Status, rec.Reason = status, reason + return nil + }) + s.signalDone(id, api.SignalDoneData{Status: status}, reason) + } + workspace := p.workspace + if p.worktree != nil { + prov := s.Provisioner + if prov == nil { + prov = GitProvisioner{} + } + path, err := prov.Provision(ctx, s.Store.Path(id), *p.worktree) + if err != nil { + a.readyErr = ErrNotSession + close(a.ready) + if ctx.Err() != nil { + finish(runstate.StatusCancelled, "cancelled while provisioning the worktree") + } else { + finish(runstate.StatusFailed, "provisioning the worktree: "+s.scrub(err.Error())) + } + return + } + workspace = path + } + if _, err := s.Store.Update(id, func(rec *runstate.Record) error { + var d detail + _ = json.Unmarshal(rec.Detail, &d) + d.Workspace = workspace + raw, err := json.Marshal(d) + rec.Status, rec.Detail = runstate.StatusWaiting, raw + return err + }); err != nil { + a.readyErr = ErrNotSession + close(a.ready) + finish(runstate.StatusFailed, "recording the start: "+err.Error()) + return + } + s.signal(id, api.SignalStarted, "agent step session open", nil) + close(a.ready) + <-ctx.Done() + // Wait for a step in flight to record its own cancellation first. + a.busy <- struct{}{} + steps, _ := agentstep.Store{RunDir: s.Store.Path(id)}.List() + finish(runstate.StatusCancelled, fmt.Sprintf("session closed after %d step(s)", len(steps))) +} + +// sessionFor is the open session id belongs to, or ErrNotSession. +func (s *Service) sessionFor(ctx context.Context, id string) (*active, error) { + rec, err := s.Store.Get(id) + if err != nil { + return nil, err + } + s.mu.Lock() + a := s.active[id] + s.mu.Unlock() + if rec.Kind != runstate.KindRun || a == nil || a.ready == nil { + return nil, ErrNotSession + } + select { + case <-a.ready: + case <-ctx.Done(): + return nil, ctx.Err() + } + if a.readyErr != nil || a.ctx.Err() != nil { + return nil, ErrNotSession + } + return a, nil +} + +// RunStep executes one agent step inside session id and returns its +// record. One step runs at a time: a second concurrent post is ErrStepBusy. +// The step runs under the session's context, so cancelling the run ends it. +func (s *Service) RunStep(ctx context.Context, id string, req api.StepRequest) (api.Step, error) { + a, err := s.sessionFor(ctx, id) + if err != nil { + return api.Step{}, err + } + select { + case a.busy <- struct{}{}: + default: + return api.Step{}, ErrStepBusy + } + defer func() { <-a.busy }() + if a.ctx.Err() != nil { + return api.Step{}, ErrNotSession + } + rec, err := s.Store.Get(id) + if err != nil { + return api.Step{}, err + } + var d detail + _ = json.Unmarshal(rec.Detail, &d) + setStatus := func(status string) { + _, _ = s.Store.Update(id, func(rec *runstate.Record) error { + if runstate.Terminal(rec.Status) { + return nil + } + rec.Status = status + return nil + }) + } + setStatus(runstate.StatusRunning) + defer setStatus(runstate.StatusWaiting) + host := agentstep.Host{ + Run: id, RunDir: s.Store.Path(id), Workspace: d.Workspace, + Image: s.Config.Image, Network: d.Network, + Env: s.allowlistedEnv(d.EnvAllowlist), EnvAllowlist: d.EnvAllowlist, + Launcher: s.Backend.Launcher, Teardown: s.Backend.Teardown, + Timeout: time.Duration(s.Config.DefaultTimeoutMs) * time.Millisecond, + MaxTimeout: time.Duration(s.Config.MaxTimeoutMs) * time.Millisecond, + Signal: func(kind, msg string, data json.RawMessage) { s.signal(id, kind, msg, data) }, + Scrub: s.scrub, + } + if s.CmdLog != nil { + if log, err := s.CmdLog(host.RunDir); err == nil { + host.CmdLog = log + } + } + // A caller that goes away does not end the step; the session's cancel + // does. + return agentstep.Run(a.ctx, host, req) +} + +// Steps lists run id's steps, oldest first. +func (s *Service) Steps(id string) (api.StepList, error) { + if _, err := s.Store.Get(id); err != nil { + return api.StepList{}, err + } + steps, err := agentstep.Store{RunDir: s.Store.Path(id)}.List() + if err != nil { + return api.StepList{}, err + } + return api.StepList{Run: id, Steps: steps}, nil +} + +// Step is step n of run id. +func (s *Service) Step(id string, n int) (api.Step, error) { + if _, err := s.Store.Get(id); err != nil { + return api.Step{}, err + } + return agentstep.Store{RunDir: s.Store.Path(id)}.Get(n) +} diff --git a/src/crux/runs/steps_test.go b/src/crux/runs/steps_test.go new file mode 100644 index 0000000000..ba3708c0d8 --- /dev/null +++ b/src/crux/runs/steps_test.go @@ -0,0 +1,189 @@ +package runs + +import ( + "context" + "encoding/json" + "errors" + "slices" + "strings" + "testing" + "time" + + "github.com/hivecommons/hive/crux/agentstep" + "github.com/hivecommons/hive/crux/api" + "github.com/hivecommons/hive/crux/runstate" + "github.com/hivecommons/hive/crux/sandbox" +) + +const stepSchema = `{"type":"object","required":["verdict"],"properties":{"verdict":{"enum":["pass","fail"]}}}` + +// session opens an agent-profile run (network none, which the fake +// backend supports) and waits for it to be ready. +func (f *fixture) session() api.ContainedRun { + f.t.Helper() + run, created, err := f.svc.Create(context.Background(), operator, api.ContainedRunRequest{ + WorkspacePath: f.workspace(), Profile: api.RunProfileAgent, Network: api.RunNetworkNone, EnvAllowlist: []string{"KEEP"}, + }) + if err != nil || !created || run.Status != api.RunAccepted || run.Profile != api.RunProfileAgent { + f.t.Fatalf("Create session = %+v %v %v", run, created, err) + } + return run +} + +func stepReq(prompt string) api.StepRequest { + return api.StepRequest{Backend: "process", Prompt: prompt, OutputSchema: json.RawMessage(stepSchema), Command: []string{"./ask"}} +} + +func (f *fixture) cancel(id string) api.ContainedRun { + f.t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + run, err := f.svc.Cancel(ctx, id) + if err != nil { + f.t.Fatal(err) + } + return run +} + +func TestSessionRunsStepsUntilCancelled(t *testing.T) { + f := withCmdLog(withSignals(newFixture(t, newFakeLauncher(func(_ context.Context, _ sandbox.LaunchSpec) (sandbox.Result, error) { + return sandbox.Result{Stdout: `{"verdict":"pass"}`}, nil + })))) + run := f.session() + // RunStep waits for the session to be ready, so no status polling. + step, err := f.svc.RunStep(context.Background(), run.ID, stepReq("first")) + if err != nil || step.N != 1 || step.Status != api.StepSucceeded || step.Run != run.ID { + t.Fatalf("RunStep = %+v, %v", step, err) + } + spec := f.spec() + if spec.Name != "crux-step-"+run.ID+"-1-1" || spec.Workspace != f.workspace() || spec.NetworkMode != api.RunNetworkNone || spec.Env["KEEP"] != "yes" || spec.Env["GITHUB_TOKEN"] != "" || spec.Image != "img:1" { + t.Fatalf("spec = %+v", spec) + } + if got, err := f.svc.Get(run.ID); err != nil || got.Status != api.RunWaiting || got.Workdir != f.workspace() { + t.Fatalf("between steps: %+v, %v", got, err) + } + if step, err := f.svc.RunStep(context.Background(), run.ID, stepReq("second")); err != nil || step.N != 2 { + t.Fatalf("second RunStep = %+v, %v", step, err) + } + list, err := f.svc.Steps(run.ID) + if err != nil || list.Run != run.ID || len(list.Steps) != 2 || list.Steps[1].N != 2 { + t.Fatalf("Steps = %+v, %v", list, err) + } + if got, err := f.svc.Step(run.ID, 2); err != nil || got.Status != api.StepSucceeded { + t.Fatalf("Step = %+v, %v", got, err) + } + if _, err := f.svc.Step(run.ID, 3); !errors.Is(err, agentstep.ErrNotFound) { + t.Fatalf("Step(3) = %v", err) + } + if _, err := f.svc.Steps("run_missing"); !errors.Is(err, ErrNotFound) { + t.Fatalf("Steps(missing) = %v", err) + } + page, err := f.svc.Commands(run.ID, 0, 0) + if err != nil || len(page.Commands) != 2 || page.Commands[0].Status != api.CommandCompleted { + t.Fatalf("Commands = %+v, %v", page, err) + } + + closed := f.cancel(run.ID) + if closed.Status != api.RunCancelled || !strings.Contains(closed.Reason, "2 step(s)") { + t.Fatalf("closed = %+v", closed) + } + kinds, done, _ := f.kinds(run.ID) + if !slices.Equal(kinds, []string{"started/runner", "step/runner", "step/runner", "done/runner"}) || done.Status != api.RunCancelled { + t.Fatalf("signals = %v %+v", kinds, done) + } + if _, err := f.svc.RunStep(context.Background(), run.ID, stepReq("late")); !errors.Is(err, ErrNotSession) { + t.Fatalf("step after close = %v", err) + } + if again := f.cancel(run.ID); !again.AlreadyFinished { + t.Fatalf("second cancel = %+v", again) + } +} + +func TestCancelEndsTheStepInFlightAndASecondStepIsBusy(t *testing.T) { + f := withSignals(newFixture(t, blocking())) + run := f.session() + type result struct { + step api.Step + err error + } + results := make(chan result, 1) + go func() { + step, err := f.svc.RunStep(context.Background(), run.ID, stepReq("slow")) + results <- result{step, err} + }() + f.spec() + if got, err := f.svc.Get(run.ID); err != nil || got.Status != api.RunRunning { + t.Fatalf("during a step: %+v, %v", got, err) + } + if _, err := f.svc.RunStep(context.Background(), run.ID, stepReq("second")); !errors.Is(err, ErrStepBusy) { + t.Fatalf("concurrent step = %v", err) + } + closed := f.cancel(run.ID) + res := <-results + if res.err != nil || res.step.Status != api.StepCancelled { + t.Fatalf("step in flight = %+v, %v", res.step, res.err) + } + if closed.Status != api.RunCancelled || !strings.Contains(closed.Reason, "1 step(s)") { + t.Fatalf("closed = %+v", closed) + } + select { + case name := <-f.torn: + if name != "crux-step-"+run.ID+"-1-1" { + t.Fatalf("torn %q", name) + } + default: + t.Fatal("the step's launch was not torn down") + } +} + +func TestStepsNeedAnOpenSession(t *testing.T) { + f := newFixture(t, newFakeLauncher(func(_ context.Context, _ sandbox.LaunchSpec) (sandbox.Result, error) { + return sandbox.Result{}, nil + })) + exp := f.wait(f.start(api.ContainedRunRequest{WorkspacePath: f.workspace(), Command: []string{"x"}}).ID) + if _, err := f.svc.RunStep(context.Background(), exp.ID, stepReq("p")); !errors.Is(err, ErrNotSession) { + t.Fatalf("step on an experiment = %v", err) + } + if _, err := f.svc.RunStep(context.Background(), "run_missing", stepReq("p")); !errors.Is(err, ErrNotFound) { + t.Fatalf("step on a missing run = %v", err) + } + if list, err := f.svc.Steps(exp.ID); err != nil || len(list.Steps) != 0 { + t.Fatalf("Steps of an experiment = %+v, %v", list, err) + } + // A malformed step is refused without a record; an unsupported one is + // recorded and refused. + run := f.session() + defer f.cancel(run.ID) + var reqErr *agentstep.RequestError + if _, err := f.svc.RunStep(context.Background(), run.ID, api.StepRequest{Backend: "process"}); !errors.As(err, &reqErr) { + t.Fatalf("malformed step = %v", err) + } + req := stepReq("p") + req.Tools = []string{"Bash"} + var unsupported *agentstep.UnsupportedError + step, err := f.svc.RunStep(context.Background(), run.ID, req) + if !errors.As(err, &unsupported) || step.N != 1 || step.Status != api.StepUnsupported { + t.Fatalf("unsupported step = %+v, %v", step, err) + } + if got, err := f.svc.Get(run.ID); err != nil || got.Status != api.RunWaiting { + t.Fatalf("after a refused step: %+v, %v", got, err) + } +} + +func TestReconcileClosesAWaitingSession(t *testing.T) { + f := newFixture(t, blocking()) + rec, _, err := f.svc.Store.Create(runstate.Record{Owner: "operator"}) + if err != nil { + t.Fatal(err) + } + if _, err := f.svc.Store.Update(rec.ID, func(r *runstate.Record) error { r.Status = runstate.StatusWaiting; return nil }); err != nil { + t.Fatal(err) + } + changed, err := f.svc.Reconcile() + if err != nil || len(changed) != 1 || changed[0].Status != runstate.StatusUnfinished { + t.Fatalf("Reconcile = %+v, %v", changed, err) + } + if _, err := f.svc.RunStep(context.Background(), rec.ID, stepReq("p")); !errors.Is(err, ErrNotSession) { + t.Fatalf("step on a reconciled run = %v", err) + } +} diff --git a/src/crux/runstate/runstate.go b/src/crux/runstate/runstate.go index 97771f2e95..3115d41539 100644 --- a/src/crux/runstate/runstate.go +++ b/src/crux/runstate/runstate.go @@ -46,7 +46,8 @@ const ( RecordName = "record.json" // IndexName is the append-only status history, one IndexEntry per line. IndexName = "index.jsonl" - // KindRun is a contained run (S2); S6 steps add their own kind later. + // KindRun is a contained run (S2) or, with profile agent, a step + // session (S6); steps are kept under the run's directory by agentstep. KindRun = "run" idRandomBytes = 6 @@ -263,8 +264,8 @@ func (s *Store) List() ([]Record, error) { return out, nil } -// Reconcile moves every accepted or running record to StatusUnfinished with -// reason and returns the records it changed. It runs at `crux serve` start: +// Reconcile moves every accepted, running or waiting record to +// StatusUnfinished with reason and returns the records it changed. It runs at `crux serve` start: // the process that owned those items is gone, so their exit is not knowable // and they are never restarted silently. func (s *Store) Reconcile(reason string) ([]Record, error) { @@ -274,7 +275,7 @@ func (s *Store) Reconcile(reason string) ([]Record, error) { } var changed []Record for _, r := range recs { - if r.Status != StatusAccepted && r.Status != StatusRunning { + if r.Status != StatusAccepted && r.Status != StatusRunning && r.Status != StatusWaiting { continue } rec, err := s.Update(r.ID, func(rec *Record) error { diff --git a/src/crux/serve/runs.go b/src/crux/serve/runs.go index a0278077ba..0d875ef840 100644 --- a/src/crux/serve/runs.go +++ b/src/crux/serve/runs.go @@ -87,7 +87,10 @@ func (s *Server) handleRunsCreate(w http.ResponseWriter, r *http.Request) { s.writeJSON(w, http.StatusOK, run) return } - if req.Detach { + // A step session (profile agent) stays open until it is cancelled, so + // it is always answered as accepted; its steps are what the caller waits + // on. + if req.Detach || run.Profile == api.RunProfileAgent { s.writeJSON(w, http.StatusAccepted, run) return } diff --git a/src/crux/serve/runs_test.go b/src/crux/serve/runs_test.go index bedfb47ed2..afecd70745 100644 --- a/src/crux/serve/runs_test.go +++ b/src/crux/serve/runs_test.go @@ -122,7 +122,8 @@ func TestRunsErrorCodes(t *testing.T) { }) f.expectCode(http.MethodPost, api.PathRuns, "{not json", http.StatusBadRequest, api.CodeBadRequest) f.expectCode(http.MethodPost, api.PathRuns, api.ContainedRunRequest{WorkspacePath: ws}, http.StatusBadRequest, api.CodeBadRequest) - f.expectCode(http.MethodPost, api.PathRuns, api.ContainedRunRequest{WorkspacePath: ws, Command: []string{"x"}, Profile: api.RunProfileAgent}, http.StatusNotImplemented, api.CodeUnsupported) + f.expectCode(http.MethodPost, api.PathRuns, api.ContainedRunRequest{WorkspacePath: ws, Command: []string{"x"}, Profile: api.RunProfileAgent}, http.StatusBadRequest, api.CodeBadRequest) + f.expectCode(http.MethodPost, api.PathRuns, api.ContainedRunRequest{WorkspacePath: ws, Profile: api.RunProfileAgent}, http.StatusNotImplemented, api.CodeUnsupported) f.expectCode(http.MethodPost, api.PathRuns, api.ContainedRunRequest{WorkspacePath: ws, Command: []string{"x"}, Network: api.RunNetworkRestricted}, http.StatusNotImplemented, api.CodeUnsupported) f.expectCode(http.MethodPost, api.PathRuns, api.ContainedRunRequest{Checkout: "chk_1", Command: []string{"x"}}, http.StatusNotImplemented, api.CodeUnsupported) f.expectCode(http.MethodPost, api.PathRuns, api.ContainedRunRequest{WorkspacePath: "/elsewhere", Command: []string{"x"}}, http.StatusForbidden, api.CodeForbidden) diff --git a/src/crux/serve/serve.go b/src/crux/serve/serve.go index 3f7bae2343..1f4e18f1b3 100644 --- a/src/crux/serve/serve.go +++ b/src/crux/serve/serve.go @@ -183,6 +183,7 @@ func (s *Server) Handler() http.Handler { s.registerFamilies(mux) s.registerCheckouts(mux) s.registerRuns(mux) + s.registerSteps(mux) s.registerGates(mux) s.registerSignals(mux) return s.laneHandler(s.authenticate(instrument(mux))) diff --git a/src/crux/serve/steps.go b/src/crux/serve/steps.go new file mode 100644 index 0000000000..41bccf2b97 --- /dev/null +++ b/src/crux/serve/steps.go @@ -0,0 +1,102 @@ +package serve + +import ( + "context" + "errors" + "net/http" + "strconv" + + "github.com/hivecommons/hive/crux/agentstep" + "github.com/hivecommons/hive/crux/api" + "github.com/hivecommons/hive/crux/runs" +) + +// Agent step routes (S6). A step is posted to an open agent-profile run +// and answers when the step has finished; the step runs under the +// session, not the request, so a caller that goes away leaves it in +// flight and reads it back with GET. +func (s *Server) registerSteps(mux *http.ServeMux) { + mux.HandleFunc("POST "+api.PathRuns+"/{id}/steps", s.handleStepsRun) + mux.HandleFunc("GET "+api.PathRuns+"/{id}/steps", s.handleStepsList) + mux.HandleFunc("GET "+api.PathRuns+"/{id}/steps/{n}", s.handleStepsGet) +} + +func (s *Server) handleStepsRun(w http.ResponseWriter, r *http.Request) { + if !s.runsReady(w) { + return + } + run, ok := s.ownedRun(w, r) + if !ok { + return + } + var req api.StepRequest + if !s.readJSON(w, r, &req) { + return + } + step, err := s.Runs.RunStep(r.Context(), run.ID, req) + if err != nil { + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return + } + s.writeStepsError(w, err) + return + } + s.writeJSON(w, http.StatusCreated, step) +} + +func (s *Server) handleStepsList(w http.ResponseWriter, r *http.Request) { + if !s.runsReady(w) { + return + } + run, ok := s.ownedRun(w, r) + if !ok { + return + } + list, err := s.Runs.Steps(run.ID) + if err != nil { + s.writeStepsError(w, err) + return + } + s.writeJSON(w, http.StatusOK, list) +} + +func (s *Server) handleStepsGet(w http.ResponseWriter, r *http.Request) { + if !s.runsReady(w) { + return + } + run, ok := s.ownedRun(w, r) + if !ok { + return + } + n, err := strconv.Atoi(r.PathValue("n")) + if err != nil || n <= 0 { + s.writeError(w, http.StatusBadRequest, api.CodeBadRequest, "step number must be a positive integer") + return + } + step, err := s.Runs.Step(run.ID, n) + if err != nil { + s.writeStepsError(w, err) + return + } + s.writeJSON(w, http.StatusOK, step) +} + +func (s *Server) writeStepsError(w http.ResponseWriter, err error) { + var reqErr *agentstep.RequestError + var unsupported *agentstep.UnsupportedError + switch { + case errors.Is(err, agentstep.ErrNotFound): + s.writeError(w, http.StatusNotFound, api.CodeNotFound, "step not found") + case errors.Is(err, runs.ErrNotSession): + s.writeError(w, http.StatusConflict, api.CodeNotSession, err.Error()) + case errors.Is(err, runs.ErrStepBusy): + s.writeError(w, http.StatusConflict, api.CodeStepBusy, err.Error()) + case errors.As(err, &reqErr): + s.writeError(w, http.StatusBadRequest, api.CodeBadRequest, err.Error()) + case errors.As(err, &unsupported): + // The refusal is also on the record: GET steps/{n} shows it. + s.writeError(w, http.StatusNotImplemented, api.CodeUnsupported, err.Error()) + default: + s.writeRunsError(w, err) + } +} diff --git a/src/crux/serve/steps_test.go b/src/crux/serve/steps_test.go new file mode 100644 index 0000000000..11155e6a63 --- /dev/null +++ b/src/crux/serve/steps_test.go @@ -0,0 +1,134 @@ +package serve + +import ( + "context" + "encoding/json" + "net/http" + "os" + "path/filepath" + "testing" + + "github.com/hivecommons/hive/crux/api" + "github.com/hivecommons/hive/crux/runs" + "github.com/hivecommons/hive/crux/runstate" + "github.com/hivecommons/hive/crux/sandbox" +) + +// withHeldRuns is withRuns with the launched channel exposed: each launch +// reports on it and then holds until cleanup or its context ends. +func (f *gateFixture) withHeldRuns() (string, <-chan struct{}) { + f.t.Helper() + store, err := runstate.Open(f.t.TempDir()) + if err != nil { + f.t.Fatal(err) + } + root := f.t.TempDir() + ws := filepath.Join(root, "proj") + if err := os.MkdirAll(ws, 0o755); err != nil { + f.t.Fatal(err) + } + launched, release := make(chan struct{}, 8), make(chan struct{}) + f.t.Cleanup(func() { close(release) }) + backend := &runs.Backend{Name: "fake", Launcher: holdingLauncher(launched, release), Capabilities: runs.Capabilities{NetworkNone: true}} + f.s.Runs = runs.New(store, backend, runs.Config{Image: "img", WorkspaceRoots: []string{root}}) + f.h = f.s.Handler() + return ws, launched +} + +const stepSchema = `{"type":"object","required":["verdict"]}` + +func stepReq(prompt string) api.StepRequest { + return api.StepRequest{Backend: "process", Prompt: prompt, OutputSchema: json.RawMessage(stepSchema), Command: []string{"./ask"}} +} + +func TestStepsRoutesAnswerNotSupportedWithoutAService(t *testing.T) { + f := newFamServer(t) + f.expectCode(http.MethodPost, api.RunStepsPath("run_x"), stepReq("p"), http.StatusNotImplemented, api.CodeNotSupported) + f.expectCode(http.MethodGet, api.RunStepsPath("run_x"), nil, http.StatusNotImplemented, api.CodeNotSupported) + f.expectCode(http.MethodGet, api.RunStepPath("run_x", 1), nil, http.StatusNotImplemented, api.CodeNotSupported) +} + +func TestSessionStepsOverHTTP(t *testing.T) { + f, ws := newRunsServer(t, func(_ context.Context, _ sandbox.LaunchSpec) (sandbox.Result, error) { + return sandbox.Result{Stdout: `{"verdict":"pass"}`}, nil + }) + // A session is answered as accepted whether or not detach was asked. + var run api.ContainedRun + f.expect(http.MethodPost, api.PathRuns, api.ContainedRunRequest{WorkspacePath: ws, Profile: api.RunProfileAgent, Network: api.RunNetworkNone}, http.StatusAccepted, &run) + if run.Status != api.RunAccepted || run.Profile != api.RunProfileAgent { + t.Fatalf("session = %+v", run) + } + var step api.Step + f.expect(http.MethodPost, api.RunStepsPath(run.ID), stepReq("first"), http.StatusCreated, &step) + if step.N != 1 || step.Status != api.StepSucceeded || string(step.Answer) != `{"verdict":"pass"}` { + t.Fatalf("step = %+v", step) + } + f.expectCode(http.MethodPost, api.RunStepsPath(run.ID), api.StepRequest{Backend: "process"}, http.StatusBadRequest, api.CodeBadRequest) + f.expectCode(http.MethodPost, api.RunStepsPath(run.ID), "{", http.StatusBadRequest, api.CodeBadRequest) + unsupported := stepReq("p") + unsupported.Tools = []string{"Bash"} + f.expectCode(http.MethodPost, api.RunStepsPath(run.ID), unsupported, http.StatusNotImplemented, api.CodeUnsupported) + + var list api.StepList + f.expect(http.MethodGet, api.RunStepsPath(run.ID), nil, http.StatusOK, &list) + if list.Run != run.ID || len(list.Steps) != 2 || list.Steps[1].Status != api.StepUnsupported { + t.Fatalf("list = %+v", list) + } + var got api.Step + f.expect(http.MethodGet, api.RunStepPath(run.ID, 1), nil, http.StatusOK, &got) + if got.N != 1 || got.Status != api.StepSucceeded { + t.Fatalf("get = %+v", got) + } + f.expectCode(http.MethodGet, api.RunStepPath(run.ID, 9), nil, http.StatusNotFound, api.CodeNotFound) + f.expectCode(http.MethodGet, api.PathRuns+"/"+run.ID+"/steps/zero", nil, http.StatusBadRequest, api.CodeBadRequest) + f.expectCode(http.MethodGet, api.RunStepsPath("run_000000000000"), nil, http.StatusNotFound, api.CodeNotFound) + + var closed api.ContainedRun + f.expect(http.MethodPost, api.RunCancelPath(run.ID), nil, http.StatusOK, &closed) + if closed.Status != api.RunCancelled { + t.Fatalf("closed = %+v", closed) + } + f.expectCode(http.MethodPost, api.RunStepsPath(run.ID), stepReq("late"), http.StatusConflict, api.CodeNotSession) + + // A finished experiment is not a session either. + var exp api.ContainedRun + f.expect(http.MethodPost, api.PathRuns, api.ContainedRunRequest{WorkspacePath: ws, Command: []string{"x"}}, http.StatusOK, &exp) + f.expectCode(http.MethodPost, api.RunStepsPath(exp.ID), stepReq("p"), http.StatusConflict, api.CodeNotSession) +} + +func TestConcurrentStepIsBusyAndStepsAreOwned(t *testing.T) { + f := newGateFixture(t) + ws, launched := f.withHeldRuns() + var run api.ContainedRun + if code, e := f.call("atlas", http.MethodPost, api.PathRuns, api.ContainedRunRequest{WorkspacePath: ws, Profile: api.RunProfileAgent, Network: api.RunNetworkNone}, &run); code != http.StatusAccepted { + t.Fatalf("session: %d %+v", code, e) + } + if code, e := f.call("beacon", http.MethodGet, api.RunStepsPath(run.ID), nil, nil); code != http.StatusForbidden || e.Code != api.CodeForbidden { + t.Fatalf("beacon on atlas's session: %d %+v", code, e) + } + if code, e := f.call("beacon", http.MethodPost, api.RunStepsPath(run.ID), stepReq("p"), nil); code != http.StatusForbidden || e.Code != api.CodeForbidden { + t.Fatalf("beacon posting to atlas's session: %d %+v", code, e) + } + // The holding launcher keeps the first step in flight until cleanup. + done := make(chan struct{}) + go func() { + defer close(done) + f.call("atlas", http.MethodPost, api.RunStepsPath(run.ID), stepReq("slow"), nil) + }() + <-launched + if code, e := f.call("atlas", http.MethodPost, api.RunStepsPath(run.ID), stepReq("second"), nil); code != http.StatusConflict || e.Code != api.CodeStepBusy { + t.Fatalf("concurrent step: %d %+v", code, e) + } + var list api.StepList + if code, _ := f.call("operator", http.MethodGet, api.RunStepsPath(run.ID), nil, &list); code != http.StatusOK || len(list.Steps) != 1 || list.Steps[0].Status != api.StepRunning { + t.Fatalf("list during a step: %d %+v", code, list) + } + if code, _ := f.call("atlas", http.MethodPost, api.RunCancelPath(run.ID), nil, nil); code != http.StatusOK { + t.Fatalf("cancel: %d", code) + } + <-done + var step api.Step + if code, _ := f.call("atlas", http.MethodGet, api.RunStepPath(run.ID, 1), nil, &step); code != http.StatusOK || step.Status != api.StepCancelled { + t.Fatalf("cancelled step: %d %+v", code, step) + } +} diff --git a/src/crux/signals/signals.go b/src/crux/signals/signals.go index de0242802b..762f3c69ae 100644 --- a/src/crux/signals/signals.go +++ b/src/crux/signals/signals.go @@ -65,15 +65,16 @@ var ( // kinds is the closed kind set. var kinds = map[string]bool{ api.SignalStarted: true, api.SignalThinking: true, api.SignalTool: true, api.SignalBlocked: true, - api.SignalNeedsInput: true, api.SignalInput: true, api.SignalDone: true, + api.SignalNeedsInput: true, api.SignalInput: true, api.SignalDone: true, api.SignalStep: true, } // bySource is what each source may append. The runner owns the run's -// boundaries; a gate reports a person being asked, answering or not -// answering in time; the caller reports everything in between but never a -// boundary, so a done signal always means the runner saw the run end. +// boundaries and the steps an agent session ran (S6); a gate reports a +// person being asked, answering or not answering in time; the caller +// reports everything in between but never a boundary, so a done signal +// always means the runner saw the run end. var bySource = map[string]map[string]bool{ - api.SignalSourceRunner: {api.SignalStarted: true, api.SignalDone: true}, + api.SignalSourceRunner: {api.SignalStarted: true, api.SignalDone: true, api.SignalStep: true}, api.SignalSourceGate: {api.SignalNeedsInput: true, api.SignalInput: true, api.SignalBlocked: true}, api.SignalSourceCaller: {api.SignalThinking: true, api.SignalTool: true, api.SignalBlocked: true, api.SignalNeedsInput: true, api.SignalInput: true}, } diff --git a/src/crux/signals/signals_test.go b/src/crux/signals/signals_test.go index 546a1e0ac7..004bf951a8 100644 --- a/src/crux/signals/signals_test.go +++ b/src/crux/signals/signals_test.go @@ -125,7 +125,10 @@ func TestSourcesAppendOnlyTheirKinds(t *testing.T) { }{ {api.SignalSourceRunner, api.SignalStarted, true}, {api.SignalSourceRunner, api.SignalDone, true}, + {api.SignalSourceRunner, api.SignalStep, true}, {api.SignalSourceRunner, api.SignalThinking, false}, + {api.SignalSourceGate, api.SignalStep, false}, + {api.SignalSourceCaller, api.SignalStep, false}, {api.SignalSourceGate, api.SignalNeedsInput, true}, {api.SignalSourceGate, api.SignalInput, true}, {api.SignalSourceGate, api.SignalBlocked, true}, diff --git a/src/docs/design/crux-work-services.md b/src/docs/design/crux-work-services.md index 366a073f0e..8e9f6734c8 100644 --- a/src/docs/design/crux-work-services.md +++ b/src/docs/design/crux-work-services.md @@ -1,7 +1,7 @@ # Crux work services: S1 pinned checkout (hivecommons/crux#114) Status: **S0 caller principals and S1 pinned checkout shipped in hive -`src/crux` (v5).** S2 contained run and S7 minimum shipped in #17014; S3 human gate is [below](#s3-human-gate); S4 command log is [below](#s4-command-log); S5 turn signals are [below](#s5-turn-signals); everything else in the RFC is unscheduled. This records +`src/crux` (v5).** S2 contained run and S7 minimum shipped in #17014; S3 human gate is [below](#s3-human-gate); S4 command log is [below](#s4-command-log); S5 turn signals are [below](#s5-turn-signals); S6 agent step is [below](#s6-agent-step); everything else in the RFC is unscheduled. This records S1 *as implemented*, including where it is weaker than the RFC text. The RFC asks for "a working copy of this repo at this commit, owned by @@ -363,12 +363,12 @@ signals.NewBook(signals.Policy{})` and `Server.KickOutcomes` on A `Signal` is `{seq, kind, source, at, message?, data?}`. `kind` is the closed set `started | thinking | tool | blocked | needs-input | input | -done`; `source` is `runner`, `gate` or `caller`, and each source appends -only its own kinds: +step | done`; `source` is `runner`, `gate` or `caller`, and each source +appends only its own kinds: | Source | Kinds | When | |---|---|---| -| `runner` | `started`, `done` | `started` just before the sandbox launcher runs the command; `done` on every terminal path — completed, failed, timed out, cancelled, a provisioning failure (no `started`), and the run `crux serve` marks `unfinished` at start because it was in flight at the last shutdown. `done.data` is `{"status", "exit_code"?, "timed_out"?}`, its message the run's reason. | +| `runner` | `started`, `step`, `done` | `started` just before the sandbox launcher runs the command (for an agent session, once its workspace is ready); `step` once per finished agent step (S6, `data` `{"step", "status", "attempts"}`, message the step's reason); `done` on every terminal path — completed, failed, timed out, cancelled, a provisioning failure (no `started`), and the run `crux serve` marks `unfinished` at start because it was in flight at the last shutdown. `done.data` is `{"status", "exit_code"?, "timed_out"?}`, its message the run's reason. | | `gate` | `needs-input`, `input`, `blocked` | a gate opened with `run`: opening is `needs-input` (message the subject), answering `input` (`data` `{"gate", "answer", "answered_by"}`), expiry `blocked`. `data.gate` names the gate; its note and context never ride along. | | `caller` | `thinking`, `tool`, `blocked`, `needs-input`, `input` | `POST /v1/runs/{id}/signals` by the run's owner or the operator. | @@ -436,6 +436,145 @@ from inside the sandbox other than through the owner's API; no per-agent signal log beyond the last kick outcome; no retention policy — a run's signals live and die with the run dir. +## S6. Agent step + +One step = one agent backend run inside a sandbox, in a prepared +checkout, with a prompt in and a schema-checked JSON answer out. Package +`src/crux/agentstep` (`Run`, `Host`, `Backend`, `Backends`, `Store`, +`ParseSchema`, `CorrectivePrompt`), API types in `src/crux/api/steps.go`, +routes in `src/crux/serve/steps.go`, the session in +`src/crux/runs/service.go`, CLI in `src/crux/cli/step.go`. No new +configuration: a step runs on the S2 backend and image with the S2 +timeouts (`default_timeout_ms` per attempt, `max_timeout_ms` the cap). + +**Session.** A step needs a sandbox that already holds the checkout, so +`POST /v1/runs` with `"profile": "agent"` opens a *step session* instead +of running a command: `command` must be empty (400), the source is any +S2 source (checkout, worktree or workspace_path), `network` defaults to +`restricted` because the agent must reach its model (a backend without +that capability answers 501 `unsupported`; a caller may ask `none` for a +process backend that needs nothing), and `env_allowlist` still refuses +credential names — a session carries no repository-write authority, as +the RFC asks. The response is always 202 with the `accepted` record, +`detach` or not: the session has no end of its own. Its runner +provisions the workspace, moves the record to `waiting`, signals +`started` and holds until `POST /v1/runs/{id}/cancel` closes it +(`cancelled`, reason `session closed after N step(s)`, `done` signal). +While a step runs the record is `running`; between steps `waiting`. +`Reconcile` at start marks a `waiting` session `unfinished` like any run +left in flight. One step at a time: a second concurrent post is 409 +`step_busy`; a step on a run that is not an open session (an experiment, +a closed or reconciled session, a session whose provisioning failed) is +409 `not_session`. A step posted while the session is still provisioning +waits for it. + +| Route | Body | Response | +|---|---|---| +| `POST /v1/runs/{id}/steps` | `StepRequest` | `201 Step` once the step has finished, whatever its status | +| `GET /v1/runs/{id}/steps` | | `200 {"run", "steps":[Step…]}` oldest first; a step in flight shows as `running` | +| `GET /v1/runs/{id}/steps/{n}` | | `200 Step` | + +`StepRequest` is `{backend, prompt | prompt_path, output_schema, model?, +tools?, turn_limit?, timeout_ms?, retry_on_mismatch?, command?}`. +`prompt` is inline (≤ 1 MiB); `prompt_path` is relative to the session +workspace and must stay inside it. `output_schema` is a JSON Schema the +answer must match; `tools` is the allowlist of tools the agent may use +(empty: the backend's default, which for `claude` is every tool); +`turn_limit` bounds agent turns per attempt; `retry_on_mismatch` (0–5) +re-asks when the answer misses the schema; `command` is the process +backend's. A `Step` is `{run, n, status, reason?, backend, model?, +created_at, finished_at?, attempts, answer?, usage{input_tokens, +output_tokens}, transcript_path?, violations?}`. `status` is the RFC's +closed set: `succeeded | schema_mismatch | timeout | backend_error | +tool_denied | turn_limit_exceeded | cancelled | unsupported`, plus +`running` while in flight. `answer` is present only on `succeeded`. + +**Execution.** The service writes the prompt to +`/.crux/steps//prompt-.md` (the same place the +agent sandbox keeps its kick prompt), builds the backend's command line +with the prompt as stdin, records it in the S4 command log (`capture: +runner`, status from the exit), launches it through the S2 launcher as +`crux-step---` with the session's network and filtered +environment, and reads the answer back. The first attempt sends the +caller's prompt verbatim; every retry appends `CorrectivePrompt`: the +schema, the violations by JSON path, and how many attempts remain. A +non-JSON or missing answer counts as a mismatch (`$: the backend returned +no JSON value`) and is retried the same way. Timeout and cancel tear the +launch down by name, as S2 does. Every finished step writes its record, +appends one `step` signal to the run and leaves a transcript at +`/runs//steps//transcript.jsonl` (one line per attempt: +command, exit, scrubbed output, status, violations, usage). + +**Schema.** `agentstep.ParseSchema` enforces the subset the service can +check without a schema library: `type` (incl. unions), `enum`, `const`, +`properties`, `required`, `additionalProperties` (bool or schema), +`items`, `minItems`/`maxItems`, `minLength`/`maxLength`, `pattern`, +`minimum`/`maximum`/`exclusiveMinimum`/`exclusiveMaximum`, `multipleOf`, +boolean schemas; `title`, `description`, `default`, `examples`, `$schema`, +`$id` and `$comment` are ignored. Any other keyword (`$ref`, `oneOf`, +`format`, …) is refused with 400 rather than silently unenforced, so a +`succeeded` step means the answer was fully checked. + +**Backends.** `agentstep.Backend` is `{Name; Command(Spec) ([]string, +error); Parse(Spec, sandbox.Result, readFile) Outcome}`: it turns the +request into a command line the sandbox runs and reads the attempt's +result back. `Command` returning `*UnsupportedError` for a request the +backend cannot honour (tools it cannot restrict, a turn limit it cannot +enforce) records a step with status `unsupported` and answers 501 +`unsupported`: the refusal is on the record, as the RFC asks. The RFC +decision's two v1 backends and one test backend ship: + +| Backend | Command | `Outcome` from | +|---|---|---| +| `claude` | `claude -p --output-format json [--model M] (--allowedTools a,b \| --dangerously-skip-permissions) [--max-turns N] < prompt` | the result object: `result` holds the answer (JSON extracted from prose or a fence), `usage` the tokens, `num_turns` the turns; `permission_denials` → `tool_denied`; subtype `error_max_turns` → `turn_limit_exceeded`; any other error → `backend_error` | +| `codex` | `codex exec --json --skip-git-repo-check --dangerously-bypass-approvals-and-sandbox [--model M] --output-last-message .crux/steps//answer-.txt - < prompt` | the last-message file, else the last `agent_message` item in the JSONL; `usage` events sum the tokens. `tools` is refused as unsupported (codex has no allowlist), and so is `turn_limit` above 1 (codex exec is one turn) | +| `process` | `sh -lc ' < prompt'` | stdout as the answer, non-zero exit `backend_error`. Takes `command`; `tools`, `turn_limit` and `model` are refused as unsupported (the command offers whatever it offers). This is the test and escape-hatch backend: any CLI that reads a prompt on stdin | + +The claude and codex command lines and result shapes are the adapter +contract this slice fixes; they are exercised against recorded output in +`backend_test.go`, not a live CLI. The RFC's other backends (`gemini`, +`agy`, `cline`) are not registered: adding one is a `Backend` +implementation and a line in `Backends`, with no change to the service, +the routes or the CLI. + +| Condition | Status | `code` | +|---|---|---| +| unknown backend, both or neither of `prompt`/`prompt_path`, path outside the workspace, bad or unenforceable `output_schema`, `retry_on_mismatch` outside 0–5, `timeout_ms` above the cap, `command` with a non-process backend, process without `command`; agent profile with `command` | 400 | `bad_request` | +| another principal's run | 403 | `forbidden` | +| unknown run or step | 404 | `not_found` | +| the run is not an open session | 409 | `not_session` | +| the session is running a step | 409 | `step_busy` | +| the backend cannot honour `tools`, `turn_limit` or `model`; a session on a backend without `restricted` | 501 | `unsupported` | +| no runs service | 501 | `not_supported` | + +``` +crux --server ADDR runs start --profile agent [--network none] --checkout ID +crux --server ADDR step run --backend B (--prompt P | --prompt-file F) --schema FILE + [--model M] [--tool T]... [--turn-limit N] [--retry N] + [--timeout-ms N] [--json] [-- CMD ARGS...] +crux --server ADDR step list [--json] +crux --server ADDR step show [--json] +crux --server ADDR runs cancel +``` + +`step run` exits 0 only on `succeeded`; any other status prints the +record (with violations) and exits 1, so a driver script can branch on +it. + +**Storage.** `/runs//steps//record.json` (the `Step`, +tmp+rename) and `transcript.jsonl`, 0600; steps are numbered from 1 in +creation order under a process-wide lock. Prompts and backend answer +files live in the workspace under `.crux/steps//` and go with it. + +**Deliberately not in S6.** No `gemini`, `agy` or `cline` backend yet; +no streaming of a step's progress beyond the S5 `step` signal and the S4 +log; no step-level idempotency key (post once, read back with `GET`); +no fan-out of one prompt to several backends; no cost accounting beyond +the usage the backend reports; no persistence of the agent's own session +between steps — each step is a fresh backend invocation in the same +checkout, and whatever the step wrote to the workspace is the state the +next one sees. + ## Deliberately not in S1 - TTL sweeper (recorded, not enforced). @@ -482,3 +621,8 @@ for runs are deferred and not scheduled. | `src/crux/serve/gates.go` | `run` on gate open (ownership check) and the gate's turn signals | | `src/crux/serve/signals.go` | `GET /v1/agents/{name}/kick-outcome`; the kick-outcome observer that saves before it publishes | | `src/crux/cli/signal.go` | `crux signal emit|list`, remote only | +| `src/crux/agentstep/` | S6: `Run`, `Host`, `Backend`, `Backends`, `Lookup`, `Spec`, `Outcome`, `Store` (`Create`, `Write`, `Get`, `List`), `ParseSchema`, `Schema.Validate`, `CorrectivePrompt`, `RequestError`, `UnsupportedError`, sentinels (`ErrSchema`, `ErrNotFound`, `ErrCorrupt`, `ErrNoAnswer`) | +| `src/crux/api/steps.go` | `RunStepsPath`, `RunStepPath`, `Step*` status and backend constants, `CodeStepBusy`, `CodeNotSession`, `SignalStep`, `SignalStepData`, `StepRequest`, `Step`, `StepUsage`, `SchemaViolation`, `StepList`, `Client.{RunStep,ListSteps,GetStep}` | +| `src/crux/runs/service.go` | S6 session: `profile: agent` validation, `session` runner, `RunStep`, `Steps`, `Step`, `ErrNotSession`, `ErrStepBusy` | +| `src/crux/serve/steps.go` | `POST`/`GET /v1/runs/{id}/steps`, `GET /v1/runs/{id}/steps/{n}`, ownership check, error mapping | +| `src/crux/cli/step.go` | `crux step run|list|show`, remote only |