From 84b42eabe8364dc3d5f2ac6e6d05094ad53cd33b Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Sun, 8 Nov 2026 13:59:25 -0500 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20crux:=20agent=20step=20service=20(S?= =?UTF-8?q?6)=20(#17272)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ✨ crux: agent step service (S6) Add the S6 work service of hivecommons/crux#114: an agent-profile run opens a step session; POST /v1/runs/{id}/steps runs one claude, codex or process backend step inside its sandbox, validates the JSON answer against the request's schema with bounded retries, records the command in the S4 log and announces a step turn signal. Routes, crux step run|list|show, design doc section and changelog fragment included. Part of hivecommons/crux#114 Signed-off-by: Andrew Anderson * crux runs: split agent step session into runs/steps.go Keeps runs/service.go under the 800-line ratchet. Signed-off-by: Andrew Anderson * crux agentstep: reject null schema; list step as a work-service verb Signed-off-by: Andrew Anderson --------- Signed-off-by: Andrew Anderson Co-authored-by: hivecommons-hive[bot] <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- changelog.d/added-crux-agent-steps.md | 1 + src/crux/agentstep/agentstep.go | 334 +++++++++++++++++++++ src/crux/agentstep/agentstep_test.go | 364 +++++++++++++++++++++++ src/crux/agentstep/backend.go | 304 +++++++++++++++++++ src/crux/agentstep/backend_test.go | 145 +++++++++ src/crux/agentstep/schema.go | 406 ++++++++++++++++++++++++++ src/crux/agentstep/schema_test.go | 103 +++++++ src/crux/agentstep/store.go | 159 ++++++++++ src/crux/api/runs.go | 5 +- src/crux/api/signals.go | 4 +- src/crux/api/steps.go | 178 +++++++++++ src/crux/cli/cli.go | 7 + src/crux/cli/remote.go | 2 + src/crux/cli/runs.go | 7 +- src/crux/cli/step.go | 224 ++++++++++++++ src/crux/cli/step_test.go | 213 ++++++++++++++ src/crux/cli/verbs_test.go | 4 +- src/crux/runs/service.go | 43 ++- src/crux/runs/service_test.go | 3 +- src/crux/runs/steps.go | 167 +++++++++++ src/crux/runs/steps_test.go | 189 ++++++++++++ src/crux/runstate/runstate.go | 9 +- src/crux/serve/runs.go | 5 +- src/crux/serve/runs_test.go | 3 +- src/crux/serve/serve.go | 1 + src/crux/serve/steps.go | 102 +++++++ src/crux/serve/steps_test.go | 134 +++++++++ src/crux/signals/signals.go | 11 +- src/crux/signals/signals_test.go | 3 + src/docs/design/crux-work-services.md | 152 +++++++++- 30 files changed, 3251 insertions(+), 31 deletions(-) create mode 100644 changelog.d/added-crux-agent-steps.md create mode 100644 src/crux/agentstep/agentstep.go create mode 100644 src/crux/agentstep/agentstep_test.go create mode 100644 src/crux/agentstep/backend.go create mode 100644 src/crux/agentstep/backend_test.go create mode 100644 src/crux/agentstep/schema.go create mode 100644 src/crux/agentstep/schema_test.go create mode 100644 src/crux/agentstep/store.go create mode 100644 src/crux/api/steps.go create mode 100644 src/crux/cli/step.go create mode 100644 src/crux/cli/step_test.go create mode 100644 src/crux/runs/steps.go create mode 100644 src/crux/runs/steps_test.go create mode 100644 src/crux/serve/steps.go create mode 100644 src/crux/serve/steps_test.go diff --git a/changelog.d/added-crux-agent-steps.md b/changelog.d/added-crux-agent-steps.md new file mode 100644 index 0000000000..7a8609f047 --- /dev/null +++ b/changelog.d/added-crux-agent-steps.md @@ -0,0 +1 @@ +- crux: agent step service, the sixth work service of the Crux work-services RFC ([hivecommons/crux#114](https://github.com/hivecommons/crux/issues/114) S6). `POST /v1/runs` with `"profile": "agent"` now opens a step session — a contained run that provisions its checkout and waits — and `POST /v1/runs/{id}/steps` runs one agent step inside it: a `claude`, `codex` or `process` backend reads the prompt in the sandbox and its JSON answer is checked against the request's `output_schema`, with bounded re-asks on a mismatch, a per-attempt timeout, tool and turn limits where the backend can enforce them, and a recorded `unsupported` refusal where it cannot. Each step is logged in the S4 command log, announced as a `step` turn signal, and read back with `GET /v1/runs/{id}/steps[/{n}]` or `crux --server … step run|list|show`. diff --git a/src/crux/agentstep/agentstep.go b/src/crux/agentstep/agentstep.go new file mode 100644 index 0000000000..1a831e2da5 --- /dev/null +++ b/src/crux/agentstep/agentstep.go @@ -0,0 +1,334 @@ +// Package agentstep is the schema-checked agent step service of the Crux +// work services (S6 in hivecommons/crux#114): run one agent turn in a +// contained workspace — a prompt, a backend CLI, a model, an allowlist of +// tools, a turn limit — and hand back the answer already validated against +// the caller's JSON Schema, with the token usage, a transcript of every +// attempt and a typed status. The loop is agent.SandboxExecutor.Run +// generalised: write the prompt into the workspace, build the backend's +// command (Backend carries the per-CLI flags), run it through +// sandbox.Launcher, read the answer back, check it, and on a mismatch retry +// with a corrective prompt. +// +// Steps belong to an agent-profile contained run (runs.Service, S2), which +// supplies the workspace, the launcher, the command log (S4) and the signal +// book (S5); this package never looks those up itself. Records live under +// the run's directory so a caller recovers them by run id and step number +// after a disconnect (S7). +package agentstep + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "time" + + "github.com/hivecommons/hive/crux/api" + "github.com/hivecommons/hive/crux/cmdlog" + "github.com/hivecommons/hive/crux/logscrub" + "github.com/hivecommons/hive/crux/sandbox" +) + +// Limits. +const ( + // MaxRetries bounds retry_on_mismatch. + MaxRetries = 5 + // MaxPromptBytes bounds an inline prompt. + MaxPromptBytes = 1 << 20 + // workspaceDir is where prompts and backend answer files go inside the + // workspace, next to the agent sandbox's .hive. + workspaceDir = ".crux/steps" + // launchNamePrefix names the container after the run, step and attempt + // so a teardown can find it. + launchNamePrefix = "crux-step-" + // teardownTimeout bounds the container removal after a cancel or a + // timeout. + teardownTimeout = 30 * time.Second +) + +// Host is what the session hands the service for one step: where to run, +// how to launch, where to record. The runs service builds it from the +// agent-profile run the step is posted to. +type Host struct { + Run string + RunDir string + Workspace string + Image string + Network string + // Env and EnvAllowlist are the session's, already filtered of + // credentials by sandbox.SanitizedEnv. + Env map[string]string + EnvAllowlist []string + Launcher sandbox.Launcher + // Teardown, when set, removes the launch by name after a cancel or a + // timeout (runs.Backend.Teardown). + Teardown func(ctx context.Context, name string) error + // Timeout is the per-attempt default; MaxTimeout caps a request's. + Timeout, MaxTimeout time.Duration + // CmdLog, when set, records each attempt's command (S4). + CmdLog cmdlog.Recorder + // Signal, when set, appends a runner signal to the run (S5): the + // service emits one step signal per finished step. + Signal func(kind, message string, data json.RawMessage) + // Scrub redacts output before it is persisted; nil uses logscrub. + Scrub func(string) string + // Now is the clock; nil uses time.Now. + Now func() time.Time +} + +func (h Host) scrub(s string) string { + if h.Scrub != nil { + return h.Scrub(s) + } + return logscrub.ScrubString(s) +} + +func (h Host) now() time.Time { + if h.Now != nil { + return h.Now().UTC() + } + return time.Now().UTC() +} + +// plan is a validated request. +type plan struct { + backend Backend + schema *Schema + rawSch json.RawMessage + prompt string + spec Spec + timeout time.Duration + retries int +} + +// Run executes one step for host and returns its saved record. A request +// the backend cannot honour is saved with status unsupported and returned +// with an *UnsupportedError, so the refusal is on the record as the RFC +// asks; a malformed request is a *RequestError with no record. +func Run(ctx context.Context, host Host, req api.StepRequest) (api.Step, error) { + p, err := validate(host, req) + if err != nil { + return api.Step{}, err + } + store := Store{RunDir: host.RunDir} + // Build the command once with placeholder paths to learn whether the + // backend can honour the request before anything is recorded as run. + probe := p.spec + probe.PromptRel, probe.AnswerRel = "prompt", "answer" + if _, cmdErr := p.backend.Command(probe); cmdErr != nil { + var unsupported *UnsupportedError + if !errors.As(cmdErr, &unsupported) { + return api.Step{}, cmdErr + } + step, err := store.Create(api.Step{Run: host.Run, Backend: req.Backend, Model: req.Model, CreatedAt: host.now()}) + if err != nil { + return api.Step{}, err + } + step = finish(host, store, step, api.StepUnsupported, unsupported.What, nil) + return step, unsupported + } + step, err := store.Create(api.Step{Run: host.Run, Backend: req.Backend, Model: req.Model, Status: api.StepRunning, CreatedAt: host.now()}) + if err != nil { + return api.Step{}, err + } + step.TranscriptPath = store.TranscriptRel(step.N) + return execute(ctx, host, store, step, p), nil +} + +func validate(host Host, req api.StepRequest) (plan, error) { + var p plan + backend, err := Lookup(strings.TrimSpace(req.Backend)) + if err != nil { + return p, err + } + p.backend = backend + switch { + case req.Prompt != "" && req.PromptPath != "": + return p, &RequestError{"prompt and prompt_path are mutually exclusive"} + case req.Prompt != "": + if len(req.Prompt) > MaxPromptBytes { + return p, &RequestError{fmt.Sprintf("prompt is %d bytes; the bound is %d (use prompt_path)", len(req.Prompt), MaxPromptBytes)} + } + p.prompt = req.Prompt + case req.PromptPath != "": + rel := filepath.Clean(req.PromptPath) + if filepath.IsAbs(rel) || rel == ".." || strings.HasPrefix(rel, "../") { + return p, &RequestError{"prompt_path must be relative to the workspace and stay inside it"} + } + data, err := os.ReadFile(filepath.Join(host.Workspace, rel)) + if err != nil { + return p, &RequestError{fmt.Sprintf("prompt_path %q: %v", req.PromptPath, err)} + } + p.prompt = string(data) + default: + return p, &RequestError{"one of prompt or prompt_path is required"} + } + if strings.TrimSpace(p.prompt) == "" { + return p, &RequestError{"prompt is empty"} + } + p.rawSch = req.OutputSchema + if p.schema, err = ParseSchema(req.OutputSchema); err != nil { + return p, &RequestError{strings.TrimPrefix(err.Error(), "agentstep: invalid output_schema: ")} + } + if req.RetryOnMismatch < 0 || req.RetryOnMismatch > MaxRetries { + return p, &RequestError{fmt.Sprintf("retry_on_mismatch must be between 0 and %d", MaxRetries)} + } + p.retries = req.RetryOnMismatch + if req.TurnLimit < 0 { + return p, &RequestError{"turn_limit must not be negative"} + } + p.timeout = host.Timeout + if req.TimeoutMs > 0 { + d := time.Duration(req.TimeoutMs) * time.Millisecond + if host.MaxTimeout > 0 && d > host.MaxTimeout { + return p, &RequestError{fmt.Sprintf("timeout_ms %d exceeds this server's cap of %d", req.TimeoutMs, host.MaxTimeout.Milliseconds())} + } + p.timeout = d + } + for _, t := range req.Tools { + if strings.TrimSpace(t) == "" { + return p, &RequestError{"tools must not contain an empty name"} + } + } + p.spec = Spec{Model: req.Model, Tools: req.Tools, TurnLimit: req.TurnLimit, Command: req.Command} + return p, nil +} + +// attempt is one transcript line: what ran and what came back, scrubbed. +type attempt struct { + Attempt int `json:"attempt"` + At time.Time `json:"at"` + Command []string `json:"command"` + ExitCode *int `json:"exit_code,omitempty"` + Stdout string `json:"stdout,omitempty"` + Stderr string `json:"stderr,omitempty"` + Status string `json:"status,omitempty"` + Reason string `json:"reason,omitempty"` + Violations []api.SchemaViolation `json:"violations,omitempty"` + Usage api.StepUsage `json:"usage"` +} + +// execute runs the attempts to a terminal status. Every exit path writes +// the record and then the run's step signal. +func execute(ctx context.Context, host Host, store Store, step api.Step, p plan) api.Step { + prompt := p.prompt + var violations []api.SchemaViolation + for n := 1; n <= 1+p.retries; n++ { + step.Attempts = n + a, outcome := runAttempt(ctx, host, store, step, p, n, prompt) + step.Usage.InputTokens += outcome.Usage.InputTokens + step.Usage.OutputTokens += outcome.Usage.OutputTokens + if outcome.Status != "" { + a.Status, a.Reason = outcome.Status, outcome.Reason + _ = store.AppendTranscript(step.N, a) + return finish(host, store, step, outcome.Status, outcome.Reason, nil) + } + if outcome.Answer == nil { + violations = []api.SchemaViolation{{Path: "$", Message: ErrNoAnswer.Error()}} + } else { + violations = p.schema.Validate(outcome.Answer) + } + if len(violations) == 0 { + a.Status = api.StepSucceeded + _ = store.AppendTranscript(step.N, a) + step.Answer = outcome.Answer + return finish(host, store, step, api.StepSucceeded, "", nil) + } + a.Status, a.Violations = api.StepSchemaMismatch, violations + _ = store.AppendTranscript(step.N, a) + prompt = p.prompt + "\n\n" + CorrectivePrompt(p.rawSch, violations, p.retries-n) + } + return finish(host, store, step, api.StepSchemaMismatch, fmt.Sprintf("the answer did not match output_schema after %d attempt(s)", step.Attempts), violations) +} + +// runAttempt launches the backend once and reads its outcome. +func runAttempt(ctx context.Context, host Host, store Store, step api.Step, p plan, n int, prompt string) (attempt, Outcome) { + a := attempt{Attempt: n, At: host.now()} + rel := filepath.ToSlash(filepath.Join(workspaceDir, fmt.Sprint(step.N))) + spec := p.spec + spec.PromptRel = rel + fmt.Sprintf("/prompt-%d.md", n) + spec.AnswerRel = rel + fmt.Sprintf("/answer-%d.txt", n) + if err := os.MkdirAll(filepath.Join(host.Workspace, rel), 0o770); err != nil { + return a, Outcome{Status: api.StepBackendError, Reason: "writing the prompt: " + host.scrub(err.Error())} + } + if err := os.WriteFile(filepath.Join(host.Workspace, filepath.FromSlash(spec.PromptRel)), []byte(prompt), 0o660); err != nil { + return a, Outcome{Status: api.StepBackendError, Reason: "writing the prompt: " + host.scrub(err.Error())} + } + cmd, err := p.backend.Command(spec) + if err != nil { + return a, Outcome{Status: api.StepUnsupported, Reason: err.Error()} + } + a.Command = cmd + var seq int64 + if host.CmdLog != nil { + if seq, err = host.CmdLog.Started(cmdlog.Start{Command: cmd, Cwd: sandbox.DefaultWorkspaceMount, Capture: api.CommandCaptureRunner}); err != nil { + seq = 0 + } + } + runCtx, cancel := context.WithTimeout(ctx, p.timeout) + defer cancel() + name := fmt.Sprintf("%s%s-%d-%d", launchNamePrefix, host.Run, step.N, n) + res, runErr := host.Launcher.Run(runCtx, sandbox.LaunchSpec{ + Image: host.Image, Name: name, Workspace: host.Workspace, Command: cmd, + Env: host.Env, EnvAllowlist: host.EnvAllowlist, NetworkMode: host.Network, + }) + cancelled := runErr != nil && ctx.Err() != nil + timedOut := runErr != nil && !cancelled && errors.Is(runCtx.Err(), context.DeadlineExceeded) + if (cancelled || timedOut) && host.Teardown != nil { + tctx, tcancel := context.WithTimeout(context.Background(), teardownTimeout) + _ = host.Teardown(tctx, name) + tcancel() + } + a.Stdout, a.Stderr = host.scrub(res.Stdout), host.scrub(res.Stderr) + launched := runErr == nil || res != (sandbox.Result{}) + if launched && res.ExitCode != -1 { + code := res.ExitCode + a.ExitCode = &code + } + var out Outcome + switch { + case cancelled: + out = Outcome{Status: api.StepCancelled, Reason: "the session was cancelled"} + case timedOut: + out = Outcome{Status: api.StepTimeout, Reason: fmt.Sprintf("the backend did not finish within %s", p.timeout)} + case !launched: + out = Outcome{Status: api.StepBackendError, Reason: host.scrub(runErr.Error())} + default: + out = p.backend.Parse(spec, res, func(rel string) ([]byte, error) { + return os.ReadFile(filepath.Join(host.Workspace, filepath.FromSlash(rel))) + }) + out.Reason = host.scrub(out.Reason) + } + a.Usage = out.Usage + if host.CmdLog != nil && seq > 0 { + end := cmdlog.End{ExitCode: a.ExitCode, Stdout: res.Stdout, Stderr: res.Stderr, Reason: out.Reason} + switch out.Status { + case api.StepCancelled: + end.Status = api.CommandCancelled + case api.StepTimeout, api.StepBackendError: + end.Status = api.CommandFailed + } + _ = host.CmdLog.Ended(seq, end) + } + return a, out +} + +// finish writes the terminal record and announces it on the run. +func finish(host Host, store Store, step api.Step, status, reason string, violations []api.SchemaViolation) api.Step { + now := host.now() + step.Status, step.Reason, step.FinishedAt, step.Violations = status, reason, &now, violations + if status != api.StepSucceeded { + step.Answer = nil + } + _ = store.Write(step) + if host.Signal != nil { + data, err := json.Marshal(api.SignalStepData{Step: step.N, Status: status, Attempts: step.Attempts}) + if err == nil { + host.Signal(api.SignalStep, reason, data) + } + } + return step +} diff --git a/src/crux/agentstep/agentstep_test.go b/src/crux/agentstep/agentstep_test.go new file mode 100644 index 0000000000..e59bedd78f --- /dev/null +++ b/src/crux/agentstep/agentstep_test.go @@ -0,0 +1,364 @@ +package agentstep + +import ( + "context" + "encoding/json" + "errors" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/hivecommons/hive/crux/api" + "github.com/hivecommons/hive/crux/cmdlog" + "github.com/hivecommons/hive/crux/sandbox" +) + +// fakeLauncher answers each launch from a queue of results, recording the +// spec and the prompt the backend would have read. +type fakeLauncher struct { + mu sync.Mutex + specs []sandbox.LaunchSpec + prompts []string + results []func(ctx context.Context, spec sandbox.LaunchSpec) (sandbox.Result, error) +} + +func (f *fakeLauncher) Run(ctx context.Context, spec sandbox.LaunchSpec) (sandbox.Result, error) { + f.mu.Lock() + f.specs = append(f.specs, spec) + // The prompt path is the last word of the sh -lc line. + line := spec.Command[len(spec.Command)-1] + rel := strings.Trim(line[strings.LastIndex(line, " < ")+3:], "'") + data, _ := os.ReadFile(filepath.Join(spec.Workspace, rel)) + f.prompts = append(f.prompts, string(data)) + next := f.results[0] + if len(f.results) > 1 { + f.results = f.results[1:] + } + f.mu.Unlock() + return next(ctx, spec) +} + +func answers(outs ...string) *fakeLauncher { + f := &fakeLauncher{} + for _, o := range outs { + o := o + f.results = append(f.results, func(context.Context, sandbox.LaunchSpec) (sandbox.Result, error) { + return sandbox.Result{Stdout: o}, nil + }) + } + return f +} + +type recorded struct { + kind string + data api.SignalStepData +} + +type fixture struct { + t *testing.T + host Host + launcher *fakeLauncher + signals []recorded + torn []string + log cmdlog.Recorder +} + +func newFixture(t *testing.T, launcher *fakeLauncher) *fixture { + t.Helper() + f := &fixture{t: t, launcher: launcher} + runDir := t.TempDir() + log, err := cmdlog.FileOpener(cmdlog.Policy{})(runDir) + if err != nil { + t.Fatal(err) + } + f.log = log + f.host = Host{ + Run: "run_1", RunDir: runDir, Workspace: t.TempDir(), Image: "img:1", Network: "restricted", + Env: map[string]string{"KEEP": "yes"}, EnvAllowlist: []string{"KEEP"}, + Launcher: launcher, Timeout: time.Minute, MaxTimeout: time.Hour, CmdLog: log, + Teardown: func(_ context.Context, name string) error { f.torn = append(f.torn, name); return nil }, + Signal: func(kind, _ string, data json.RawMessage) { + var d api.SignalStepData + _ = json.Unmarshal(data, &d) + f.signals = append(f.signals, recorded{kind, d}) + }, + } + return f +} + +const okSchema = `{"type":"object","required":["verdict"],"properties":{"verdict":{"enum":["pass","fail"]}}}` + +func (f *fixture) process(prompt string, schema string) api.StepRequest { + return api.StepRequest{Backend: "process", Prompt: prompt, OutputSchema: json.RawMessage(schema), Command: []string{"./ask"}} +} + +func (f *fixture) run(req api.StepRequest) api.Step { + f.t.Helper() + step, err := Run(context.Background(), f.host, req) + if err != nil { + f.t.Fatalf("Run: %v", err) + } + return step +} + +func (f *fixture) commands() []api.CommandRecord { + f.t.Helper() + page, err := cmdlog.Read(f.host.RunDir, 0, 0) + if err != nil { + f.t.Fatal(err) + } + return page.Commands +} + +func TestSucceededStepIsRecordedLoggedAndSignalled(t *testing.T) { + f := newFixture(t, answers(`{"verdict":"pass"}`)) + step := f.run(f.process("Is it fine?", okSchema)) + if step.N != 1 || step.Status != api.StepSucceeded || step.Attempts != 1 || string(step.Answer) != `{"verdict":"pass"}` || step.FinishedAt == nil || step.Backend != "process" { + t.Fatalf("step = %+v", step) + } + spec := f.launcher.specs[0] + if spec.Name != "crux-step-run_1-1-1" || spec.Image != "img:1" || spec.NetworkMode != "restricted" || spec.Env["KEEP"] != "yes" || spec.Workspace != f.host.Workspace { + t.Fatalf("spec = %+v", spec) + } + if f.launcher.prompts[0] != "Is it fine?" { + t.Fatalf("first prompt = %q, want the caller's verbatim", f.launcher.prompts[0]) + } + got, err := Store{RunDir: f.host.RunDir}.Get(1) + if err != nil || got.Status != api.StepSucceeded || got.TranscriptPath != "steps/1/transcript.jsonl" { + t.Fatalf("Get = %+v, %v", got, err) + } + transcript, _ := os.ReadFile(filepath.Join(f.host.RunDir, got.TranscriptPath)) + if n := strings.Count(string(transcript), "\n"); n != 1 || !strings.Contains(string(transcript), `"status":"succeeded"`) { + t.Fatalf("transcript = %s", transcript) + } + if cmds := f.commands(); len(cmds) != 1 || cmds[0].Status != api.CommandCompleted || cmds[0].Command[0] != "sh" { + t.Fatalf("cmdlog = %+v", cmds) + } + if len(f.signals) != 1 || f.signals[0].kind != api.SignalStep || f.signals[0].data != (api.SignalStepData{Step: 1, Status: api.StepSucceeded, Attempts: 1}) { + t.Fatalf("signals = %+v", f.signals) + } + if len(f.torn) != 0 { + t.Fatal("a finished step was torn down") + } +} + +func TestMismatchRetriesWithTheViolationsThenGivesUp(t *testing.T) { + f := newFixture(t, answers(`{"verdict":"maybe"}`, `prose`, `{"verdict":"pass"}`)) + req := f.process("Judge it.", okSchema) + req.RetryOnMismatch = 2 + step := f.run(req) + if step.Status != api.StepSucceeded || step.Attempts != 3 || step.N != 1 { + t.Fatalf("step = %+v", step) + } + if p := f.launcher.prompts[1]; !strings.HasPrefix(p, "Judge it.\n\n") || !strings.Contains(p, "$.verdict") || !strings.Contains(p, "remaining after this one: 1") { + t.Fatalf("second prompt = %q", p) + } + if p := f.launcher.prompts[2]; !strings.Contains(p, ErrNoAnswer.Error()) || !strings.Contains(p, "remaining after this one: 0") { + t.Fatalf("third prompt = %q", p) + } + if len(f.commands()) != 3 || len(f.signals) != 1 { + t.Fatalf("cmdlog %d signals %d", len(f.commands()), len(f.signals)) + } + + f = newFixture(t, answers(`{"verdict":"maybe"}`)) + req = f.process("Judge it.", okSchema) + req.RetryOnMismatch = 1 + step = f.run(req) + if step.Status != api.StepSchemaMismatch || step.Attempts != 2 || step.Answer != nil || len(step.Violations) != 1 || step.Violations[0].Path != "$.verdict" || !strings.Contains(step.Reason, "after 2 attempt") { + t.Fatalf("step = %+v", step) + } + if f.signals[0].data.Status != api.StepSchemaMismatch { + t.Fatalf("signal = %+v", f.signals) + } +} + +func TestStepsNumberUpAndListInOrder(t *testing.T) { + f := newFixture(t, answers(`{"verdict":"pass"}`)) + for i := 1; i <= 3; i++ { + if step := f.run(f.process("p", okSchema)); step.N != i { + t.Fatalf("step %d numbered %d", i, step.N) + } + } + store := Store{RunDir: f.host.RunDir} + steps, err := store.List() + if err != nil || len(steps) != 3 || steps[0].N != 1 || steps[2].N != 3 { + t.Fatalf("List = %+v, %v", steps, err) + } + if _, err := store.Get(4); !errors.Is(err, ErrNotFound) { + t.Fatalf("Get(4) = %v", err) + } + if _, err := store.Get(0); !errors.Is(err, ErrNotFound) { + t.Fatalf("Get(0) = %v", err) + } + // A corrupt record is reported after the readable ones. + if err := os.WriteFile(filepath.Join(store.Dir(2), RecordName), []byte("{"), 0o600); err != nil { + t.Fatal(err) + } + steps, err = store.List() + if !errors.Is(err, ErrCorrupt) || len(steps) != 2 { + t.Fatalf("List with a corrupt record = %+v, %v", steps, err) + } + // Numbering still moves past it. + if step := f.run(f.process("p", okSchema)); step.N != 4 { + t.Fatalf("after corrupt: N = %d", step.N) + } + if steps, err := (Store{RunDir: t.TempDir()}).List(); err != nil || len(steps) != 0 { + t.Fatalf("empty List = %+v, %v", steps, err) + } +} + +func TestPromptPathReadsFromTheWorkspace(t *testing.T) { + f := newFixture(t, answers(`{"verdict":"pass"}`)) + if err := os.WriteFile(filepath.Join(f.host.Workspace, "task.md"), []byte("from file"), 0o600); err != nil { + t.Fatal(err) + } + req := f.process("", okSchema) + req.PromptPath = "task.md" + f.run(req) + if f.launcher.prompts[0] != "from file" { + t.Fatalf("prompt = %q", f.launcher.prompts[0]) + } +} + +func TestRequestRefusals(t *testing.T) { + f := newFixture(t, answers(`{}`)) + long := strings.Repeat("x", MaxPromptBytes+1) + for _, tc := range []struct { + name string + req api.StepRequest + msg string + }{ + {"backend", api.StepRequest{Backend: "gemini", Prompt: "p", OutputSchema: json.RawMessage(okSchema)}, "unknown backend"}, + {"both prompts", api.StepRequest{Backend: "process", Prompt: "p", PromptPath: "x", OutputSchema: json.RawMessage(okSchema)}, "mutually exclusive"}, + {"no prompt", api.StepRequest{Backend: "process", OutputSchema: json.RawMessage(okSchema)}, "one of prompt or prompt_path"}, + {"blank prompt", api.StepRequest{Backend: "process", Prompt: " ", OutputSchema: json.RawMessage(okSchema)}, "prompt is empty"}, + {"long prompt", api.StepRequest{Backend: "process", Prompt: long, OutputSchema: json.RawMessage(okSchema)}, "bound"}, + {"absolute prompt path", api.StepRequest{Backend: "process", PromptPath: "/etc/passwd", OutputSchema: json.RawMessage(okSchema)}, "relative"}, + {"escaping prompt path", api.StepRequest{Backend: "process", PromptPath: "../x", OutputSchema: json.RawMessage(okSchema)}, "relative"}, + {"missing prompt path", api.StepRequest{Backend: "process", PromptPath: "nope.md", OutputSchema: json.RawMessage(okSchema)}, "nope.md"}, + {"no schema", api.StepRequest{Backend: "process", Prompt: "p"}, "output_schema"}, + {"bad schema", api.StepRequest{Backend: "process", Prompt: "p", OutputSchema: json.RawMessage(`{"oneOf":[]}`)}, "oneOf"}, + {"retries", api.StepRequest{Backend: "process", Prompt: "p", OutputSchema: json.RawMessage(okSchema), RetryOnMismatch: MaxRetries + 1}, "retry_on_mismatch"}, + {"turn limit", api.StepRequest{Backend: "process", Prompt: "p", OutputSchema: json.RawMessage(okSchema), TurnLimit: -1}, "turn_limit"}, + {"timeout cap", api.StepRequest{Backend: "process", Prompt: "p", OutputSchema: json.RawMessage(okSchema), TimeoutMs: (2 * time.Hour).Milliseconds()}, "timeout_ms"}, + {"empty tool", api.StepRequest{Backend: "claude", Prompt: "p", OutputSchema: json.RawMessage(okSchema), Tools: []string{" "}}, "empty name"}, + {"process without command", api.StepRequest{Backend: "process", Prompt: "p", OutputSchema: json.RawMessage(okSchema)}, "needs command"}, + {"claude with command", api.StepRequest{Backend: "claude", Prompt: "p", OutputSchema: json.RawMessage(okSchema), Command: []string{"x"}}, "builds its own command"}, + } { + t.Run(tc.name, func(t *testing.T) { + var reqErr *RequestError + _, err := Run(context.Background(), f.host, tc.req) + if !errors.As(err, &reqErr) || !strings.Contains(err.Error(), tc.msg) { + t.Fatalf("err = %T %v, want RequestError containing %q", err, err, tc.msg) + } + }) + } + if len(f.launcher.specs) != 0 { + t.Fatal("a refused request was launched") + } + if steps, _ := (Store{RunDir: f.host.RunDir}).List(); len(steps) != 0 { + t.Fatalf("a refused request was recorded: %+v", steps) + } +} + +func TestUnsupportedRequestIsRecordedAndRefused(t *testing.T) { + f := newFixture(t, answers(`{}`)) + req := f.process("p", okSchema) + req.Tools = []string{"Bash"} + step, err := Run(context.Background(), f.host, req) + var unsupported *UnsupportedError + if !errors.As(err, &unsupported) || !strings.Contains(err.Error(), "cannot restrict tools") { + t.Fatalf("err = %v", err) + } + if step.N != 1 || step.Status != api.StepUnsupported || step.Reason != err.Error() || step.Attempts != 0 { + t.Fatalf("step = %+v", step) + } + if got, err := (Store{RunDir: f.host.RunDir}).Get(1); err != nil || got.Status != api.StepUnsupported { + t.Fatalf("Get = %+v, %v", got, err) + } + if len(f.launcher.specs) != 0 || len(f.signals) != 1 || f.signals[0].data.Status != api.StepUnsupported { + t.Fatalf("launched %d, signals %+v", len(f.launcher.specs), f.signals) + } +} + +func TestBackendFailureStatuses(t *testing.T) { + // A non-zero exit is a backend error, logged failed, no retry. + f := newFixture(t, &fakeLauncher{results: []func(context.Context, sandbox.LaunchSpec) (sandbox.Result, error){ + func(context.Context, sandbox.LaunchSpec) (sandbox.Result, error) { + return sandbox.Result{ExitCode: 2, Stderr: "ghp_0123456789abcdefXYZ boom"}, nil + }, + }}) + req := f.process("p", okSchema) + req.RetryOnMismatch = 3 + step := f.run(req) + if step.Status != api.StepBackendError || step.Attempts != 1 || !strings.Contains(step.Reason, "exited 2") { + t.Fatalf("step = %+v", step) + } + if cmds := f.commands(); len(cmds) != 1 || cmds[0].Status != api.CommandFailed || *cmds[0].ExitCode != 2 { + t.Fatalf("cmdlog = %+v", cmds) + } + transcript, _ := os.ReadFile(filepath.Join(f.host.RunDir, step.TranscriptPath)) + if strings.Contains(string(transcript), "ghp_0123456789abcdefXYZ") { + t.Fatalf("transcript kept a secret: %s", transcript) + } + + // A launcher that could not start anything is a backend error too. + f = newFixture(t, &fakeLauncher{results: []func(context.Context, sandbox.LaunchSpec) (sandbox.Result, error){ + func(context.Context, sandbox.LaunchSpec) (sandbox.Result, error) { + return sandbox.Result{}, errors.New("podman: image missing") + }, + }}) + step = f.run(f.process("p", okSchema)) + if step.Status != api.StepBackendError || !strings.Contains(step.Reason, "image missing") { + t.Fatalf("step = %+v", step) + } + if cmds := f.commands(); len(cmds) != 1 || cmds[0].Status != api.CommandFailed || cmds[0].ExitCode != nil { + t.Fatalf("cmdlog = %+v", cmds) + } +} + +func TestTimeoutAndCancelTearDownTheLaunch(t *testing.T) { + wait := func(ctx context.Context, _ sandbox.LaunchSpec) (sandbox.Result, error) { + <-ctx.Done() + return sandbox.Result{ExitCode: -1}, ctx.Err() + } + f := newFixture(t, &fakeLauncher{results: []func(context.Context, sandbox.LaunchSpec) (sandbox.Result, error){wait}}) + req := f.process("p", okSchema) + req.TimeoutMs = 1 + step := f.run(req) + if step.Status != api.StepTimeout || !strings.Contains(step.Reason, "did not finish within 1ms") { + t.Fatalf("step = %+v", step) + } + if len(f.torn) != 1 || f.torn[0] != "crux-step-run_1-1-1" { + t.Fatalf("torn = %v", f.torn) + } + if cmds := f.commands(); len(cmds) != 1 || cmds[0].Status != api.CommandFailed { + t.Fatalf("cmdlog = %+v", cmds) + } + + f = newFixture(t, &fakeLauncher{results: []func(context.Context, sandbox.LaunchSpec) (sandbox.Result, error){wait}}) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + step, err := Run(ctx, f.host, f.process("p", okSchema)) + if err != nil || step.Status != api.StepCancelled || len(f.torn) != 1 { + t.Fatalf("step = %+v, %v, torn %v", step, err, f.torn) + } + if cmds := f.commands(); len(cmds) != 1 || cmds[0].Status != api.CommandCancelled { + t.Fatalf("cmdlog = %+v", cmds) + } + if f.signals[0].data.Status != api.StepCancelled { + t.Fatalf("signals = %+v", f.signals) + } +} + +func TestHostWithoutLogOrSignalsStillRuns(t *testing.T) { + f := newFixture(t, answers(`{"verdict":"fail"}`)) + f.host.CmdLog, f.host.Signal, f.host.Teardown = nil, nil, nil + step := f.run(f.process("p", okSchema)) + if step.Status != api.StepSucceeded || len(f.signals) != 0 { + t.Fatalf("step = %+v", step) + } +} diff --git a/src/crux/agentstep/backend.go b/src/crux/agentstep/backend.go new file mode 100644 index 0000000000..c464572247 --- /dev/null +++ b/src/crux/agentstep/backend.go @@ -0,0 +1,304 @@ +package agentstep + +import ( + "encoding/json" + "errors" + "fmt" + "strings" + + "github.com/hivecommons/hive/crux/api" + "github.com/hivecommons/hive/crux/jsonextract" + "github.com/hivecommons/hive/crux/sandbox" +) + +// Spec is what a Backend turns into the command the sandbox runs. Paths +// are relative to the workspace, which is the sandbox's working directory. +type Spec struct { + // PromptRel is the file holding this attempt's prompt. + PromptRel string + // AnswerRel is where a backend that can write its final message to a + // file should put it; the adapter reads it back with Outcome. + AnswerRel string + Model string + Tools []string + TurnLimit int + // Command is the process backend's argv; other backends refuse it. + Command []string +} + +// Outcome is what a Backend read out of one attempt. Answer is the +// candidate the schema is checked against; Status is empty when the +// attempt produced one, otherwise a terminal api.Step* status with Reason. +type Outcome struct { + Answer json.RawMessage + Usage api.StepUsage + Turns int + Status string + Reason string +} + +// Backend is the per-CLI adapter: it knows the flags that make the CLI +// read a prompt, pick a model, restrict its tools and stop after N turns, +// and how to read its answer and usage back. What it cannot enforce it +// refuses in Command with an *UnsupportedError; it never runs weakened. +type Backend interface { + Name() string + Command(spec Spec) ([]string, error) + // Parse reads the attempt's result. readFile returns a workspace file + // by relative path, for adapters that had the CLI write its answer to + // spec.AnswerRel. + Parse(spec Spec, res sandbox.Result, readFile func(rel string) ([]byte, error)) Outcome +} + +// UnsupportedError names a request the backend cannot honour. +type UnsupportedError struct{ What string } + +func (e *UnsupportedError) Error() string { return e.What } + +// RequestError is a malformed step request. +type RequestError struct{ Msg string } + +func (e *RequestError) Error() string { return e.Msg } + +// Backends is the shipped adapter set by name. +var Backends = map[string]Backend{ + api.StepBackendClaude: claudeBackend{}, + api.StepBackendCodex: codexBackend{}, + api.StepBackendProcess: processBackend{}, +} + +// Lookup returns the adapter for name or a RequestError listing the known +// ones. +func Lookup(name string) (Backend, error) { + if b, ok := Backends[name]; ok { + return b, nil + } + return nil, &RequestError{fmt.Sprintf("unknown backend %q; one of %s", name, strings.Join(backendNames(), ", "))} +} + +func backendNames() []string { + return []string{api.StepBackendClaude, api.StepBackendCodex, api.StepBackendProcess} +} + +// shellQuote single-quotes s for sh. +func shellQuote(s string) string { return "'" + strings.ReplaceAll(s, "'", `'\''`) + "'" } + +// shell wraps a command line so the prompt file is its stdin. +func shell(cmd, promptRel string) []string { + return []string{"sh", "-lc", cmd + " < " + shellQuote(promptRel)} +} + +// extractJSON finds the JSON value in model output: the whole text when it +// parses, else the text inside a ```json fence, else the first '{' to the +// last '}' (jsonextract.Object). Empty when nothing parses. +func extractJSON(text string) json.RawMessage { + candidates := []string{strings.TrimSpace(text)} + if i := strings.Index(text, "```"); i >= 0 { + rest := text[i+3:] + if nl := strings.IndexByte(rest, '\n'); nl >= 0 { + rest = rest[nl+1:] + } + if j := strings.Index(rest, "```"); j >= 0 { + candidates = append(candidates, strings.TrimSpace(rest[:j])) + } + } + candidates = append(candidates, jsonextract.Object(text)) + for _, c := range candidates { + if c != "" && json.Valid([]byte(c)) { + return json.RawMessage(c) + } + } + return nil +} + +// --- process ----------------------------------------------------------------- + +// processBackend runs the caller's command with the prompt on stdin and +// takes stdout as the answer. It restricts nothing and enforces no turn +// limit, so a request for either is refused. +type processBackend struct{} + +func (processBackend) Name() string { return api.StepBackendProcess } + +func (processBackend) Command(spec Spec) ([]string, error) { + if len(spec.Command) == 0 || strings.TrimSpace(spec.Command[0]) == "" { + return nil, &RequestError{"backend process needs command"} + } + if len(spec.Tools) > 0 { + return nil, &UnsupportedError{"backend process cannot restrict tools; it offers whatever the command offers"} + } + if spec.TurnLimit > 0 { + return nil, &UnsupportedError{"backend process cannot enforce a turn limit"} + } + if spec.Model != "" { + return nil, &UnsupportedError{"backend process takes no model; put it in the command"} + } + quoted := make([]string, len(spec.Command)) + for i, a := range spec.Command { + quoted[i] = shellQuote(a) + } + return shell(strings.Join(quoted, " "), spec.PromptRel), nil +} + +func (processBackend) Parse(_ Spec, res sandbox.Result, _ func(string) ([]byte, error)) Outcome { + if res.ExitCode != 0 { + return Outcome{Status: api.StepBackendError, Reason: fmt.Sprintf("command exited %d", res.ExitCode)} + } + return Outcome{Answer: extractJSON(res.Stdout), Turns: 1} +} + +// --- claude ------------------------------------------------------------------ + +// claudeBackend is `claude -p --output-format json`: one result object on +// stdout with the final text in result, token usage in usage, the turn +// count in num_turns and, when a tool outside --allowedTools was asked +// for, permission_denials. An empty Tools list runs with +// --dangerously-skip-permissions (every tool, the existing sandbox-kick +// behaviour); a non-empty one passes --allowedTools and leaves permission +// prompting on, so anything else is denied in -p mode. +type claudeBackend struct{} + +func (claudeBackend) Name() string { return api.StepBackendClaude } + +func (claudeBackend) Command(spec Spec) ([]string, error) { + if len(spec.Command) > 0 { + return nil, &RequestError{"backend claude builds its own command; command is for backend process"} + } + cmd := "claude -p --output-format json" + if spec.Model != "" { + cmd += " --model " + shellQuote(spec.Model) + } + if len(spec.Tools) > 0 { + cmd += " --allowedTools " + shellQuote(strings.Join(spec.Tools, ",")) + } else { + cmd += " --dangerously-skip-permissions" + } + if spec.TurnLimit > 0 { + cmd += fmt.Sprintf(" --max-turns %d", spec.TurnLimit) + } + return shell(cmd, spec.PromptRel), nil +} + +// claudeResult is the subset of the -p JSON result crux reads. +type claudeResult struct { + Type string `json:"type"` + Subtype string `json:"subtype"` + IsError bool `json:"is_error"` + Result string `json:"result"` + NumTurns int `json:"num_turns"` + Usage struct { + InputTokens int64 `json:"input_tokens"` + OutputTokens int64 `json:"output_tokens"` + } `json:"usage"` + PermissionDenials []struct { + ToolName string `json:"tool_name"` + } `json:"permission_denials"` +} + +func (claudeBackend) Parse(_ Spec, res sandbox.Result, _ func(string) ([]byte, error)) Outcome { + raw := extractJSON(res.Stdout) + var r claudeResult + if raw == nil || json.Unmarshal(raw, &r) != nil || r.Type != "result" { + return Outcome{Status: api.StepBackendError, Reason: fmt.Sprintf("claude exited %d without a result object", res.ExitCode)} + } + out := Outcome{Turns: r.NumTurns} + out.Usage.InputTokens, out.Usage.OutputTokens = r.Usage.InputTokens, r.Usage.OutputTokens + switch { + case len(r.PermissionDenials) > 0: + names := make([]string, len(r.PermissionDenials)) + for i, d := range r.PermissionDenials { + names[i] = d.ToolName + } + out.Status, out.Reason = api.StepToolDenied, "tool outside the allowlist requested: "+strings.Join(names, ", ") + case r.Subtype == "error_max_turns": + out.Status, out.Reason = api.StepTurnLimitExceeded, fmt.Sprintf("stopped after %d turns without an answer", r.NumTurns) + case r.IsError || (r.Subtype != "" && r.Subtype != "success"): + out.Status, out.Reason = api.StepBackendError, "claude reported "+r.Subtype + default: + out.Answer = extractJSON(r.Result) + } + return out +} + +// --- codex ------------------------------------------------------------------- + +// codexBackend is `codex exec --json`: a JSONL event stream on stdout and, +// with --output-last-message, the final message in a file crux names +// under the workspace. Usage comes from the turn.completed event. codex +// exec is one turn by construction, so a TurnLimit above 1 is refused, and +// it has no per-tool allowlist, so Tools is refused. The CLI's own sandbox +// and approvals are bypassed: crux's sandbox is the containment. +type codexBackend struct{} + +func (codexBackend) Name() string { return api.StepBackendCodex } + +func (codexBackend) Command(spec Spec) ([]string, error) { + if len(spec.Command) > 0 { + return nil, &RequestError{"backend codex builds its own command; command is for backend process"} + } + if len(spec.Tools) > 0 { + return nil, &UnsupportedError{"backend codex cannot restrict tools to an allowlist"} + } + if spec.TurnLimit > 1 { + return nil, &UnsupportedError{"backend codex runs exactly one turn; turn_limit above 1 cannot be enforced"} + } + cmd := "codex exec --json --skip-git-repo-check --dangerously-bypass-approvals-and-sandbox" + if spec.Model != "" { + cmd += " --model " + shellQuote(spec.Model) + } + cmd += " --output-last-message " + shellQuote(spec.AnswerRel) + " -" + return shell(cmd, spec.PromptRel), nil +} + +// codexEvent is the subset of the exec --json events crux reads. +type codexEvent struct { + Type string `json:"type"` + Usage *struct { + InputTokens int64 `json:"input_tokens"` + OutputTokens int64 `json:"output_tokens"` + } `json:"usage"` + Item *struct { + Type string `json:"type"` + Text string `json:"text"` + } `json:"item"` + Message string `json:"message"` +} + +func (codexBackend) Parse(spec Spec, res sandbox.Result, readFile func(string) ([]byte, error)) Outcome { + out := Outcome{Turns: 1} + var last, failure string + for _, line := range strings.Split(res.Stdout, "\n") { + var ev codexEvent + if json.Unmarshal([]byte(line), &ev) != nil { + continue + } + if ev.Usage != nil { + out.Usage.InputTokens += ev.Usage.InputTokens + out.Usage.OutputTokens += ev.Usage.OutputTokens + } + if ev.Item != nil && ev.Item.Type == "agent_message" { + last = ev.Item.Text + } + if ev.Type == "error" || ev.Type == "turn.failed" { + failure = ev.Message + } + } + if res.ExitCode != 0 { + if failure == "" { + failure = fmt.Sprintf("codex exited %d", res.ExitCode) + } + out.Status, out.Reason = api.StepBackendError, failure + return out + } + if readFile != nil { + if data, err := readFile(spec.AnswerRel); err == nil && len(strings.TrimSpace(string(data))) > 0 { + last = string(data) + } + } + out.Answer = extractJSON(last) + return out +} + +// ErrNoAnswer is the reason an attempt that ran clean produced nothing a +// schema could be checked against. +var ErrNoAnswer = errors.New("the backend returned no JSON value") diff --git a/src/crux/agentstep/backend_test.go b/src/crux/agentstep/backend_test.go new file mode 100644 index 0000000000..cf25bb17d9 --- /dev/null +++ b/src/crux/agentstep/backend_test.go @@ -0,0 +1,145 @@ +package agentstep + +import ( + "errors" + "strings" + "testing" + + "github.com/hivecommons/hive/crux/api" + "github.com/hivecommons/hive/crux/sandbox" +) + +func TestLookupKnowsTheThreeBackends(t *testing.T) { + for _, name := range []string{api.StepBackendClaude, api.StepBackendCodex, api.StepBackendProcess} { + b, err := Lookup(name) + if err != nil || b.Name() != name { + t.Fatalf("Lookup(%s) = %v, %v", name, b, err) + } + } + var reqErr *RequestError + if _, err := Lookup("gemini"); !errors.As(err, &reqErr) || !strings.Contains(err.Error(), "claude") { + t.Fatalf("Lookup(gemini) = %v", err) + } +} + +func TestProcessBackendCommandAndParse(t *testing.T) { + spec := Spec{PromptRel: ".crux/steps/1/prompt-1.md", Command: []string{"python3", "ask.py", "it's"}} + cmd, err := Backends["process"].Command(spec) + if err != nil { + t.Fatal(err) + } + want := []string{"sh", "-lc", `'python3' 'ask.py' 'it'\''s' < '.crux/steps/1/prompt-1.md'`} + if strings.Join(cmd, "\x00") != strings.Join(want, "\x00") { + t.Fatalf("Command = %q", cmd) + } + var reqErr *RequestError + var unsupported *UnsupportedError + if _, err := Backends["process"].Command(Spec{}); !errors.As(err, &reqErr) { + t.Errorf("no command: %v", err) + } + for _, s := range []Spec{ + {Command: []string{"x"}, Tools: []string{"Bash"}}, + {Command: []string{"x"}, TurnLimit: 2}, + {Command: []string{"x"}, Model: "m"}, + } { + if _, err := Backends["process"].Command(s); !errors.As(err, &unsupported) { + t.Errorf("%+v: %v", s, err) + } + } + out := Backends["process"].Parse(spec, sandbox.Result{Stdout: "note\n```json\n{\"a\":1}\n```\n"}, nil) + if out.Status != "" || string(out.Answer) != `{"a":1}` { + t.Fatalf("Parse fenced = %+v", out) + } + out = Backends["process"].Parse(spec, sandbox.Result{ExitCode: 3}, nil) + if out.Status != api.StepBackendError || !strings.Contains(out.Reason, "exited 3") { + t.Fatalf("Parse failure = %+v", out) + } + if out := Backends["process"].Parse(spec, sandbox.Result{Stdout: "nothing here"}, nil); out.Answer != nil { + t.Fatalf("Parse prose = %+v", out) + } +} + +func TestClaudeBackendCommandAndParse(t *testing.T) { + cmd, err := Backends["claude"].Command(Spec{PromptRel: "p.md", Model: "opus", Tools: []string{"Read", "Grep"}, TurnLimit: 3}) + if err != nil { + t.Fatal(err) + } + if line := cmd[2]; !strings.Contains(line, "claude -p --output-format json --model 'opus' --allowedTools 'Read,Grep' --max-turns 3 < 'p.md'") { + t.Fatalf("Command = %q", line) + } + cmd, _ = Backends["claude"].Command(Spec{PromptRel: "p.md"}) + if !strings.Contains(cmd[2], "--dangerously-skip-permissions") || strings.Contains(cmd[2], "--max-turns") { + t.Fatalf("no-tools command = %q", cmd[2]) + } + var reqErr *RequestError + if _, err := Backends["claude"].Command(Spec{Command: []string{"x"}}); !errors.As(err, &reqErr) { + t.Errorf("command with claude: %v", err) + } + for _, tc := range []struct { + stdout string + status string + answer string + }{ + {`{"type":"result","subtype":"success","result":"Sure: {\"ok\":true}","num_turns":2,"usage":{"input_tokens":10,"output_tokens":5}}`, "", `{"ok":true}`}, + {`{"type":"result","subtype":"success","result":"no json","num_turns":1}`, "", ""}, + {`{"type":"result","subtype":"success","result":"{}","permission_denials":[{"tool_name":"Bash"}]}`, api.StepToolDenied, ""}, + {`{"type":"result","subtype":"error_max_turns","num_turns":3}`, api.StepTurnLimitExceeded, ""}, + {`{"type":"result","subtype":"error_during_execution","is_error":true}`, api.StepBackendError, ""}, + {`garbage`, api.StepBackendError, ""}, + } { + out := Backends["claude"].Parse(Spec{}, sandbox.Result{Stdout: tc.stdout}, nil) + if out.Status != tc.status || string(out.Answer) != tc.answer { + t.Errorf("Parse(%s) = %+v", tc.stdout, out) + } + if tc.answer != "" && (out.Usage.InputTokens != 10 || out.Usage.OutputTokens != 5 || out.Turns != 2) { + t.Errorf("usage = %+v", out) + } + } +} + +func TestCodexBackendCommandAndParse(t *testing.T) { + cmd, err := Backends["codex"].Command(Spec{PromptRel: "p.md", AnswerRel: "a.txt", Model: "o3", TurnLimit: 1}) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(cmd[2], "codex exec --json --skip-git-repo-check --dangerously-bypass-approvals-and-sandbox --model 'o3' --output-last-message 'a.txt' - < 'p.md'") { + t.Fatalf("Command = %q", cmd[2]) + } + var unsupported *UnsupportedError + if _, err := Backends["codex"].Command(Spec{Tools: []string{"x"}}); !errors.As(err, &unsupported) { + t.Errorf("tools: %v", err) + } + if _, err := Backends["codex"].Command(Spec{TurnLimit: 2}); !errors.As(err, &unsupported) { + t.Errorf("turn limit: %v", err) + } + stdout := strings.Join([]string{ + `{"type":"thread.started"}`, + `not json`, + `{"type":"item.completed","item":{"type":"agent_message","text":"{\"from\":\"stdout\"}"}}`, + `{"type":"turn.completed","usage":{"input_tokens":7,"output_tokens":3}}`, + }, "\n") + files := map[string]string{"a.txt": "{\"from\":\"file\"}\n"} + read := func(rel string) ([]byte, error) { + if v, ok := files[rel]; ok { + return []byte(v), nil + } + return nil, errors.New("missing") + } + spec := Spec{AnswerRel: "a.txt"} + out := Backends["codex"].Parse(spec, sandbox.Result{Stdout: stdout}, read) + if out.Status != "" || string(out.Answer) != `{"from":"file"}` || out.Usage.InputTokens != 7 || out.Usage.OutputTokens != 3 { + t.Fatalf("Parse = %+v", out) + } + out = Backends["codex"].Parse(Spec{AnswerRel: "none"}, sandbox.Result{Stdout: stdout}, read) + if string(out.Answer) != `{"from":"stdout"}` { + t.Fatalf("Parse without a file = %+v", out) + } + out = Backends["codex"].Parse(spec, sandbox.Result{Stdout: `{"type":"error","message":"quota"}`, ExitCode: 1}, read) + if out.Status != api.StepBackendError || out.Reason != "quota" { + t.Fatalf("Parse failure = %+v", out) + } + out = Backends["codex"].Parse(spec, sandbox.Result{ExitCode: 2}, read) + if out.Status != api.StepBackendError || !strings.Contains(out.Reason, "exited 2") { + t.Fatalf("Parse silent failure = %+v", out) + } +} diff --git a/src/crux/agentstep/schema.go b/src/crux/agentstep/schema.go new file mode 100644 index 0000000000..078b6e723d --- /dev/null +++ b/src/crux/agentstep/schema.go @@ -0,0 +1,406 @@ +package agentstep + +import ( + "encoding/json" + "errors" + "fmt" + "math" + "regexp" + "sort" + "strconv" + "strings" + + "github.com/hivecommons/hive/crux/api" +) + +// MaxSchemaBytes bounds the schema a request may carry. +const MaxSchemaBytes = 256 << 10 + +// ErrSchema is a schema that is not usable: not an object, over the size +// bound, or using a keyword this validator does not implement (a keyword +// it cannot enforce is refused rather than ignored, so a caller never +// believes a constraint was checked when it was not). +var ErrSchema = errors.New("agentstep: invalid output_schema") + +// Schema is the subset of JSON Schema (draft 2020-12 vocabulary) crux +// checks answers against: type, enum, const, properties, required, +// additionalProperties (boolean or schema), items, minItems, maxItems, +// minLength, maxLength, pattern, minimum, maximum, exclusiveMinimum, +// exclusiveMaximum, multipleOf and the annotations title, description, +// default, examples and $schema. Anything else is ErrSchema. +type Schema struct { + types []string + enum []json.RawMessage + constant json.RawMessage + hasConst bool + properties map[string]*Schema + required []string + additional *Schema + additionalForbidden bool + items *Schema + minItems, maxItems *int + minLength, maxLength *int + pattern *regexp.Regexp + minimum, maximum *float64 + exclusiveMin, exclusiveMax *float64 + multipleOf *float64 +} + +// annotationKeywords are accepted and ignored: they constrain nothing. +var annotationKeywords = map[string]bool{ + "title": true, "description": true, "default": true, "examples": true, "$schema": true, "$id": true, "$comment": true, +} + +// ParseSchema parses raw into a Schema or returns ErrSchema. +func ParseSchema(raw json.RawMessage) (*Schema, error) { + if len(raw) == 0 { + return nil, fmt.Errorf("%w: output_schema is required", ErrSchema) + } + if len(raw) > MaxSchemaBytes { + return nil, fmt.Errorf("%w: %d bytes exceeds the %d-byte bound", ErrSchema, len(raw), MaxSchemaBytes) + } + return parseSchema(raw, "$") +} + +func parseSchema(raw json.RawMessage, at string) (*Schema, error) { + var fields map[string]json.RawMessage + if err := json.Unmarshal(raw, &fields); err != nil || fields == nil { + if err == nil { + // JSON null decodes to a nil map without error. + return nil, fmt.Errorf("%w: %s must be an object", ErrSchema, at) + } + // A bare boolean schema: true accepts anything, false nothing. + var b bool + if json.Unmarshal(raw, &b) == nil { + if b { + return &Schema{}, nil + } + return &Schema{enum: []json.RawMessage{}}, nil + } + return nil, fmt.Errorf("%w: %s must be an object", ErrSchema, at) + } + s := &Schema{} + for key, val := range fields { + var err error + switch key { + case "type": + err = s.parseType(val, at) + case "enum": + err = json.Unmarshal(val, &s.enum) + case "const": + s.constant, s.hasConst = val, true + case "properties": + var props map[string]json.RawMessage + if err = json.Unmarshal(val, &props); err == nil { + s.properties = map[string]*Schema{} + for name, sub := range props { + if s.properties[name], err = parseSchema(sub, at+"."+name); err != nil { + return nil, err + } + } + } + case "required": + err = json.Unmarshal(val, &s.required) + case "additionalProperties": + var b bool + if json.Unmarshal(val, &b) == nil { + s.additionalForbidden = !b + } else if s.additional, err = parseSchema(val, at+".*"); err != nil { + return nil, err + } + case "items": + if s.items, err = parseSchema(val, at+"[]"); err != nil { + return nil, err + } + case "minItems": + s.minItems, err = parseInt(val) + case "maxItems": + s.maxItems, err = parseInt(val) + case "minLength": + s.minLength, err = parseInt(val) + case "maxLength": + s.maxLength, err = parseInt(val) + case "pattern": + var p string + if err = json.Unmarshal(val, &p); err == nil { + s.pattern, err = regexp.Compile(p) + } + case "minimum": + s.minimum, err = parseNumber(val) + case "maximum": + s.maximum, err = parseNumber(val) + case "exclusiveMinimum": + s.exclusiveMin, err = parseNumber(val) + case "exclusiveMaximum": + s.exclusiveMax, err = parseNumber(val) + case "multipleOf": + if s.multipleOf, err = parseNumber(val); err == nil && *s.multipleOf <= 0 { + err = errors.New("must be positive") + } + default: + if annotationKeywords[key] { + continue + } + return nil, fmt.Errorf("%w: %s uses keyword %q, which this validator does not enforce", ErrSchema, at, key) + } + if err != nil { + return nil, fmt.Errorf("%w: %s.%s: %v", ErrSchema, at, key, err) + } + } + return s, nil +} + +func (s *Schema) parseType(val json.RawMessage, at string) error { + var one string + if json.Unmarshal(val, &one) == nil { + s.types = []string{one} + } else if err := json.Unmarshal(val, &s.types); err != nil { + return errors.New("must be a string or an array of strings") + } + for _, t := range s.types { + switch t { + case "object", "array", "string", "number", "integer", "boolean", "null": + default: + return fmt.Errorf("unknown type %q", t) + } + } + return nil +} + +func parseInt(val json.RawMessage) (*int, error) { + var n int + if err := json.Unmarshal(val, &n); err != nil || n < 0 { + return nil, errors.New("must be a non-negative integer") + } + return &n, nil +} + +func parseNumber(val json.RawMessage) (*float64, error) { + var f float64 + if err := json.Unmarshal(val, &f); err != nil { + return nil, errors.New("must be a number") + } + return &f, nil +} + +// Validate checks answer against the schema and returns every violation +// found, in path order; none means the answer matches. answer must be one +// JSON value. +func (s *Schema) Validate(answer json.RawMessage) []api.SchemaViolation { + var v any + dec := json.NewDecoder(strings.NewReader(string(answer))) + dec.UseNumber() + if err := dec.Decode(&v); err != nil { + return []api.SchemaViolation{{Path: "$", Message: "not valid JSON: " + err.Error()}} + } + if dec.More() { + return []api.SchemaViolation{{Path: "$", Message: "must be exactly one JSON value"}} + } + var out []api.SchemaViolation + s.check(v, "$", &out) + sort.SliceStable(out, func(i, j int) bool { return out[i].Path < out[j].Path }) + return out +} + +func (s *Schema) check(v any, path string, out *[]api.SchemaViolation) { + add := func(msg string) { *out = append(*out, api.SchemaViolation{Path: path, Message: msg}) } + if len(s.types) > 0 && !typeMatches(s.types, v) { + add(fmt.Sprintf("expected %s, got %s", strings.Join(s.types, " or "), typeName(v))) + return + } + if s.hasConst && !jsonEqual(s.constant, v) { + add("must equal the const value") + } + if s.enum != nil { + found := false + for _, e := range s.enum { + if jsonEqual(e, v) { + found = true + break + } + } + if !found { + add("not one of the enum values") + } + } + switch x := v.(type) { + case map[string]any: + s.checkObject(x, path, out) + case []any: + s.checkArray(x, path, out) + case string: + s.checkString(x, add) + case json.Number: + s.checkNumber(x, add) + } +} + +func (s *Schema) checkObject(x map[string]any, path string, out *[]api.SchemaViolation) { + for _, name := range s.required { + if _, ok := x[name]; !ok { + *out = append(*out, api.SchemaViolation{Path: path + "." + name, Message: "required property is missing"}) + } + } + keys := make([]string, 0, len(x)) + for k := range x { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + sub, declared := s.properties[k] + switch { + case declared: + sub.check(x[k], path+"."+k, out) + case s.additionalForbidden: + *out = append(*out, api.SchemaViolation{Path: path + "." + k, Message: "property is not allowed"}) + case s.additional != nil: + s.additional.check(x[k], path+"."+k, out) + } + } +} + +func (s *Schema) checkArray(x []any, path string, out *[]api.SchemaViolation) { + if s.minItems != nil && len(x) < *s.minItems { + *out = append(*out, api.SchemaViolation{Path: path, Message: fmt.Sprintf("needs at least %d items, has %d", *s.minItems, len(x))}) + } + if s.maxItems != nil && len(x) > *s.maxItems { + *out = append(*out, api.SchemaViolation{Path: path, Message: fmt.Sprintf("allows at most %d items, has %d", *s.maxItems, len(x))}) + } + if s.items != nil { + for i, item := range x { + s.items.check(item, path+"["+strconv.Itoa(i)+"]", out) + } + } +} + +func (s *Schema) checkString(x string, add func(string)) { + n := len([]rune(x)) + if s.minLength != nil && n < *s.minLength { + add(fmt.Sprintf("shorter than %d characters", *s.minLength)) + } + if s.maxLength != nil && n > *s.maxLength { + add(fmt.Sprintf("longer than %d characters", *s.maxLength)) + } + if s.pattern != nil && !s.pattern.MatchString(x) { + add("does not match pattern " + s.pattern.String()) + } +} + +func (s *Schema) checkNumber(x json.Number, add func(string)) { + f, err := x.Float64() + if err != nil { + add("not a representable number") + return + } + if s.minimum != nil && f < *s.minimum { + add(fmt.Sprintf("below minimum %v", *s.minimum)) + } + if s.maximum != nil && f > *s.maximum { + add(fmt.Sprintf("above maximum %v", *s.maximum)) + } + if s.exclusiveMin != nil && f <= *s.exclusiveMin { + add(fmt.Sprintf("not above %v", *s.exclusiveMin)) + } + if s.exclusiveMax != nil && f >= *s.exclusiveMax { + add(fmt.Sprintf("not below %v", *s.exclusiveMax)) + } + if s.multipleOf != nil { + q := f / *s.multipleOf + if math.Abs(q-math.Round(q)) > 1e-9 { + add(fmt.Sprintf("not a multiple of %v", *s.multipleOf)) + } + } +} + +func typeMatches(types []string, v any) bool { + for _, t := range types { + switch t { + case "null": + if v == nil { + return true + } + case "boolean": + if _, ok := v.(bool); ok { + return true + } + case "string": + if _, ok := v.(string); ok { + return true + } + case "object": + if _, ok := v.(map[string]any); ok { + return true + } + case "array": + if _, ok := v.([]any); ok { + return true + } + case "number": + if _, ok := v.(json.Number); ok { + return true + } + case "integer": + if n, ok := v.(json.Number); ok { + if _, err := n.Int64(); err == nil { + return true + } + if f, err := n.Float64(); err == nil && f == math.Trunc(f) { + return true + } + } + } + } + return false +} + +func typeName(v any) string { + switch x := v.(type) { + case nil: + return "null" + case bool: + return "boolean" + case string: + return "string" + case json.Number: + if _, err := x.Int64(); err == nil { + return "integer" + } + return "number" + case []any: + return "array" + case map[string]any: + return "object" + } + return fmt.Sprintf("%T", v) +} + +// jsonEqual compares a raw schema value with a decoded answer value by +// re-encoding both canonically. +func jsonEqual(raw json.RawMessage, v any) bool { + var want any + dec := json.NewDecoder(strings.NewReader(string(raw))) + dec.UseNumber() + if err := dec.Decode(&want); err != nil { + return false + } + a, err1 := json.Marshal(want) + b, err2 := json.Marshal(v) + return err1 == nil && err2 == nil && string(a) == string(b) +} + +// CorrectivePrompt is the repair instruction appended to the prompt of the +// attempt after a schema mismatch: the violations, verbatim, and the shape +// asked for. The pattern is outputschema.CorrectivePrompt, generalised to +// a caller-supplied schema. +func CorrectivePrompt(schema json.RawMessage, violations []api.SchemaViolation, remaining int) string { + var b strings.Builder + b.WriteString("Your previous answer did not match the required JSON Schema and could not be used. ") + b.WriteString("Reply with exactly one JSON value matching this schema and nothing else:\n") + b.Write(schema) + b.WriteString("\nFix these validation errors:\n") + for _, v := range violations { + fmt.Fprintf(&b, "- %s: %s\n", v.Path, v.Message) + } + fmt.Fprintf(&b, "Attempts remaining after this one: %d.", remaining) + return b.String() +} diff --git a/src/crux/agentstep/schema_test.go b/src/crux/agentstep/schema_test.go new file mode 100644 index 0000000000..b4a88bf042 --- /dev/null +++ b/src/crux/agentstep/schema_test.go @@ -0,0 +1,103 @@ +package agentstep + +import ( + "encoding/json" + "errors" + "strings" + "testing" +) + +func mustSchema(t *testing.T, raw string) *Schema { + t.Helper() + s, err := ParseSchema(json.RawMessage(raw)) + if err != nil { + t.Fatalf("ParseSchema(%s): %v", raw, err) + } + return s +} + +func paths(s *Schema, answer string) string { + var out []string + for _, v := range s.Validate(json.RawMessage(answer)) { + out = append(out, v.Path) + } + return strings.Join(out, " ") +} + +func TestSchemaValidatesTheSupportedKeywords(t *testing.T) { + s := mustSchema(t, `{ + "type": "object", + "required": ["verdict", "items"], + "additionalProperties": false, + "properties": { + "verdict": {"enum": ["pass", "fail"]}, + "score": {"type": "number", "minimum": 0, "maximum": 1, "exclusiveMaximum": 1}, + "count": {"type": "integer", "multipleOf": 2}, + "name": {"type": "string", "minLength": 2, "maxLength": 4, "pattern": "^[a-z]+$"}, + "items": {"type": "array", "minItems": 1, "maxItems": 2, "items": {"type": "string"}}, + "flag": {"const": true}, + "any": true, + "never": false, + "nullable": {"type": ["string", "null"]}, + "nested": {"type": "object", "additionalProperties": {"type": "integer"}} + } + }`) + for _, tc := range []struct{ answer, want string }{ + {`{"verdict":"pass","items":["a"]}`, ""}, + {`{"verdict":"pass","items":["a"],"score":0.5,"count":4,"name":"ab","flag":true,"any":[1],"nullable":null,"nested":{"x":1}}`, ""}, + {`{"verdict":"maybe","items":[]}`, "$.items $.verdict"}, + {`{"items":["a"]}`, "$.verdict"}, + {`{"verdict":"pass","items":["a"],"extra":1}`, "$.extra"}, + {`{"verdict":"pass","items":["a"],"score":1}`, "$.score"}, + {`{"verdict":"pass","items":["a"],"score":-1}`, "$.score"}, + {`{"verdict":"pass","items":["a"],"count":3}`, "$.count"}, + {`{"verdict":"pass","items":["a"],"count":2.5}`, "$.count"}, + {`{"verdict":"pass","items":["a"],"name":"a"}`, "$.name"}, + {`{"verdict":"pass","items":["a"],"name":"abcde"}`, "$.name"}, + {`{"verdict":"pass","items":["a"],"name":"AB"}`, "$.name"}, + {`{"verdict":"pass","items":["a","b","c"]}`, "$.items"}, + {`{"verdict":"pass","items":[1]}`, "$.items[0]"}, + {`{"verdict":"pass","items":["a"],"flag":false}`, "$.flag"}, + {`{"verdict":"pass","items":["a"],"never":1}`, "$.never"}, + {`{"verdict":"pass","items":["a"],"nullable":1}`, "$.nullable"}, + {`{"verdict":"pass","items":["a"],"nested":{"x":"y"}}`, "$.nested.x"}, + {`[]`, "$"}, + {`not json`, "$"}, + } { + if got := paths(s, tc.answer); got != tc.want { + t.Errorf("Validate(%s) violated %q, want %q", tc.answer, got, tc.want) + } + } +} + +func TestSchemaParseRefusesWhatItCannotEnforce(t *testing.T) { + for _, raw := range []string{ + ``, `null`, `[]`, `{"type": "thing"}`, `{"type": 3}`, `{"$ref": "#/x"}`, `{"oneOf": []}`, + `{"minimum": "1"}`, `{"minItems": -1}`, `{"pattern": "("}`, `{"properties": {"a": 1}}`, + `{"required": "a"}`, `{"enum": 1}`, `{"additionalProperties": 1}`, `{"items": [1]}`, + `{"multipleOf": 0}`, + } { + if _, err := ParseSchema(json.RawMessage(raw)); !errors.Is(err, ErrSchema) { + t.Errorf("ParseSchema(%s) = %v, want ErrSchema", raw, err) + } + } + if _, err := ParseSchema(json.RawMessage(`{"type":"string","` + strings.Repeat("x", MaxSchemaBytes) + `":1}`)); !errors.Is(err, ErrSchema) { + t.Errorf("oversized schema: %v", err) + } + // Annotations are allowed and ignored. + s := mustSchema(t, `{"$schema":"https://json-schema.org/draft/2020-12/schema","title":"t","description":"d","examples":[1],"default":1,"$id":"x","$comment":"c","type":"integer"}`) + if got := paths(s, `1`); got != "" { + t.Errorf("annotated schema violated %q", got) + } +} + +func TestCorrectivePromptNamesTheViolations(t *testing.T) { + s := mustSchema(t, `{"type":"object","required":["a"]}`) + v := s.Validate(json.RawMessage(`{}`)) + got := CorrectivePrompt(json.RawMessage(`{"type":"object","required":["a"]}`), v, 2) + for _, want := range []string{`"required":["a"]`, "- $.a: ", "remaining after this one: 2"} { + if !strings.Contains(got, want) { + t.Errorf("CorrectivePrompt lacks %q:\n%s", want, got) + } + } +} diff --git a/src/crux/agentstep/store.go b/src/crux/agentstep/store.go new file mode 100644 index 0000000000..876fc23815 --- /dev/null +++ b/src/crux/agentstep/store.go @@ -0,0 +1,159 @@ +package agentstep + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "sort" + "strconv" + "sync" + + "github.com/hivecommons/hive/crux/api" +) + +// Storage under the run's directory: +// +// steps//record.json the step's saved record (api.Step) +// steps//transcript.jsonl one line per attempt: command, exit, +// scrubbed output, status, violations, usage +// +// Steps are numbered from 1 in the order they were created. +const ( + StepsDir = "steps" + RecordName = "record.json" + TranscriptName = "transcript.jsonl" + fileMode = 0o600 + dirMode = 0o700 +) + +// Typed errors the serve routes map onto api.Code* values. +var ( + // ErrNotFound: no such step number for this run. + ErrNotFound = errors.New("agentstep: step not found") + // ErrCorrupt: a saved record did not parse. + ErrCorrupt = errors.New("agentstep: corrupt record") +) + +// createMu serialises step numbering across the process: two sessions +// never share a run directory, but one session may receive two posts. +var createMu sync.Mutex + +// Store reads and writes the steps of one run directory. +type Store struct{ RunDir string } + +// Dir is the directory of step n. +func (s Store) Dir(n int) string { return filepath.Join(s.RunDir, StepsDir, strconv.Itoa(n)) } + +// TranscriptRel is the transcript path relative to the run directory, as +// the record reports it. +func (s Store) TranscriptRel(n int) string { + return filepath.ToSlash(filepath.Join(StepsDir, strconv.Itoa(n), TranscriptName)) +} + +// Create assigns the next step number, writes the record and returns it. +func (s Store) Create(step api.Step) (api.Step, error) { + createMu.Lock() + defer createMu.Unlock() + steps, err := s.List() + if err != nil && !errors.Is(err, ErrCorrupt) { + return api.Step{}, err + } + step.N = 1 + if len(steps) > 0 { + step.N = steps[len(steps)-1].N + 1 + } + if err := os.MkdirAll(s.Dir(step.N), dirMode); err != nil { + return api.Step{}, fmt.Errorf("agentstep: %w", err) + } + return step, s.Write(step) +} + +// Write saves step's record, tmp+rename. +func (s Store) Write(step api.Step) error { + data, err := json.Marshal(step) + if err != nil { + return fmt.Errorf("agentstep: %w", err) + } + path := filepath.Join(s.Dir(step.N), RecordName) + tmp := path + ".tmp" + if err := os.WriteFile(tmp, data, fileMode); err != nil { + return fmt.Errorf("agentstep: %w", err) + } + if err := os.Rename(tmp, path); err != nil { + return fmt.Errorf("agentstep: %w", err) + } + return nil +} + +// Get is step n's saved record. +func (s Store) Get(n int) (api.Step, error) { + if n <= 0 { + return api.Step{}, ErrNotFound + } + data, err := os.ReadFile(filepath.Join(s.Dir(n), RecordName)) + if errors.Is(err, os.ErrNotExist) { + return api.Step{}, ErrNotFound + } + if err != nil { + return api.Step{}, fmt.Errorf("agentstep: %w", err) + } + var step api.Step + if err := json.Unmarshal(data, &step); err != nil { + return api.Step{}, fmt.Errorf("%w: step %d: %v", ErrCorrupt, n, err) + } + return step, nil +} + +// List is every step of the run, by number. A run with no steps yet is an +// empty list. A record that does not parse is reported after the ones +// that do, as ErrCorrupt, so a caller still sees the rest. +func (s Store) List() ([]api.Step, error) { + entries, err := os.ReadDir(filepath.Join(s.RunDir, StepsDir)) + if errors.Is(err, os.ErrNotExist) { + return []api.Step{}, nil + } + if err != nil { + return nil, fmt.Errorf("agentstep: %w", err) + } + var nums []int + for _, e := range entries { + if n, err := strconv.Atoi(e.Name()); err == nil && n > 0 && e.IsDir() { + nums = append(nums, n) + } + } + sort.Ints(nums) + steps := make([]api.Step, 0, len(nums)) + var corrupt error + for _, n := range nums { + step, err := s.Get(n) + switch { + case errors.Is(err, ErrNotFound): + // A directory without a record: a create that did not finish. + continue + case err != nil: + corrupt = err + continue + } + steps = append(steps, step) + } + return steps, corrupt +} + +// AppendTranscript adds one attempt line to step n's transcript. +func (s Store) AppendTranscript(n int, a attempt) error { + line, err := json.Marshal(a) + if err != nil { + return err + } + f, err := os.OpenFile(filepath.Join(s.Dir(n), TranscriptName), os.O_CREATE|os.O_APPEND|os.O_WRONLY, fileMode) + if err != nil { + return err + } + _, werr := f.Write(append(line, '\n')) + if cerr := f.Close(); werr == nil { + werr = cerr + } + return werr +} diff --git a/src/crux/api/runs.go b/src/crux/api/runs.go index fb42c54f57..dd017cc5d1 100644 --- a/src/crux/api/runs.go +++ b/src/crux/api/runs.go @@ -44,8 +44,9 @@ const ( // RunProfileExperiment (the default) runs with network none, no // credentials and time and output limits. RunProfileExperiment = "experiment" - // RunProfileAgent is S6's execution half and is not yet accepted - // (CodeUnsupported). + // RunProfileAgent opens a step session (S6): no command, network + // restricted by default, held waiting until cancelled; steps are + // posted to RunStepsPath. RunProfileAgent = "agent" ) diff --git a/src/crux/api/signals.go b/src/crux/api/signals.go index 2c9505eadd..a325825d13 100644 --- a/src/crux/api/signals.go +++ b/src/crux/api/signals.go @@ -47,8 +47,8 @@ func RunSignalsQuery(id string, sinceSeq int64, limit int) string { // AgentKickOutcomePath is the route of an agent's saved kick outcome. func AgentKickOutcomePath(name string) string { return AgentPath(name) + "/kick-outcome" } -// Signal kinds, the closed set every signal reports. Started and done are -// the runner's; the rest may come from the run's owner, and needs-input, +// Signal kinds, the closed set every signal reports. Started, done and step +// (SignalStep, steps.go) are the runner's; the rest may come from the run's owner, and needs-input, // input and blocked also from a gate opened for the run. const ( // SignalStarted: the sandbox was launched. diff --git a/src/crux/api/steps.go b/src/crux/api/steps.go new file mode 100644 index 0000000000..d17499d446 --- /dev/null +++ b/src/crux/api/steps.go @@ -0,0 +1,178 @@ +package api + +import ( + "context" + "encoding/json" + "net/http" + "strconv" + "time" +) + +// Schema-checked agent steps (hivecommons/crux#114 S6): run one agent turn +// in the workspace of an agent-profile contained run and get the answer +// back validated against a JSON shape. +// +// POST /v1/runs/{id}/steps run a step in the session's workspace +// GET /v1/runs/{id}/steps every step of the session, oldest first +// GET /v1/runs/{id}/steps/{n} one step's saved record +// +// A session is a run created with profile "agent" and no command: it +// provisions its workspace, reports "waiting" and stays open until it is +// cancelled, which is how a driver closes it. The routes admit the run's +// owner and the operator, like the run itself. Each step is appended to the +// run's command log (S4) and announced with a "step" turn signal (S5). + +// RunStepsPath is the route of one run's steps. +func RunStepsPath(id string) string { return RunPath(id) + "/steps" } + +// RunStepPath is the route of one step's saved record. +func RunStepPath(id string, n int) string { return RunStepsPath(id) + "/" + strconv.Itoa(n) } + +// Step statuses, the closed set every record reports. Every terminal status +// carries the attempts made, their usage and the transcript path, so a +// driver budgets and debugs from a failure as well as a success. +const ( + // StepRunning: the step is in flight. + StepRunning = "running" + // StepSucceeded: the answer matched the schema. + StepSucceeded = "succeeded" + // StepSchemaMismatch: every attempt's answer failed validation; the + // violations of the last one are on the record, no partial answer is. + StepSchemaMismatch = "schema_mismatch" + // StepTimeout: the backend did not finish within timeout_ms. + StepTimeout = "timeout" + // StepBackendError: the backend exited non-zero or its output was not + // what the adapter expects. + StepBackendError = "backend_error" + // StepToolDenied: the backend reported a tool outside the allowlist + // was requested and refused. + StepToolDenied = "tool_denied" + // StepTurnLimitExceeded: the backend stopped at the turn limit before + // answering. + StepTurnLimitExceeded = "turn_limit_exceeded" + // StepCancelled: the session was cancelled while the step ran. + StepCancelled = "cancelled" + // StepUnsupported: the request asked for something the backend cannot + // enforce (a tool allowlist, a turn limit); the step never ran. + StepUnsupported = "unsupported" +) + +// Step backends shipped with crux. Process is the generic adapter: the +// caller names the command, crux feeds the prompt on stdin and reads the +// answer from stdout; it enforces neither tools nor turns and says so. +const ( + StepBackendClaude = "claude" + StepBackendCodex = "codex" + StepBackendProcess = "process" +) + +// CodeStepBusy is a step posted while the session already runs one (409). +const CodeStepBusy = "step_busy" + +// CodeNotSession is a step posted to a run that is not an agent-profile +// session, or whose session has ended (409). +const CodeNotSession = "not_session" + +// SignalStep is the runner's turn signal for a finished step; Data is +// SignalStepData and the message the step's reason. +const SignalStep = "step" + +// SignalStepData is the data of a step signal. +type SignalStepData struct { + Step int `json:"step"` + Status string `json:"status"` + Attempts int `json:"attempts"` +} + +// StepRequest is POST /v1/runs/{id}/steps. +type StepRequest struct { + // Prompt is the step's instruction; PromptPath names a file in the + // workspace holding it instead (for large prompts). Exactly one. + Prompt string `json:"prompt,omitempty"` + PromptPath string `json:"prompt_path,omitempty"` + // Backend is StepBackendClaude, StepBackendCodex or StepBackendProcess. + Backend string `json:"backend"` + // Model, when set, is passed to the backend; empty is its default. + Model string `json:"model,omitempty"` + // Command is the process backend's argv, run in the workspace with the + // prompt on stdin; other backends refuse it. + Command []string `json:"command,omitempty"` + // OutputSchema is the JSON Schema the answer must match. Required. + OutputSchema json.RawMessage `json:"output_schema"` + // Tools is the allowlist of tools offered to the agent; empty is the + // backend's default set. A backend that cannot restrict its tools + // refuses a non-empty list (StepUnsupported). + Tools []string `json:"tools,omitempty"` + // TurnLimit stops the backend after this many turns; zero is the + // backend default. A backend that cannot enforce it refuses it. + TurnLimit int `json:"turn_limit,omitempty"` + // TimeoutMs bounds each attempt; zero uses the session's timeout. + TimeoutMs int64 `json:"timeout_ms,omitempty"` + // RetryOnMismatch is how many corrective attempts follow an answer that + // fails the schema; zero means one attempt only. + RetryOnMismatch int `json:"retry_on_mismatch,omitempty"` +} + +// StepUsage is the token usage a backend reported, summed over attempts. +type StepUsage struct { + InputTokens int64 `json:"input_tokens"` + OutputTokens int64 `json:"output_tokens"` +} + +// SchemaViolation is one way an answer missed the schema. +type SchemaViolation struct { + // Path is the JSON pointer-ish location ("$", "$.items[2].name"). + Path string `json:"path"` + Message string `json:"message"` +} + +// Step is the saved record of one agent step: the response of every steps +// route. Answer is present only on StepSucceeded; Violations only on +// StepSchemaMismatch. +type Step struct { + Run string `json:"run"` + N int `json:"n"` + Status string `json:"status"` + // Reason explains any status other than succeeded. + Reason string `json:"reason,omitempty"` + Backend string `json:"backend"` + Model string `json:"model,omitempty"` + CreatedAt time.Time `json:"created_at"` + FinishedAt *time.Time `json:"finished_at,omitempty"` + // Attempts is how many times the backend ran; each is one line of + // the transcript. + Attempts int `json:"attempts"` + Answer json.RawMessage `json:"answer,omitempty"` + Usage StepUsage `json:"usage"` + // TranscriptPath is the attempts' raw output, relative to the run's + // directory. + TranscriptPath string `json:"transcript_path,omitempty"` + Violations []SchemaViolation `json:"violations,omitempty"` +} + +// StepList is GET /v1/runs/{id}/steps. +type StepList struct { + Run string `json:"run"` + Steps []Step `json:"steps"` +} + +// RunStep is POST /v1/runs/{id}/steps: the finished step. +func (c *Client) RunStep(ctx context.Context, run string, req StepRequest) (Step, error) { + var out Step + err := c.Do(ctx, http.MethodPost, RunStepsPath(run), req, &out) + return out, err +} + +// ListSteps is GET /v1/runs/{id}/steps. +func (c *Client) ListSteps(ctx context.Context, run string) (StepList, error) { + var out StepList + err := c.Do(ctx, http.MethodGet, RunStepsPath(run), nil, &out) + return out, err +} + +// GetStep is GET /v1/runs/{id}/steps/{n}. +func (c *Client) GetStep(ctx context.Context, run string, n int) (Step, error) { + var out Step + err := c.Do(ctx, http.MethodGet, RunStepPath(run, n), nil, &out) + return out, err +} diff --git a/src/crux/cli/cli.go b/src/crux/cli/cli.go index 19a5bb209e..61e8058737 100644 --- a/src/crux/cli/cli.go +++ b/src/crux/cli/cli.go @@ -229,6 +229,10 @@ func (e *Env) dispatch(ctx context.Context, opts globalOptions, name string, arg // gates and its API) and read through that API. fmt.Fprintln(e.Stderr, "crux: signal needs --server ADDR (or CRUX_SERVER); turn signals are kept by a crux serve") return ExitUsage + case "step": + // Agent steps run inside a session a `crux serve` holds open. + fmt.Fprintln(e.Stderr, "crux: step needs --server ADDR (or CRUX_SERVER); agent steps run on a crux serve") + return ExitUsage case "help", "-h", "--help": fs := flag.NewFlagSet("crux", flag.ContinueOnError) e.usage(fs) @@ -373,6 +377,9 @@ Work services (no runtime; hivecommons/crux#114): signal emit|list a contained run's turn signals: where the turn is and how it ended (--server only; 'crux signal help' for flags) + step run|list|show schema-checked agent steps inside a + 'runs start --profile agent' session + (--server only; 'crux step help' for flags) While 'crux run' is in the foreground it serves /crux.sock; ps, logs, kick and stop use that socket when present so they act on the live runtime. diff --git a/src/crux/cli/remote.go b/src/crux/cli/remote.go index c16a638928..5576bec557 100644 --- a/src/crux/cli/remote.go +++ b/src/crux/cli/remote.go @@ -61,6 +61,8 @@ func (e *Env) remote(ctx context.Context, opts globalOptions, name string, args return e.remoteCmdlog(ctx, c, args) case "signal": return e.remoteSignal(ctx, c, args) + case "step": + return e.remoteStep(ctx, c, args) default: fmt.Fprintf(e.Stderr, "crux: %q is not available with --server; it runs where the runtime is\n", name) return ExitUsage diff --git a/src/crux/cli/runs.go b/src/crux/cli/runs.go index 483a78f601..9f0825b7d1 100644 --- a/src/crux/cli/runs.go +++ b/src/crux/cli/runs.go @@ -24,6 +24,9 @@ start flags: --timeout DURATION wall-clock limit (server default when unset) --max-output BYTES bytes of stdout/stderr returned (server default) --profile experiment|agent + agent opens a step session: no command, answered + as accepted, steps via 'crux step run', closed by + 'runs cancel' --idempotency-key KEY return the existing run for a repeated request --detach return the accepted record instead of waiting --json print the record as JSON` @@ -78,7 +81,9 @@ func (e *Env) remoteRunsStart(ctx context.Context, c *api.Client, args []string) if code, ok := parseSub(fs, args); !ok { return code } - if fs.NArg() == 0 { + // An agent session takes no command: its steps are posted with + // `crux step run`. + if fs.NArg() == 0 && req.Profile != api.RunProfileAgent { fmt.Fprintln(e.Stderr, runsUsage) return ExitUsage } diff --git a/src/crux/cli/step.go b/src/crux/cli/step.go new file mode 100644 index 0000000000..97efe60e87 --- /dev/null +++ b/src/crux/cli/step.go @@ -0,0 +1,224 @@ +package cli + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "strconv" + "strings" + + "github.com/hivecommons/hive/crux/api" + "github.com/hivecommons/hive/crux/config" +) + +// stepUsage is the `crux step` synopsis. +const stepUsage = `usage: crux --server ADDR step run --backend B (--prompt P | --prompt-file F) --schema FILE + [--model M] [--tool T]... [--turn-limit N] [--retry N] + [--timeout-ms N] [--json] [-- CMD ARGS...] + crux --server ADDR step list [--json] + crux --server ADDR step show [--json] + +run posts one agent step to an open agent-profile session (crux runs +start --profile agent) and waits for it: the backend (claude, codex or +process) reads the prompt inside the session's sandbox and its answer is +checked against the JSON Schema in --schema. --retry N re-asks up to N +times with the violations when the answer does not match. --prompt-file +is a path relative to the session's workspace. The process backend runs +the command after -- with the prompt on stdin and its stdout as the +answer. list prints one line per step; show prints one step with its +answer.` + +// remoteStep is `crux --server … step`: the agent-step family (RFC +// crux#114 S6). Steps run on crux serve; there is no in-process form. +func (e *Env) remoteStep(ctx context.Context, c *api.Client, args []string) int { + if len(args) == 0 { + fmt.Fprintln(e.Stderr, stepUsage) + return ExitUsage + } + switch args[0] { + case "run": + return e.remoteStepRun(ctx, c, args[1:]) + case "list": + return e.remoteStepList(ctx, c, args[1:]) + case "show": + return e.remoteStepShow(ctx, c, args[1:]) + case "help", "-h", "--help": + fmt.Fprintln(e.Stderr, stepUsage) + return ExitOK + default: + fmt.Fprintf(e.Stderr, "crux: step: unknown subcommand %q\n%s\n", args[0], stepUsage) + return ExitUsage + } +} + +func (e *Env) remoteStepRun(ctx context.Context, c *api.Client, args []string) int { + fs := e.subFlags("step run") + var req api.StepRequest + var tools stringList + var schemaFile string + fs.StringVar(&req.Backend, "backend", "", "claude, codex or process") + fs.StringVar(&req.Model, "model", "", "model the backend should use") + fs.StringVar(&req.Prompt, "prompt", "", "the prompt text") + fs.StringVar(&req.PromptPath, "prompt-file", "", "prompt file, relative to the session's workspace") + fs.StringVar(&schemaFile, "schema", "", "JSON Schema file the answer must match (required)") + fs.Var(&tools, "tool", "tool the backend may use (repeatable; none means no tools)") + fs.IntVar(&req.TurnLimit, "turn-limit", 0, "most agent turns per attempt (0: backend default)") + fs.IntVar(&req.RetryOnMismatch, "retry", 0, "re-ask up to N times when the answer misses the schema") + fs.Int64Var(&req.TimeoutMs, "timeout-ms", 0, "per-attempt wall clock in ms (server default when unset)") + asJSON := fs.Bool("json", false, "print the step as JSON") + var run string + if len(args) > 0 && !strings.HasPrefix(args[0], "-") { + run, args = args[0], args[1:] + } + if code, ok := parseSub(fs, args); !ok { + return code + } + rest := fs.Args() + if run == "" && len(rest) > 0 { + run, rest = rest[0], rest[1:] + } + // The flag package only consumes "--" while it is still parsing flags; + // after a bare it is ours to drop. + if len(rest) > 0 && rest[0] == "--" { + rest = rest[1:] + } + req.Command = rest + req.Tools = tools + if run == "" || req.Backend == "" || schemaFile == "" { + fmt.Fprintln(e.Stderr, stepUsage) + return ExitUsage + } + schema, err := os.ReadFile(schemaFile) + if err != nil { + fmt.Fprintf(e.Stderr, "crux: step run: --schema: %v\n", err) + return ExitUsage + } + if !json.Valid(schema) { + fmt.Fprintf(e.Stderr, "crux: step run: --schema %s is not valid JSON\n", schemaFile) + return ExitUsage + } + req.OutputSchema = json.RawMessage(schema) + step, err := c.RunStep(ctx, run, req) + if err != nil { + return e.stepErr("run", run, err) + } + if *asJSON { + return e.printJSON(step) + } + e.printStep(step) + if step.Status != api.StepSucceeded { + return ExitError + } + return ExitOK +} + +func (e *Env) remoteStepList(ctx context.Context, c *api.Client, args []string) int { + fs := e.subFlags("step list") + asJSON := fs.Bool("json", false, "print the steps as JSON") + run, code, ok := parseWithID(fs, args) + if !ok { + return code + } + if run == "" { + fmt.Fprintln(e.Stderr, stepUsage) + return ExitUsage + } + list, err := c.ListSteps(ctx, run) + if err != nil { + return e.stepErr("list", run, err) + } + if *asJSON { + return e.printJSON(list.Steps) + } + for _, step := range list.Steps { + fmt.Fprintln(e.Stdout, stepLine(step)) + } + return ExitOK +} + +func (e *Env) remoteStepShow(ctx context.Context, c *api.Client, args []string) int { + fs := e.subFlags("step show") + asJSON := fs.Bool("json", false, "print the step as JSON") + if code, ok := parseSub(fs, args); !ok { + return code + } + if fs.NArg() != 2 { + fmt.Fprintln(e.Stderr, stepUsage) + return ExitUsage + } + run := fs.Arg(0) + n, err := strconv.Atoi(fs.Arg(1)) + if err != nil || n <= 0 { + fmt.Fprintln(e.Stderr, "crux: step show: must be a positive integer") + return ExitUsage + } + step, err := c.GetStep(ctx, run, n) + if err != nil { + return e.stepErr("show", run, err) + } + if *asJSON { + return e.printJSON(step) + } + e.printStep(step) + return ExitOK +} + +// stepLine is the one-line form list prints. +func stepLine(step api.Step) string { + line := fmt.Sprintf("%d %s %s %s", step.N, step.CreatedAt.UTC().Format(timeLayout), step.Backend, step.Status) + if step.Attempts > 1 { + line += fmt.Sprintf(" (%d attempts)", step.Attempts) + } + if step.Reason != "" { + line += " " + step.Reason + } + return line +} + +// printStep is the multi-line form run and show print. +func (e *Env) printStep(step api.Step) { + fmt.Fprintf(e.Stdout, "step: %d\n", step.N) + fmt.Fprintf(e.Stdout, "run: %s\n", step.Run) + fmt.Fprintf(e.Stdout, "status: %s\n", step.Status) + if step.Reason != "" { + fmt.Fprintf(e.Stdout, "reason: %s\n", step.Reason) + } + backend := step.Backend + if step.Model != "" { + backend += " (" + step.Model + ")" + } + fmt.Fprintf(e.Stdout, "backend: %s\n", backend) + fmt.Fprintf(e.Stdout, "attempts: %d\n", step.Attempts) + fmt.Fprintf(e.Stdout, "created: %s\n", step.CreatedAt.UTC().Format(timeLayout)) + if step.FinishedAt != nil { + fmt.Fprintf(e.Stdout, "finished: %s\n", step.FinishedAt.UTC().Format(timeLayout)) + } + if step.Usage.InputTokens > 0 || step.Usage.OutputTokens > 0 { + fmt.Fprintf(e.Stdout, "tokens: %d in, %d out\n", step.Usage.InputTokens, step.Usage.OutputTokens) + } + for _, v := range step.Violations { + fmt.Fprintf(e.Stdout, "violation: %s: %s\n", v.Path, v.Message) + } + if len(step.Answer) > 0 { + fmt.Fprintf(e.Stdout, "answer: %s\n", string(step.Answer)) + } +} + +// stepErr prints a failure; refusals the caller can fix are usage +// errors, everything else a runtime failure. +func (e *Env) stepErr(verb, run string, err error) int { + fmt.Fprintf(e.Stderr, "crux: step %s %s: %v\n", verb, run, err) + var se *api.StatusError + if errors.As(err, &se) { + switch se.Body.Code { + case api.CodeBadRequest, api.CodeForbidden, api.CodeNotSession, api.CodeStepBusy: + return ExitUsage + } + } + if api.IsUnauthorized(err) { + fmt.Fprintf(e.Stderr, "crux: the server wants a bearer token; pass --token or set %s\n", config.ServeTokenEnvVar) + } + return ExitError +} diff --git a/src/crux/cli/step_test.go b/src/crux/cli/step_test.go new file mode 100644 index 0000000000..b9d1995d37 --- /dev/null +++ b/src/crux/cli/step_test.go @@ -0,0 +1,213 @@ +package cli + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/hivecommons/hive/crux/api" + "github.com/hivecommons/hive/crux/config" +) + +// fakeStepsAPI serves run_ok's steps and records each posted request. +func fakeStepsAPI(t *testing.T) (string, func() []api.StepRequest) { + t.Helper() + at := time.Date(2026, 11, 8, 12, 0, 0, 0, time.UTC) + fin := at.Add(time.Minute) + steps := []api.Step{ + {Run: "run_ok", N: 1, Status: api.StepSucceeded, Backend: "claude", Model: "opus", CreatedAt: at, FinishedAt: &fin, Attempts: 2, + Answer: json.RawMessage(`{"verdict":"pass"}`), Usage: api.StepUsage{InputTokens: 10, OutputTokens: 4}}, + {Run: "run_ok", N: 2, Status: api.StepSchemaMismatch, Backend: "process", CreatedAt: at, FinishedAt: &fin, Attempts: 1, + Reason: "the answer did not match output_schema after 1 attempt(s)", Violations: []api.SchemaViolation{{Path: "$.verdict", Message: "required property is missing"}}}, + } + var mu sync.Mutex + var posted []api.StepRequest + fail := func(w http.ResponseWriter, status int, code, msg string) { + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(api.Error{Error: msg, Code: code}) + } + mux := http.NewServeMux() + mux.HandleFunc("GET "+api.PathRuns+"/{id}/steps", func(w http.ResponseWriter, r *http.Request) { + if r.PathValue("id") != "run_ok" { + fail(w, http.StatusNotFound, api.CodeNotFound, "run not found") + return + } + _ = json.NewEncoder(w).Encode(api.StepList{Run: "run_ok", Steps: steps}) + }) + mux.HandleFunc("GET "+api.PathRuns+"/{id}/steps/{n}", func(w http.ResponseWriter, r *http.Request) { + if r.PathValue("id") != "run_ok" || r.PathValue("n") != "1" { + fail(w, http.StatusNotFound, api.CodeNotFound, "step not found") + return + } + _ = json.NewEncoder(w).Encode(steps[0]) + }) + mux.HandleFunc("POST "+api.PathRuns+"/{id}/steps", func(w http.ResponseWriter, r *http.Request) { + var in api.StepRequest + _ = json.NewDecoder(r.Body).Decode(&in) + mu.Lock() + posted = append(posted, in) + mu.Unlock() + switch { + case r.PathValue("id") == "run_busy": + fail(w, http.StatusConflict, api.CodeStepBusy, "the session is already running a step") + case r.PathValue("id") != "run_ok": + fail(w, http.StatusConflict, api.CodeNotSession, "not an open agent step session") + case in.Backend == "process": + w.WriteHeader(http.StatusCreated) + _ = json.NewEncoder(w).Encode(steps[1]) + default: + w.WriteHeader(http.StatusCreated) + _ = json.NewEncoder(w).Encode(steps[0]) + } + }) + ts := httptest.NewServer(mux) + t.Cleanup(ts.Close) + return ts.URL, func() []api.StepRequest { + mu.Lock() + defer mu.Unlock() + return append([]api.StepRequest(nil), posted...) + } +} + +func TestStepIsRemoteOnly(t *testing.T) { + t.Setenv(config.ServerEnvVar, "") + h := newHarness(t) + if code := h.run("step", "list", "run_ok"); code != ExitUsage || !strings.Contains(h.stderr.String(), "needs --server") { + t.Fatalf("local step: %d %s", code, h.stderr) + } + if len(h.rt.Calls()) != 0 { + t.Fatalf("step touched the local runtime: %v", h.rt.Calls()) + } + h = newHarness(t) + h.run("help") + if !strings.Contains(h.stderr.String(), "step run|list|show ") { + t.Fatalf("usage does not list step:\n%s", h.stderr) + } +} + +func TestStepRunPostsTheRequest(t *testing.T) { + t.Setenv(config.ServerEnvVar, "") + t.Setenv(config.ServeTokenEnvVar, "") + server, posted := fakeStepsAPI(t) + schema := filepath.Join(t.TempDir(), "schema.json") + if err := os.WriteFile(schema, []byte(`{"type":"object"}`), 0o600); err != nil { + t.Fatal(err) + } + remote := func(args ...string) (*harness, int) { + h := newHarness(t) + return h, h.run(append([]string{"--server", server, "step"}, args...)...) + } + + h, code := remote("run", "run_ok", "--backend", "claude", "--model", "opus", "--prompt", "Judge it", "--schema", schema, + "--tool", "Read", "--tool", "Grep", "--turn-limit", "3", "--retry", "1", "--timeout-ms", "5000") + if code != ExitOK { + t.Fatalf("run: %d %s", code, h.stderr) + } + out := h.stdout.String() + for _, want := range []string{"step: 1", "status: succeeded", "backend: claude (opus)", "attempts: 2", "tokens: 10 in, 4 out", `answer: {"verdict":"pass"}`, "finished: 2026-11-08T12:01:00.000Z"} { + if !strings.Contains(out, want) { + t.Errorf("run output lacks %q:\n%s", want, out) + } + } + reqs := posted() + if len(reqs) != 1 { + t.Fatalf("posted %d", len(reqs)) + } + got := reqs[0] + if got.Backend != "claude" || got.Model != "opus" || got.Prompt != "Judge it" || string(got.OutputSchema) != `{"type":"object"}` || + strings.Join(got.Tools, ",") != "Read,Grep" || got.TurnLimit != 3 || got.RetryOnMismatch != 1 || got.TimeoutMs != 5000 || len(got.Command) != 0 { + t.Fatalf("posted = %+v", got) + } + + // The process backend takes its command after --; a step that did not + // succeed exits non-zero and prints its violations. + h, code = remote("run", "--backend", "process", "--prompt-file", "task.md", "--schema", schema, "run_ok", "--", "python3", "ask.py") + if code != ExitError || !strings.Contains(h.stdout.String(), "status: schema_mismatch") || !strings.Contains(h.stdout.String(), "violation: $.verdict: required property is missing") { + t.Fatalf("process run: %d %s %s", code, h.stdout, h.stderr) + } + if got := posted()[1]; got.PromptPath != "task.md" || strings.Join(got.Command, " ") != "python3 ask.py" { + t.Fatalf("process posted = %+v", got) + } + h, code = remote("run", "run_ok", "--backend", "claude", "--prompt", "p", "--schema", schema, "--json") + var step api.Step + if err := json.Unmarshal(h.stdout.Bytes(), &step); code != ExitOK || err != nil || step.N != 1 { + t.Fatalf("run --json: %d %v %s", code, err, h.stdout) + } + + // Refusals the caller can fix are usage errors. + if h, code = remote("run", "run_ok", "--backend", "claude", "--prompt", "p"); code != ExitUsage { + t.Fatalf("no schema: %d %s", code, h.stderr) + } + if h, code = remote("run", "run_ok", "--backend", "claude", "--prompt", "p", "--schema", filepath.Join(t.TempDir(), "nope.json")); code != ExitUsage || !strings.Contains(h.stderr.String(), "--schema") { + t.Fatalf("missing schema file: %d %s", code, h.stderr) + } + bad := filepath.Join(t.TempDir(), "bad.json") + _ = os.WriteFile(bad, []byte("{"), 0o600) + if h, code = remote("run", "run_ok", "--backend", "claude", "--prompt", "p", "--schema", bad); code != ExitUsage || !strings.Contains(h.stderr.String(), "not valid JSON") { + t.Fatalf("bad schema file: %d %s", code, h.stderr) + } + if h, code = remote("run", "run_busy", "--backend", "claude", "--prompt", "p", "--schema", schema); code != ExitUsage || !strings.Contains(h.stderr.String(), "already running") { + t.Fatalf("busy: %d %s", code, h.stderr) + } + if h, code = remote("run", "run_exp", "--backend", "claude", "--prompt", "p", "--schema", schema); code != ExitUsage || !strings.Contains(h.stderr.String(), "not an open") { + t.Fatalf("not a session: %d %s", code, h.stderr) + } +} + +func TestStepListAndShow(t *testing.T) { + t.Setenv(config.ServerEnvVar, "") + t.Setenv(config.ServeTokenEnvVar, "") + server, _ := fakeStepsAPI(t) + remote := func(args ...string) (*harness, int) { + h := newHarness(t) + return h, h.run(append([]string{"--server", server, "step"}, args...)...) + } + h, code := remote("list", "run_ok") + want := "1 2026-11-08T12:00:00.000Z claude succeeded (2 attempts)\n" + + "2 2026-11-08T12:00:00.000Z process schema_mismatch the answer did not match output_schema after 1 attempt(s)\n" + if code != ExitOK || h.stdout.String() != want { + t.Fatalf("list: %d stdout=%q stderr=%q", code, h.stdout, h.stderr) + } + h, code = remote("list", "--json", "run_ok") + var got []api.Step + if err := json.Unmarshal(h.stdout.Bytes(), &got); code != ExitOK || err != nil || len(got) != 2 { + t.Fatalf("list --json: %d %v %s", code, err, h.stdout) + } + if h, code = remote("list", "run_missing"); code != ExitError || !strings.Contains(h.stderr.String(), "run not found") { + t.Fatalf("missing run: %d %s", code, h.stderr) + } + if h, code = remote("list"); code != ExitUsage { + t.Fatalf("list without a run: %d %s", code, h.stderr) + } + + h, code = remote("show", "run_ok", "1") + if code != ExitOK || !strings.Contains(h.stdout.String(), "step: 1\nrun: run_ok\nstatus: succeeded\n") { + t.Fatalf("show: %d %q", code, h.stdout) + } + h, code = remote("show", "--json", "run_ok", "1") + var step api.Step + if err := json.Unmarshal(h.stdout.Bytes(), &step); code != ExitOK || err != nil || step.N != 1 { + t.Fatalf("show --json: %d %v %s", code, err, h.stdout) + } + if h, code = remote("show", "run_ok", "2"); code != ExitError || !strings.Contains(h.stderr.String(), "step not found") { + t.Fatalf("show missing: %d %s", code, h.stderr) + } + if h, code = remote("show", "run_ok", "x"); code != ExitUsage { + t.Fatalf("show non-number: %d %s", code, h.stderr) + } + if h, code = remote("show", "run_ok"); code != ExitUsage { + t.Fatalf("show without n: %d %s", code, h.stderr) + } + if h, code = remote("bogus"); code != ExitUsage || !strings.Contains(h.stderr.String(), "unknown subcommand") { + t.Fatalf("bogus: %d %s", code, h.stderr) + } + if h, code = remote("help"); code != ExitOK || !strings.Contains(h.stderr.String(), "usage: crux --server ADDR step run") { + t.Fatalf("help: %d %s", code, h.stderr) + } +} diff --git a/src/crux/cli/verbs_test.go b/src/crux/cli/verbs_test.go index 1d2aa805f8..313f31efe9 100644 --- a/src/crux/cli/verbs_test.go +++ b/src/crux/cli/verbs_test.go @@ -15,8 +15,8 @@ import ( // nonRuntimeVerbs are dispatch cases that do not operate on a runtime: serve // exposes one, help prints usage, cline-turn executes one pane-side worker, -// checkout, runs, gate, cmdlog and signal drive work services on a server. -var nonRuntimeVerbs = map[string]bool{"serve": true, "help": true, "-h": true, "--help": true, "cline-turn": true, "checkout": true, "runs": true, "gate": true, "cmdlog": true, "signal": true} +// checkout, runs, gate, cmdlog, signal and step drive work services on a server. +var nonRuntimeVerbs = map[string]bool{"serve": true, "help": true, "-h": true, "--help": true, "cline-turn": true, "checkout": true, "runs": true, "gate": true, "cmdlog": true, "signal": true, "step": true} func TestVerbsHaveRuntimeMethods(t *testing.T) { rt := reflect.TypeOf((*crux.Runtime)(nil)).Elem() diff --git a/src/crux/runs/service.go b/src/crux/runs/service.go index 18a7bb69a2..fb524c2496 100644 --- a/src/crux/runs/service.go +++ b/src/crux/runs/service.go @@ -43,6 +43,11 @@ var ( ErrNotFound = runstate.ErrNotFound // ErrNoSignals: the server keeps no turn signals (Service.Signals nil). ErrNoSignals = errors.New("runs: turn signals are not enabled on this server") + // ErrNotSession: a step was posted to a run that is not an open + // agent-profile session. + ErrNotSession = errors.New("runs: not an open agent step session") + // ErrStepBusy: the session is already running a step. + ErrStepBusy = errors.New("runs: the session is already running a step") ) // RequestError is a malformed request (400). @@ -71,6 +76,7 @@ type detail struct { Command []string `json:"command"` Checkout string `json:"checkout,omitempty"` Workspace string `json:"workspace,omitempty"` + EnvAllowlist []string `json:"env_allowlist,omitempty"` ExitCode *int `json:"exit_code,omitempty"` TimedOut bool `json:"timed_out,omitempty"` Stdout string `json:"stdout,omitempty"` @@ -98,6 +104,13 @@ type active struct { cancelled bool // checkout is the S1 id the run holds open, if any (see InUse). checkout string + // The rest is the agent-profile session's (S6): ctx is what its steps + // run under, ready closes once the workspace is provisioned (readyErr + // says if that failed), and busy is held while a step runs. + ctx context.Context + ready chan struct{} + readyErr error + busy chan struct{} } // Caller is who asked for a run: the authenticated principal's name and @@ -177,7 +190,7 @@ func (s *Service) Create(launchCtx context.Context, caller Caller, req api.Conta return api.ContainedRun{}, false, err } owner := caller.Name - d := detail{Profile: p.profile, Network: p.network, Backend: s.Backend.Name, Command: p.command, Checkout: p.checkout, Workspace: p.workspace} + d := detail{Profile: p.profile, Network: p.network, Backend: s.Backend.Name, Command: p.command, Checkout: p.checkout, Workspace: p.workspace, EnvAllowlist: p.allowlist} raw, err := json.Marshal(d) if err != nil { return api.ContainedRun{}, false, err @@ -199,7 +212,12 @@ func (s *Service) Create(launchCtx context.Context, caller Caller, req api.Conta s.mu.Lock() s.active[rec.ID] = a s.mu.Unlock() - go s.execute(ctx, rec.ID, p, a) + if p.profile == api.RunProfileAgent { + a.ctx, a.ready, a.busy = ctx, make(chan struct{}), make(chan struct{}, 1) + go s.session(ctx, rec.ID, p, a) + } else { + go s.execute(ctx, rec.ID, p, a) + } return toWire(rec), true, nil } @@ -319,22 +337,31 @@ func (s *Service) Cancel(ctx context.Context, id string) (api.ContainedRun, erro func (s *Service) validate(caller Caller, req api.ContainedRunRequest) (plan, error) { var p plan - if len(req.Command) == 0 || strings.TrimSpace(req.Command[0]) == "" { - return p, &RequestError{"command is required"} - } - p.command = req.Command switch p.profile = req.Profile; p.profile { case "": p.profile = api.RunProfileExperiment case api.RunProfileExperiment: case api.RunProfileAgent: - return p, &UnsupportedError{fmt.Sprintf("profile %q is not available yet (schema-checked agent steps, S6)", req.Profile)} + // A session runs no command of its own: its steps are posted to it. + if len(req.Command) > 0 { + return p, &RequestError{fmt.Sprintf("profile %q takes no command; post steps to the session instead", req.Profile)} + } default: return p, &RequestError{fmt.Sprintf("profile must be %q or %q", api.RunProfileExperiment, api.RunProfileAgent)} } + if p.profile != api.RunProfileAgent && (len(req.Command) == 0 || strings.TrimSpace(req.Command[0]) == "") { + return p, &RequestError{"command is required"} + } + p.command = req.Command switch p.network = req.Network; p.network { case "": + // An agent needs its model, so a session defaults to the restricted + // network and is refused when the backend has none; an experiment + // defaults to none. p.network = api.RunNetworkNone + if p.profile == api.RunProfileAgent { + p.network = api.RunNetworkRestricted + } case api.RunNetworkNone, api.RunNetworkRestricted: default: return p, &RequestError{fmt.Sprintf("network must be %q or %q", api.RunNetworkNone, api.RunNetworkRestricted)} @@ -344,7 +371,7 @@ func (s *Service) validate(caller Caller, req api.ContainedRunRequest) (plan, er } for _, name := range req.EnvAllowlist { if sandbox.IsCredentialName(name) { - return p, &RequestError{fmt.Sprintf("env_allowlist names credential %q; the experiment profile carries no credentials", name)} + return p, &RequestError{fmt.Sprintf("env_allowlist names credential %q; contained runs carry no repository credentials", name)} } } p.allowlist = req.EnvAllowlist diff --git a/src/crux/runs/service_test.go b/src/crux/runs/service_test.go index 6c7ed81714..73cdb96750 100644 --- a/src/crux/runs/service_test.go +++ b/src/crux/runs/service_test.go @@ -444,7 +444,8 @@ func TestValidationErrors(t *testing.T) { }{ {"no command", api.ContainedRunRequest{WorkspacePath: ws}, &RequestError{}, "command is required"}, {"blank command", api.ContainedRunRequest{WorkspacePath: ws, Command: []string{" "}}, &RequestError{}, "command is required"}, - {"agent profile", api.ContainedRunRequest{WorkspacePath: ws, Command: []string{"x"}, Profile: "agent"}, &UnsupportedError{}, "S6"}, + {"agent with command", api.ContainedRunRequest{WorkspacePath: ws, Command: []string{"x"}, Profile: "agent"}, &RequestError{}, "takes no command"}, + {"agent needs restricted", api.ContainedRunRequest{WorkspacePath: ws, Profile: "agent"}, &UnsupportedError{}, `cannot enforce network "restricted"`}, {"bad profile", api.ContainedRunRequest{WorkspacePath: ws, Command: []string{"x"}, Profile: "root"}, &RequestError{}, "profile must be"}, {"bad network", api.ContainedRunRequest{WorkspacePath: ws, Command: []string{"x"}, Network: "host"}, &RequestError{}, "network must be"}, {"restricted unsupported", api.ContainedRunRequest{WorkspacePath: ws, Command: []string{"x"}, Network: "restricted"}, &UnsupportedError{}, `cannot enforce network "restricted"`}, diff --git a/src/crux/runs/steps.go b/src/crux/runs/steps.go new file mode 100644 index 0000000000..17856464ac --- /dev/null +++ b/src/crux/runs/steps.go @@ -0,0 +1,167 @@ +package runs + +import ( + "context" + "encoding/json" + "fmt" + "time" + + "github.com/hivecommons/hive/crux/agentstep" + "github.com/hivecommons/hive/crux/api" + "github.com/hivecommons/hive/crux/runstate" +) + +// session runs an agent-profile run (S6): it provisions the workspace, +// marks the run waiting and holds it open for steps until it is cancelled. +// The steps themselves run through RunStep under this goroutine's ctx. +func (s *Service) session(ctx context.Context, id string, p plan, a *active) { + defer func() { + s.mu.Lock() + delete(s.active, id) + s.mu.Unlock() + close(a.done) + }() + finish := func(status, reason string) { + _, _ = s.Store.Update(id, func(rec *runstate.Record) error { + rec.Status, rec.Reason = status, reason + return nil + }) + s.signalDone(id, api.SignalDoneData{Status: status}, reason) + } + workspace := p.workspace + if p.worktree != nil { + prov := s.Provisioner + if prov == nil { + prov = GitProvisioner{} + } + path, err := prov.Provision(ctx, s.Store.Path(id), *p.worktree) + if err != nil { + a.readyErr = ErrNotSession + close(a.ready) + if ctx.Err() != nil { + finish(runstate.StatusCancelled, "cancelled while provisioning the worktree") + } else { + finish(runstate.StatusFailed, "provisioning the worktree: "+s.scrub(err.Error())) + } + return + } + workspace = path + } + if _, err := s.Store.Update(id, func(rec *runstate.Record) error { + var d detail + _ = json.Unmarshal(rec.Detail, &d) + d.Workspace = workspace + raw, err := json.Marshal(d) + rec.Status, rec.Detail = runstate.StatusWaiting, raw + return err + }); err != nil { + a.readyErr = ErrNotSession + close(a.ready) + finish(runstate.StatusFailed, "recording the start: "+err.Error()) + return + } + s.signal(id, api.SignalStarted, "agent step session open", nil) + close(a.ready) + <-ctx.Done() + // Wait for a step in flight to record its own cancellation first. + a.busy <- struct{}{} + steps, _ := agentstep.Store{RunDir: s.Store.Path(id)}.List() + finish(runstate.StatusCancelled, fmt.Sprintf("session closed after %d step(s)", len(steps))) +} + +// sessionFor is the open session id belongs to, or ErrNotSession. +func (s *Service) sessionFor(ctx context.Context, id string) (*active, error) { + rec, err := s.Store.Get(id) + if err != nil { + return nil, err + } + s.mu.Lock() + a := s.active[id] + s.mu.Unlock() + if rec.Kind != runstate.KindRun || a == nil || a.ready == nil { + return nil, ErrNotSession + } + select { + case <-a.ready: + case <-ctx.Done(): + return nil, ctx.Err() + } + if a.readyErr != nil || a.ctx.Err() != nil { + return nil, ErrNotSession + } + return a, nil +} + +// RunStep executes one agent step inside session id and returns its +// record. One step runs at a time: a second concurrent post is ErrStepBusy. +// The step runs under the session's context, so cancelling the run ends it. +func (s *Service) RunStep(ctx context.Context, id string, req api.StepRequest) (api.Step, error) { + a, err := s.sessionFor(ctx, id) + if err != nil { + return api.Step{}, err + } + select { + case a.busy <- struct{}{}: + default: + return api.Step{}, ErrStepBusy + } + defer func() { <-a.busy }() + if a.ctx.Err() != nil { + return api.Step{}, ErrNotSession + } + rec, err := s.Store.Get(id) + if err != nil { + return api.Step{}, err + } + var d detail + _ = json.Unmarshal(rec.Detail, &d) + setStatus := func(status string) { + _, _ = s.Store.Update(id, func(rec *runstate.Record) error { + if runstate.Terminal(rec.Status) { + return nil + } + rec.Status = status + return nil + }) + } + setStatus(runstate.StatusRunning) + defer setStatus(runstate.StatusWaiting) + host := agentstep.Host{ + Run: id, RunDir: s.Store.Path(id), Workspace: d.Workspace, + Image: s.Config.Image, Network: d.Network, + Env: s.allowlistedEnv(d.EnvAllowlist), EnvAllowlist: d.EnvAllowlist, + Launcher: s.Backend.Launcher, Teardown: s.Backend.Teardown, + Timeout: time.Duration(s.Config.DefaultTimeoutMs) * time.Millisecond, + MaxTimeout: time.Duration(s.Config.MaxTimeoutMs) * time.Millisecond, + Signal: func(kind, msg string, data json.RawMessage) { s.signal(id, kind, msg, data) }, + Scrub: s.scrub, + } + if s.CmdLog != nil { + if log, err := s.CmdLog(host.RunDir); err == nil { + host.CmdLog = log + } + } + // A caller that goes away does not end the step; the session's cancel + // does. + return agentstep.Run(a.ctx, host, req) +} + +// Steps lists run id's steps, oldest first. +func (s *Service) Steps(id string) (api.StepList, error) { + if _, err := s.Store.Get(id); err != nil { + return api.StepList{}, err + } + steps, err := agentstep.Store{RunDir: s.Store.Path(id)}.List() + if err != nil { + return api.StepList{}, err + } + return api.StepList{Run: id, Steps: steps}, nil +} + +// Step is step n of run id. +func (s *Service) Step(id string, n int) (api.Step, error) { + if _, err := s.Store.Get(id); err != nil { + return api.Step{}, err + } + return agentstep.Store{RunDir: s.Store.Path(id)}.Get(n) +} diff --git a/src/crux/runs/steps_test.go b/src/crux/runs/steps_test.go new file mode 100644 index 0000000000..ba3708c0d8 --- /dev/null +++ b/src/crux/runs/steps_test.go @@ -0,0 +1,189 @@ +package runs + +import ( + "context" + "encoding/json" + "errors" + "slices" + "strings" + "testing" + "time" + + "github.com/hivecommons/hive/crux/agentstep" + "github.com/hivecommons/hive/crux/api" + "github.com/hivecommons/hive/crux/runstate" + "github.com/hivecommons/hive/crux/sandbox" +) + +const stepSchema = `{"type":"object","required":["verdict"],"properties":{"verdict":{"enum":["pass","fail"]}}}` + +// session opens an agent-profile run (network none, which the fake +// backend supports) and waits for it to be ready. +func (f *fixture) session() api.ContainedRun { + f.t.Helper() + run, created, err := f.svc.Create(context.Background(), operator, api.ContainedRunRequest{ + WorkspacePath: f.workspace(), Profile: api.RunProfileAgent, Network: api.RunNetworkNone, EnvAllowlist: []string{"KEEP"}, + }) + if err != nil || !created || run.Status != api.RunAccepted || run.Profile != api.RunProfileAgent { + f.t.Fatalf("Create session = %+v %v %v", run, created, err) + } + return run +} + +func stepReq(prompt string) api.StepRequest { + return api.StepRequest{Backend: "process", Prompt: prompt, OutputSchema: json.RawMessage(stepSchema), Command: []string{"./ask"}} +} + +func (f *fixture) cancel(id string) api.ContainedRun { + f.t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + run, err := f.svc.Cancel(ctx, id) + if err != nil { + f.t.Fatal(err) + } + return run +} + +func TestSessionRunsStepsUntilCancelled(t *testing.T) { + f := withCmdLog(withSignals(newFixture(t, newFakeLauncher(func(_ context.Context, _ sandbox.LaunchSpec) (sandbox.Result, error) { + return sandbox.Result{Stdout: `{"verdict":"pass"}`}, nil + })))) + run := f.session() + // RunStep waits for the session to be ready, so no status polling. + step, err := f.svc.RunStep(context.Background(), run.ID, stepReq("first")) + if err != nil || step.N != 1 || step.Status != api.StepSucceeded || step.Run != run.ID { + t.Fatalf("RunStep = %+v, %v", step, err) + } + spec := f.spec() + if spec.Name != "crux-step-"+run.ID+"-1-1" || spec.Workspace != f.workspace() || spec.NetworkMode != api.RunNetworkNone || spec.Env["KEEP"] != "yes" || spec.Env["GITHUB_TOKEN"] != "" || spec.Image != "img:1" { + t.Fatalf("spec = %+v", spec) + } + if got, err := f.svc.Get(run.ID); err != nil || got.Status != api.RunWaiting || got.Workdir != f.workspace() { + t.Fatalf("between steps: %+v, %v", got, err) + } + if step, err := f.svc.RunStep(context.Background(), run.ID, stepReq("second")); err != nil || step.N != 2 { + t.Fatalf("second RunStep = %+v, %v", step, err) + } + list, err := f.svc.Steps(run.ID) + if err != nil || list.Run != run.ID || len(list.Steps) != 2 || list.Steps[1].N != 2 { + t.Fatalf("Steps = %+v, %v", list, err) + } + if got, err := f.svc.Step(run.ID, 2); err != nil || got.Status != api.StepSucceeded { + t.Fatalf("Step = %+v, %v", got, err) + } + if _, err := f.svc.Step(run.ID, 3); !errors.Is(err, agentstep.ErrNotFound) { + t.Fatalf("Step(3) = %v", err) + } + if _, err := f.svc.Steps("run_missing"); !errors.Is(err, ErrNotFound) { + t.Fatalf("Steps(missing) = %v", err) + } + page, err := f.svc.Commands(run.ID, 0, 0) + if err != nil || len(page.Commands) != 2 || page.Commands[0].Status != api.CommandCompleted { + t.Fatalf("Commands = %+v, %v", page, err) + } + + closed := f.cancel(run.ID) + if closed.Status != api.RunCancelled || !strings.Contains(closed.Reason, "2 step(s)") { + t.Fatalf("closed = %+v", closed) + } + kinds, done, _ := f.kinds(run.ID) + if !slices.Equal(kinds, []string{"started/runner", "step/runner", "step/runner", "done/runner"}) || done.Status != api.RunCancelled { + t.Fatalf("signals = %v %+v", kinds, done) + } + if _, err := f.svc.RunStep(context.Background(), run.ID, stepReq("late")); !errors.Is(err, ErrNotSession) { + t.Fatalf("step after close = %v", err) + } + if again := f.cancel(run.ID); !again.AlreadyFinished { + t.Fatalf("second cancel = %+v", again) + } +} + +func TestCancelEndsTheStepInFlightAndASecondStepIsBusy(t *testing.T) { + f := withSignals(newFixture(t, blocking())) + run := f.session() + type result struct { + step api.Step + err error + } + results := make(chan result, 1) + go func() { + step, err := f.svc.RunStep(context.Background(), run.ID, stepReq("slow")) + results <- result{step, err} + }() + f.spec() + if got, err := f.svc.Get(run.ID); err != nil || got.Status != api.RunRunning { + t.Fatalf("during a step: %+v, %v", got, err) + } + if _, err := f.svc.RunStep(context.Background(), run.ID, stepReq("second")); !errors.Is(err, ErrStepBusy) { + t.Fatalf("concurrent step = %v", err) + } + closed := f.cancel(run.ID) + res := <-results + if res.err != nil || res.step.Status != api.StepCancelled { + t.Fatalf("step in flight = %+v, %v", res.step, res.err) + } + if closed.Status != api.RunCancelled || !strings.Contains(closed.Reason, "1 step(s)") { + t.Fatalf("closed = %+v", closed) + } + select { + case name := <-f.torn: + if name != "crux-step-"+run.ID+"-1-1" { + t.Fatalf("torn %q", name) + } + default: + t.Fatal("the step's launch was not torn down") + } +} + +func TestStepsNeedAnOpenSession(t *testing.T) { + f := newFixture(t, newFakeLauncher(func(_ context.Context, _ sandbox.LaunchSpec) (sandbox.Result, error) { + return sandbox.Result{}, nil + })) + exp := f.wait(f.start(api.ContainedRunRequest{WorkspacePath: f.workspace(), Command: []string{"x"}}).ID) + if _, err := f.svc.RunStep(context.Background(), exp.ID, stepReq("p")); !errors.Is(err, ErrNotSession) { + t.Fatalf("step on an experiment = %v", err) + } + if _, err := f.svc.RunStep(context.Background(), "run_missing", stepReq("p")); !errors.Is(err, ErrNotFound) { + t.Fatalf("step on a missing run = %v", err) + } + if list, err := f.svc.Steps(exp.ID); err != nil || len(list.Steps) != 0 { + t.Fatalf("Steps of an experiment = %+v, %v", list, err) + } + // A malformed step is refused without a record; an unsupported one is + // recorded and refused. + run := f.session() + defer f.cancel(run.ID) + var reqErr *agentstep.RequestError + if _, err := f.svc.RunStep(context.Background(), run.ID, api.StepRequest{Backend: "process"}); !errors.As(err, &reqErr) { + t.Fatalf("malformed step = %v", err) + } + req := stepReq("p") + req.Tools = []string{"Bash"} + var unsupported *agentstep.UnsupportedError + step, err := f.svc.RunStep(context.Background(), run.ID, req) + if !errors.As(err, &unsupported) || step.N != 1 || step.Status != api.StepUnsupported { + t.Fatalf("unsupported step = %+v, %v", step, err) + } + if got, err := f.svc.Get(run.ID); err != nil || got.Status != api.RunWaiting { + t.Fatalf("after a refused step: %+v, %v", got, err) + } +} + +func TestReconcileClosesAWaitingSession(t *testing.T) { + f := newFixture(t, blocking()) + rec, _, err := f.svc.Store.Create(runstate.Record{Owner: "operator"}) + if err != nil { + t.Fatal(err) + } + if _, err := f.svc.Store.Update(rec.ID, func(r *runstate.Record) error { r.Status = runstate.StatusWaiting; return nil }); err != nil { + t.Fatal(err) + } + changed, err := f.svc.Reconcile() + if err != nil || len(changed) != 1 || changed[0].Status != runstate.StatusUnfinished { + t.Fatalf("Reconcile = %+v, %v", changed, err) + } + if _, err := f.svc.RunStep(context.Background(), rec.ID, stepReq("p")); !errors.Is(err, ErrNotSession) { + t.Fatalf("step on a reconciled run = %v", err) + } +} diff --git a/src/crux/runstate/runstate.go b/src/crux/runstate/runstate.go index 97771f2e95..3115d41539 100644 --- a/src/crux/runstate/runstate.go +++ b/src/crux/runstate/runstate.go @@ -46,7 +46,8 @@ const ( RecordName = "record.json" // IndexName is the append-only status history, one IndexEntry per line. IndexName = "index.jsonl" - // KindRun is a contained run (S2); S6 steps add their own kind later. + // KindRun is a contained run (S2) or, with profile agent, a step + // session (S6); steps are kept under the run's directory by agentstep. KindRun = "run" idRandomBytes = 6 @@ -263,8 +264,8 @@ func (s *Store) List() ([]Record, error) { return out, nil } -// Reconcile moves every accepted or running record to StatusUnfinished with -// reason and returns the records it changed. It runs at `crux serve` start: +// Reconcile moves every accepted, running or waiting record to +// StatusUnfinished with reason and returns the records it changed. It runs at `crux serve` start: // the process that owned those items is gone, so their exit is not knowable // and they are never restarted silently. func (s *Store) Reconcile(reason string) ([]Record, error) { @@ -274,7 +275,7 @@ func (s *Store) Reconcile(reason string) ([]Record, error) { } var changed []Record for _, r := range recs { - if r.Status != StatusAccepted && r.Status != StatusRunning { + if r.Status != StatusAccepted && r.Status != StatusRunning && r.Status != StatusWaiting { continue } rec, err := s.Update(r.ID, func(rec *Record) error { diff --git a/src/crux/serve/runs.go b/src/crux/serve/runs.go index a0278077ba..0d875ef840 100644 --- a/src/crux/serve/runs.go +++ b/src/crux/serve/runs.go @@ -87,7 +87,10 @@ func (s *Server) handleRunsCreate(w http.ResponseWriter, r *http.Request) { s.writeJSON(w, http.StatusOK, run) return } - if req.Detach { + // A step session (profile agent) stays open until it is cancelled, so + // it is always answered as accepted; its steps are what the caller waits + // on. + if req.Detach || run.Profile == api.RunProfileAgent { s.writeJSON(w, http.StatusAccepted, run) return } diff --git a/src/crux/serve/runs_test.go b/src/crux/serve/runs_test.go index bedfb47ed2..afecd70745 100644 --- a/src/crux/serve/runs_test.go +++ b/src/crux/serve/runs_test.go @@ -122,7 +122,8 @@ func TestRunsErrorCodes(t *testing.T) { }) f.expectCode(http.MethodPost, api.PathRuns, "{not json", http.StatusBadRequest, api.CodeBadRequest) f.expectCode(http.MethodPost, api.PathRuns, api.ContainedRunRequest{WorkspacePath: ws}, http.StatusBadRequest, api.CodeBadRequest) - f.expectCode(http.MethodPost, api.PathRuns, api.ContainedRunRequest{WorkspacePath: ws, Command: []string{"x"}, Profile: api.RunProfileAgent}, http.StatusNotImplemented, api.CodeUnsupported) + f.expectCode(http.MethodPost, api.PathRuns, api.ContainedRunRequest{WorkspacePath: ws, Command: []string{"x"}, Profile: api.RunProfileAgent}, http.StatusBadRequest, api.CodeBadRequest) + f.expectCode(http.MethodPost, api.PathRuns, api.ContainedRunRequest{WorkspacePath: ws, Profile: api.RunProfileAgent}, http.StatusNotImplemented, api.CodeUnsupported) f.expectCode(http.MethodPost, api.PathRuns, api.ContainedRunRequest{WorkspacePath: ws, Command: []string{"x"}, Network: api.RunNetworkRestricted}, http.StatusNotImplemented, api.CodeUnsupported) f.expectCode(http.MethodPost, api.PathRuns, api.ContainedRunRequest{Checkout: "chk_1", Command: []string{"x"}}, http.StatusNotImplemented, api.CodeUnsupported) f.expectCode(http.MethodPost, api.PathRuns, api.ContainedRunRequest{WorkspacePath: "/elsewhere", Command: []string{"x"}}, http.StatusForbidden, api.CodeForbidden) diff --git a/src/crux/serve/serve.go b/src/crux/serve/serve.go index 1b1b9707e2..c740cc4398 100644 --- a/src/crux/serve/serve.go +++ b/src/crux/serve/serve.go @@ -183,6 +183,7 @@ func (s *Server) Handler() http.Handler { s.registerFamilies(mux) s.registerCheckouts(mux) s.registerRuns(mux) + s.registerSteps(mux) s.registerGates(mux) s.registerSignals(mux) return s.laneHandler(s.authenticate(instrument(mux))) diff --git a/src/crux/serve/steps.go b/src/crux/serve/steps.go new file mode 100644 index 0000000000..41bccf2b97 --- /dev/null +++ b/src/crux/serve/steps.go @@ -0,0 +1,102 @@ +package serve + +import ( + "context" + "errors" + "net/http" + "strconv" + + "github.com/hivecommons/hive/crux/agentstep" + "github.com/hivecommons/hive/crux/api" + "github.com/hivecommons/hive/crux/runs" +) + +// Agent step routes (S6). A step is posted to an open agent-profile run +// and answers when the step has finished; the step runs under the +// session, not the request, so a caller that goes away leaves it in +// flight and reads it back with GET. +func (s *Server) registerSteps(mux *http.ServeMux) { + mux.HandleFunc("POST "+api.PathRuns+"/{id}/steps", s.handleStepsRun) + mux.HandleFunc("GET "+api.PathRuns+"/{id}/steps", s.handleStepsList) + mux.HandleFunc("GET "+api.PathRuns+"/{id}/steps/{n}", s.handleStepsGet) +} + +func (s *Server) handleStepsRun(w http.ResponseWriter, r *http.Request) { + if !s.runsReady(w) { + return + } + run, ok := s.ownedRun(w, r) + if !ok { + return + } + var req api.StepRequest + if !s.readJSON(w, r, &req) { + return + } + step, err := s.Runs.RunStep(r.Context(), run.ID, req) + if err != nil { + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return + } + s.writeStepsError(w, err) + return + } + s.writeJSON(w, http.StatusCreated, step) +} + +func (s *Server) handleStepsList(w http.ResponseWriter, r *http.Request) { + if !s.runsReady(w) { + return + } + run, ok := s.ownedRun(w, r) + if !ok { + return + } + list, err := s.Runs.Steps(run.ID) + if err != nil { + s.writeStepsError(w, err) + return + } + s.writeJSON(w, http.StatusOK, list) +} + +func (s *Server) handleStepsGet(w http.ResponseWriter, r *http.Request) { + if !s.runsReady(w) { + return + } + run, ok := s.ownedRun(w, r) + if !ok { + return + } + n, err := strconv.Atoi(r.PathValue("n")) + if err != nil || n <= 0 { + s.writeError(w, http.StatusBadRequest, api.CodeBadRequest, "step number must be a positive integer") + return + } + step, err := s.Runs.Step(run.ID, n) + if err != nil { + s.writeStepsError(w, err) + return + } + s.writeJSON(w, http.StatusOK, step) +} + +func (s *Server) writeStepsError(w http.ResponseWriter, err error) { + var reqErr *agentstep.RequestError + var unsupported *agentstep.UnsupportedError + switch { + case errors.Is(err, agentstep.ErrNotFound): + s.writeError(w, http.StatusNotFound, api.CodeNotFound, "step not found") + case errors.Is(err, runs.ErrNotSession): + s.writeError(w, http.StatusConflict, api.CodeNotSession, err.Error()) + case errors.Is(err, runs.ErrStepBusy): + s.writeError(w, http.StatusConflict, api.CodeStepBusy, err.Error()) + case errors.As(err, &reqErr): + s.writeError(w, http.StatusBadRequest, api.CodeBadRequest, err.Error()) + case errors.As(err, &unsupported): + // The refusal is also on the record: GET steps/{n} shows it. + s.writeError(w, http.StatusNotImplemented, api.CodeUnsupported, err.Error()) + default: + s.writeRunsError(w, err) + } +} diff --git a/src/crux/serve/steps_test.go b/src/crux/serve/steps_test.go new file mode 100644 index 0000000000..11155e6a63 --- /dev/null +++ b/src/crux/serve/steps_test.go @@ -0,0 +1,134 @@ +package serve + +import ( + "context" + "encoding/json" + "net/http" + "os" + "path/filepath" + "testing" + + "github.com/hivecommons/hive/crux/api" + "github.com/hivecommons/hive/crux/runs" + "github.com/hivecommons/hive/crux/runstate" + "github.com/hivecommons/hive/crux/sandbox" +) + +// withHeldRuns is withRuns with the launched channel exposed: each launch +// reports on it and then holds until cleanup or its context ends. +func (f *gateFixture) withHeldRuns() (string, <-chan struct{}) { + f.t.Helper() + store, err := runstate.Open(f.t.TempDir()) + if err != nil { + f.t.Fatal(err) + } + root := f.t.TempDir() + ws := filepath.Join(root, "proj") + if err := os.MkdirAll(ws, 0o755); err != nil { + f.t.Fatal(err) + } + launched, release := make(chan struct{}, 8), make(chan struct{}) + f.t.Cleanup(func() { close(release) }) + backend := &runs.Backend{Name: "fake", Launcher: holdingLauncher(launched, release), Capabilities: runs.Capabilities{NetworkNone: true}} + f.s.Runs = runs.New(store, backend, runs.Config{Image: "img", WorkspaceRoots: []string{root}}) + f.h = f.s.Handler() + return ws, launched +} + +const stepSchema = `{"type":"object","required":["verdict"]}` + +func stepReq(prompt string) api.StepRequest { + return api.StepRequest{Backend: "process", Prompt: prompt, OutputSchema: json.RawMessage(stepSchema), Command: []string{"./ask"}} +} + +func TestStepsRoutesAnswerNotSupportedWithoutAService(t *testing.T) { + f := newFamServer(t) + f.expectCode(http.MethodPost, api.RunStepsPath("run_x"), stepReq("p"), http.StatusNotImplemented, api.CodeNotSupported) + f.expectCode(http.MethodGet, api.RunStepsPath("run_x"), nil, http.StatusNotImplemented, api.CodeNotSupported) + f.expectCode(http.MethodGet, api.RunStepPath("run_x", 1), nil, http.StatusNotImplemented, api.CodeNotSupported) +} + +func TestSessionStepsOverHTTP(t *testing.T) { + f, ws := newRunsServer(t, func(_ context.Context, _ sandbox.LaunchSpec) (sandbox.Result, error) { + return sandbox.Result{Stdout: `{"verdict":"pass"}`}, nil + }) + // A session is answered as accepted whether or not detach was asked. + var run api.ContainedRun + f.expect(http.MethodPost, api.PathRuns, api.ContainedRunRequest{WorkspacePath: ws, Profile: api.RunProfileAgent, Network: api.RunNetworkNone}, http.StatusAccepted, &run) + if run.Status != api.RunAccepted || run.Profile != api.RunProfileAgent { + t.Fatalf("session = %+v", run) + } + var step api.Step + f.expect(http.MethodPost, api.RunStepsPath(run.ID), stepReq("first"), http.StatusCreated, &step) + if step.N != 1 || step.Status != api.StepSucceeded || string(step.Answer) != `{"verdict":"pass"}` { + t.Fatalf("step = %+v", step) + } + f.expectCode(http.MethodPost, api.RunStepsPath(run.ID), api.StepRequest{Backend: "process"}, http.StatusBadRequest, api.CodeBadRequest) + f.expectCode(http.MethodPost, api.RunStepsPath(run.ID), "{", http.StatusBadRequest, api.CodeBadRequest) + unsupported := stepReq("p") + unsupported.Tools = []string{"Bash"} + f.expectCode(http.MethodPost, api.RunStepsPath(run.ID), unsupported, http.StatusNotImplemented, api.CodeUnsupported) + + var list api.StepList + f.expect(http.MethodGet, api.RunStepsPath(run.ID), nil, http.StatusOK, &list) + if list.Run != run.ID || len(list.Steps) != 2 || list.Steps[1].Status != api.StepUnsupported { + t.Fatalf("list = %+v", list) + } + var got api.Step + f.expect(http.MethodGet, api.RunStepPath(run.ID, 1), nil, http.StatusOK, &got) + if got.N != 1 || got.Status != api.StepSucceeded { + t.Fatalf("get = %+v", got) + } + f.expectCode(http.MethodGet, api.RunStepPath(run.ID, 9), nil, http.StatusNotFound, api.CodeNotFound) + f.expectCode(http.MethodGet, api.PathRuns+"/"+run.ID+"/steps/zero", nil, http.StatusBadRequest, api.CodeBadRequest) + f.expectCode(http.MethodGet, api.RunStepsPath("run_000000000000"), nil, http.StatusNotFound, api.CodeNotFound) + + var closed api.ContainedRun + f.expect(http.MethodPost, api.RunCancelPath(run.ID), nil, http.StatusOK, &closed) + if closed.Status != api.RunCancelled { + t.Fatalf("closed = %+v", closed) + } + f.expectCode(http.MethodPost, api.RunStepsPath(run.ID), stepReq("late"), http.StatusConflict, api.CodeNotSession) + + // A finished experiment is not a session either. + var exp api.ContainedRun + f.expect(http.MethodPost, api.PathRuns, api.ContainedRunRequest{WorkspacePath: ws, Command: []string{"x"}}, http.StatusOK, &exp) + f.expectCode(http.MethodPost, api.RunStepsPath(exp.ID), stepReq("p"), http.StatusConflict, api.CodeNotSession) +} + +func TestConcurrentStepIsBusyAndStepsAreOwned(t *testing.T) { + f := newGateFixture(t) + ws, launched := f.withHeldRuns() + var run api.ContainedRun + if code, e := f.call("atlas", http.MethodPost, api.PathRuns, api.ContainedRunRequest{WorkspacePath: ws, Profile: api.RunProfileAgent, Network: api.RunNetworkNone}, &run); code != http.StatusAccepted { + t.Fatalf("session: %d %+v", code, e) + } + if code, e := f.call("beacon", http.MethodGet, api.RunStepsPath(run.ID), nil, nil); code != http.StatusForbidden || e.Code != api.CodeForbidden { + t.Fatalf("beacon on atlas's session: %d %+v", code, e) + } + if code, e := f.call("beacon", http.MethodPost, api.RunStepsPath(run.ID), stepReq("p"), nil); code != http.StatusForbidden || e.Code != api.CodeForbidden { + t.Fatalf("beacon posting to atlas's session: %d %+v", code, e) + } + // The holding launcher keeps the first step in flight until cleanup. + done := make(chan struct{}) + go func() { + defer close(done) + f.call("atlas", http.MethodPost, api.RunStepsPath(run.ID), stepReq("slow"), nil) + }() + <-launched + if code, e := f.call("atlas", http.MethodPost, api.RunStepsPath(run.ID), stepReq("second"), nil); code != http.StatusConflict || e.Code != api.CodeStepBusy { + t.Fatalf("concurrent step: %d %+v", code, e) + } + var list api.StepList + if code, _ := f.call("operator", http.MethodGet, api.RunStepsPath(run.ID), nil, &list); code != http.StatusOK || len(list.Steps) != 1 || list.Steps[0].Status != api.StepRunning { + t.Fatalf("list during a step: %d %+v", code, list) + } + if code, _ := f.call("atlas", http.MethodPost, api.RunCancelPath(run.ID), nil, nil); code != http.StatusOK { + t.Fatalf("cancel: %d", code) + } + <-done + var step api.Step + if code, _ := f.call("atlas", http.MethodGet, api.RunStepPath(run.ID, 1), nil, &step); code != http.StatusOK || step.Status != api.StepCancelled { + t.Fatalf("cancelled step: %d %+v", code, step) + } +} diff --git a/src/crux/signals/signals.go b/src/crux/signals/signals.go index de0242802b..762f3c69ae 100644 --- a/src/crux/signals/signals.go +++ b/src/crux/signals/signals.go @@ -65,15 +65,16 @@ var ( // kinds is the closed kind set. var kinds = map[string]bool{ api.SignalStarted: true, api.SignalThinking: true, api.SignalTool: true, api.SignalBlocked: true, - api.SignalNeedsInput: true, api.SignalInput: true, api.SignalDone: true, + api.SignalNeedsInput: true, api.SignalInput: true, api.SignalDone: true, api.SignalStep: true, } // bySource is what each source may append. The runner owns the run's -// boundaries; a gate reports a person being asked, answering or not -// answering in time; the caller reports everything in between but never a -// boundary, so a done signal always means the runner saw the run end. +// boundaries and the steps an agent session ran (S6); a gate reports a +// person being asked, answering or not answering in time; the caller +// reports everything in between but never a boundary, so a done signal +// always means the runner saw the run end. var bySource = map[string]map[string]bool{ - api.SignalSourceRunner: {api.SignalStarted: true, api.SignalDone: true}, + api.SignalSourceRunner: {api.SignalStarted: true, api.SignalDone: true, api.SignalStep: true}, api.SignalSourceGate: {api.SignalNeedsInput: true, api.SignalInput: true, api.SignalBlocked: true}, api.SignalSourceCaller: {api.SignalThinking: true, api.SignalTool: true, api.SignalBlocked: true, api.SignalNeedsInput: true, api.SignalInput: true}, } diff --git a/src/crux/signals/signals_test.go b/src/crux/signals/signals_test.go index 546a1e0ac7..004bf951a8 100644 --- a/src/crux/signals/signals_test.go +++ b/src/crux/signals/signals_test.go @@ -125,7 +125,10 @@ func TestSourcesAppendOnlyTheirKinds(t *testing.T) { }{ {api.SignalSourceRunner, api.SignalStarted, true}, {api.SignalSourceRunner, api.SignalDone, true}, + {api.SignalSourceRunner, api.SignalStep, true}, {api.SignalSourceRunner, api.SignalThinking, false}, + {api.SignalSourceGate, api.SignalStep, false}, + {api.SignalSourceCaller, api.SignalStep, false}, {api.SignalSourceGate, api.SignalNeedsInput, true}, {api.SignalSourceGate, api.SignalInput, true}, {api.SignalSourceGate, api.SignalBlocked, true}, diff --git a/src/docs/design/crux-work-services.md b/src/docs/design/crux-work-services.md index 366a073f0e..8e9f6734c8 100644 --- a/src/docs/design/crux-work-services.md +++ b/src/docs/design/crux-work-services.md @@ -1,7 +1,7 @@ # Crux work services: S1 pinned checkout (hivecommons/crux#114) Status: **S0 caller principals and S1 pinned checkout shipped in hive -`src/crux` (v5).** S2 contained run and S7 minimum shipped in #17014; S3 human gate is [below](#s3-human-gate); S4 command log is [below](#s4-command-log); S5 turn signals are [below](#s5-turn-signals); everything else in the RFC is unscheduled. This records +`src/crux` (v5).** S2 contained run and S7 minimum shipped in #17014; S3 human gate is [below](#s3-human-gate); S4 command log is [below](#s4-command-log); S5 turn signals are [below](#s5-turn-signals); S6 agent step is [below](#s6-agent-step); everything else in the RFC is unscheduled. This records S1 *as implemented*, including where it is weaker than the RFC text. The RFC asks for "a working copy of this repo at this commit, owned by @@ -363,12 +363,12 @@ signals.NewBook(signals.Policy{})` and `Server.KickOutcomes` on A `Signal` is `{seq, kind, source, at, message?, data?}`. `kind` is the closed set `started | thinking | tool | blocked | needs-input | input | -done`; `source` is `runner`, `gate` or `caller`, and each source appends -only its own kinds: +step | done`; `source` is `runner`, `gate` or `caller`, and each source +appends only its own kinds: | Source | Kinds | When | |---|---|---| -| `runner` | `started`, `done` | `started` just before the sandbox launcher runs the command; `done` on every terminal path — completed, failed, timed out, cancelled, a provisioning failure (no `started`), and the run `crux serve` marks `unfinished` at start because it was in flight at the last shutdown. `done.data` is `{"status", "exit_code"?, "timed_out"?}`, its message the run's reason. | +| `runner` | `started`, `step`, `done` | `started` just before the sandbox launcher runs the command (for an agent session, once its workspace is ready); `step` once per finished agent step (S6, `data` `{"step", "status", "attempts"}`, message the step's reason); `done` on every terminal path — completed, failed, timed out, cancelled, a provisioning failure (no `started`), and the run `crux serve` marks `unfinished` at start because it was in flight at the last shutdown. `done.data` is `{"status", "exit_code"?, "timed_out"?}`, its message the run's reason. | | `gate` | `needs-input`, `input`, `blocked` | a gate opened with `run`: opening is `needs-input` (message the subject), answering `input` (`data` `{"gate", "answer", "answered_by"}`), expiry `blocked`. `data.gate` names the gate; its note and context never ride along. | | `caller` | `thinking`, `tool`, `blocked`, `needs-input`, `input` | `POST /v1/runs/{id}/signals` by the run's owner or the operator. | @@ -436,6 +436,145 @@ from inside the sandbox other than through the owner's API; no per-agent signal log beyond the last kick outcome; no retention policy — a run's signals live and die with the run dir. +## S6. Agent step + +One step = one agent backend run inside a sandbox, in a prepared +checkout, with a prompt in and a schema-checked JSON answer out. Package +`src/crux/agentstep` (`Run`, `Host`, `Backend`, `Backends`, `Store`, +`ParseSchema`, `CorrectivePrompt`), API types in `src/crux/api/steps.go`, +routes in `src/crux/serve/steps.go`, the session in +`src/crux/runs/service.go`, CLI in `src/crux/cli/step.go`. No new +configuration: a step runs on the S2 backend and image with the S2 +timeouts (`default_timeout_ms` per attempt, `max_timeout_ms` the cap). + +**Session.** A step needs a sandbox that already holds the checkout, so +`POST /v1/runs` with `"profile": "agent"` opens a *step session* instead +of running a command: `command` must be empty (400), the source is any +S2 source (checkout, worktree or workspace_path), `network` defaults to +`restricted` because the agent must reach its model (a backend without +that capability answers 501 `unsupported`; a caller may ask `none` for a +process backend that needs nothing), and `env_allowlist` still refuses +credential names — a session carries no repository-write authority, as +the RFC asks. The response is always 202 with the `accepted` record, +`detach` or not: the session has no end of its own. Its runner +provisions the workspace, moves the record to `waiting`, signals +`started` and holds until `POST /v1/runs/{id}/cancel` closes it +(`cancelled`, reason `session closed after N step(s)`, `done` signal). +While a step runs the record is `running`; between steps `waiting`. +`Reconcile` at start marks a `waiting` session `unfinished` like any run +left in flight. One step at a time: a second concurrent post is 409 +`step_busy`; a step on a run that is not an open session (an experiment, +a closed or reconciled session, a session whose provisioning failed) is +409 `not_session`. A step posted while the session is still provisioning +waits for it. + +| Route | Body | Response | +|---|---|---| +| `POST /v1/runs/{id}/steps` | `StepRequest` | `201 Step` once the step has finished, whatever its status | +| `GET /v1/runs/{id}/steps` | | `200 {"run", "steps":[Step…]}` oldest first; a step in flight shows as `running` | +| `GET /v1/runs/{id}/steps/{n}` | | `200 Step` | + +`StepRequest` is `{backend, prompt | prompt_path, output_schema, model?, +tools?, turn_limit?, timeout_ms?, retry_on_mismatch?, command?}`. +`prompt` is inline (≤ 1 MiB); `prompt_path` is relative to the session +workspace and must stay inside it. `output_schema` is a JSON Schema the +answer must match; `tools` is the allowlist of tools the agent may use +(empty: the backend's default, which for `claude` is every tool); +`turn_limit` bounds agent turns per attempt; `retry_on_mismatch` (0–5) +re-asks when the answer misses the schema; `command` is the process +backend's. A `Step` is `{run, n, status, reason?, backend, model?, +created_at, finished_at?, attempts, answer?, usage{input_tokens, +output_tokens}, transcript_path?, violations?}`. `status` is the RFC's +closed set: `succeeded | schema_mismatch | timeout | backend_error | +tool_denied | turn_limit_exceeded | cancelled | unsupported`, plus +`running` while in flight. `answer` is present only on `succeeded`. + +**Execution.** The service writes the prompt to +`/.crux/steps//prompt-.md` (the same place the +agent sandbox keeps its kick prompt), builds the backend's command line +with the prompt as stdin, records it in the S4 command log (`capture: +runner`, status from the exit), launches it through the S2 launcher as +`crux-step---` with the session's network and filtered +environment, and reads the answer back. The first attempt sends the +caller's prompt verbatim; every retry appends `CorrectivePrompt`: the +schema, the violations by JSON path, and how many attempts remain. A +non-JSON or missing answer counts as a mismatch (`$: the backend returned +no JSON value`) and is retried the same way. Timeout and cancel tear the +launch down by name, as S2 does. Every finished step writes its record, +appends one `step` signal to the run and leaves a transcript at +`/runs//steps//transcript.jsonl` (one line per attempt: +command, exit, scrubbed output, status, violations, usage). + +**Schema.** `agentstep.ParseSchema` enforces the subset the service can +check without a schema library: `type` (incl. unions), `enum`, `const`, +`properties`, `required`, `additionalProperties` (bool or schema), +`items`, `minItems`/`maxItems`, `minLength`/`maxLength`, `pattern`, +`minimum`/`maximum`/`exclusiveMinimum`/`exclusiveMaximum`, `multipleOf`, +boolean schemas; `title`, `description`, `default`, `examples`, `$schema`, +`$id` and `$comment` are ignored. Any other keyword (`$ref`, `oneOf`, +`format`, …) is refused with 400 rather than silently unenforced, so a +`succeeded` step means the answer was fully checked. + +**Backends.** `agentstep.Backend` is `{Name; Command(Spec) ([]string, +error); Parse(Spec, sandbox.Result, readFile) Outcome}`: it turns the +request into a command line the sandbox runs and reads the attempt's +result back. `Command` returning `*UnsupportedError` for a request the +backend cannot honour (tools it cannot restrict, a turn limit it cannot +enforce) records a step with status `unsupported` and answers 501 +`unsupported`: the refusal is on the record, as the RFC asks. The RFC +decision's two v1 backends and one test backend ship: + +| Backend | Command | `Outcome` from | +|---|---|---| +| `claude` | `claude -p --output-format json [--model M] (--allowedTools a,b \| --dangerously-skip-permissions) [--max-turns N] < prompt` | the result object: `result` holds the answer (JSON extracted from prose or a fence), `usage` the tokens, `num_turns` the turns; `permission_denials` → `tool_denied`; subtype `error_max_turns` → `turn_limit_exceeded`; any other error → `backend_error` | +| `codex` | `codex exec --json --skip-git-repo-check --dangerously-bypass-approvals-and-sandbox [--model M] --output-last-message .crux/steps//answer-.txt - < prompt` | the last-message file, else the last `agent_message` item in the JSONL; `usage` events sum the tokens. `tools` is refused as unsupported (codex has no allowlist), and so is `turn_limit` above 1 (codex exec is one turn) | +| `process` | `sh -lc ' < prompt'` | stdout as the answer, non-zero exit `backend_error`. Takes `command`; `tools`, `turn_limit` and `model` are refused as unsupported (the command offers whatever it offers). This is the test and escape-hatch backend: any CLI that reads a prompt on stdin | + +The claude and codex command lines and result shapes are the adapter +contract this slice fixes; they are exercised against recorded output in +`backend_test.go`, not a live CLI. The RFC's other backends (`gemini`, +`agy`, `cline`) are not registered: adding one is a `Backend` +implementation and a line in `Backends`, with no change to the service, +the routes or the CLI. + +| Condition | Status | `code` | +|---|---|---| +| unknown backend, both or neither of `prompt`/`prompt_path`, path outside the workspace, bad or unenforceable `output_schema`, `retry_on_mismatch` outside 0–5, `timeout_ms` above the cap, `command` with a non-process backend, process without `command`; agent profile with `command` | 400 | `bad_request` | +| another principal's run | 403 | `forbidden` | +| unknown run or step | 404 | `not_found` | +| the run is not an open session | 409 | `not_session` | +| the session is running a step | 409 | `step_busy` | +| the backend cannot honour `tools`, `turn_limit` or `model`; a session on a backend without `restricted` | 501 | `unsupported` | +| no runs service | 501 | `not_supported` | + +``` +crux --server ADDR runs start --profile agent [--network none] --checkout ID +crux --server ADDR step run --backend B (--prompt P | --prompt-file F) --schema FILE + [--model M] [--tool T]... [--turn-limit N] [--retry N] + [--timeout-ms N] [--json] [-- CMD ARGS...] +crux --server ADDR step list [--json] +crux --server ADDR step show [--json] +crux --server ADDR runs cancel +``` + +`step run` exits 0 only on `succeeded`; any other status prints the +record (with violations) and exits 1, so a driver script can branch on +it. + +**Storage.** `/runs//steps//record.json` (the `Step`, +tmp+rename) and `transcript.jsonl`, 0600; steps are numbered from 1 in +creation order under a process-wide lock. Prompts and backend answer +files live in the workspace under `.crux/steps//` and go with it. + +**Deliberately not in S6.** No `gemini`, `agy` or `cline` backend yet; +no streaming of a step's progress beyond the S5 `step` signal and the S4 +log; no step-level idempotency key (post once, read back with `GET`); +no fan-out of one prompt to several backends; no cost accounting beyond +the usage the backend reports; no persistence of the agent's own session +between steps — each step is a fresh backend invocation in the same +checkout, and whatever the step wrote to the workspace is the state the +next one sees. + ## Deliberately not in S1 - TTL sweeper (recorded, not enforced). @@ -482,3 +621,8 @@ for runs are deferred and not scheduled. | `src/crux/serve/gates.go` | `run` on gate open (ownership check) and the gate's turn signals | | `src/crux/serve/signals.go` | `GET /v1/agents/{name}/kick-outcome`; the kick-outcome observer that saves before it publishes | | `src/crux/cli/signal.go` | `crux signal emit|list`, remote only | +| `src/crux/agentstep/` | S6: `Run`, `Host`, `Backend`, `Backends`, `Lookup`, `Spec`, `Outcome`, `Store` (`Create`, `Write`, `Get`, `List`), `ParseSchema`, `Schema.Validate`, `CorrectivePrompt`, `RequestError`, `UnsupportedError`, sentinels (`ErrSchema`, `ErrNotFound`, `ErrCorrupt`, `ErrNoAnswer`) | +| `src/crux/api/steps.go` | `RunStepsPath`, `RunStepPath`, `Step*` status and backend constants, `CodeStepBusy`, `CodeNotSession`, `SignalStep`, `SignalStepData`, `StepRequest`, `Step`, `StepUsage`, `SchemaViolation`, `StepList`, `Client.{RunStep,ListSteps,GetStep}` | +| `src/crux/runs/service.go` | S6 session: `profile: agent` validation, `session` runner, `RunStep`, `Steps`, `Step`, `ErrNotSession`, `ErrStepBusy` | +| `src/crux/serve/steps.go` | `POST`/`GET /v1/runs/{id}/steps`, `GET /v1/runs/{id}/steps/{n}`, ownership check, error mapping | +| `src/crux/cli/step.go` | `crux step run|list|show`, remote only |