From 03094cecbffacb4fda65b69c50ce246cd580f144 Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Sat, 7 Nov 2026 12:50:21 -0500 Subject: [PATCH 1/7] =?UTF-8?q?=E2=9C=A8=20crux:=20contained=20run=20servi?= =?UTF-8?q?ce=20with=20saved=20run=20state=20(S2=20+=20S7=20minimum)=20(#1?= =?UTF-8?q?7014)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ✨ crux: contained run service with saved run state (S2 + S7 minimum) Refs hivecommons/crux#114 Signed-off-by: Andrew Anderson * πŸ“ changelog: crux contained run service Signed-off-by: Andrew Anderson * 🌱 crux/runstate: cover write obstacles, unreadable records and list ordering Signed-off-by: Andrew Anderson --------- Signed-off-by: Andrew Anderson Co-authored-by: hivecommons-hive[bot] <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- .../added-crux-contained-run-service.md | 1 + src/crux/api/runs.go | 122 +++++ src/crux/api/runs_client.go | 27 + src/crux/api/runs_test.go | 12 + src/crux/cli/cli.go | 7 + src/crux/cli/remote.go | 2 + src/crux/cli/runs.go | 187 +++++++ src/crux/cli/runs_test.go | 252 +++++++++ src/crux/cli/serve.go | 51 +- src/crux/cli/verbs_test.go | 5 +- src/crux/runs/backend.go | 118 +++++ src/crux/runs/config.go | 159 ++++++ src/crux/runs/config_test.go | 244 +++++++++ src/crux/runs/provision.go | 78 +++ src/crux/runs/service.go | 494 ++++++++++++++++++ src/crux/runs/service_test.go | 458 ++++++++++++++++ src/crux/runstate/runstate.go | 362 +++++++++++++ src/crux/runstate/runstate_test.go | 381 ++++++++++++++ src/crux/serve/runs.go | 121 +++++ src/crux/serve/runs_test.go | 131 +++++ src/crux/serve/serve.go | 6 + 21 files changed, 3215 insertions(+), 3 deletions(-) create mode 100644 changelog.d/added-crux-contained-run-service.md create mode 100644 src/crux/api/runs.go create mode 100644 src/crux/api/runs_client.go create mode 100644 src/crux/api/runs_test.go create mode 100644 src/crux/cli/runs.go create mode 100644 src/crux/cli/runs_test.go create mode 100644 src/crux/runs/backend.go create mode 100644 src/crux/runs/config.go create mode 100644 src/crux/runs/config_test.go create mode 100644 src/crux/runs/provision.go create mode 100644 src/crux/runs/service.go create mode 100644 src/crux/runs/service_test.go create mode 100644 src/crux/runstate/runstate.go create mode 100644 src/crux/runstate/runstate_test.go create mode 100644 src/crux/serve/runs.go create mode 100644 src/crux/serve/runs_test.go diff --git a/changelog.d/added-crux-contained-run-service.md b/changelog.d/added-crux-contained-run-service.md new file mode 100644 index 0000000000..fb9167d36a --- /dev/null +++ b/changelog.d/added-crux-contained-run-service.md @@ -0,0 +1 @@ +- crux: contained run service (`crux runs`) on top of the sandbox launcher (Podman + kubejob) with saved run state under `runstate/` and `/v1/runs` serve routes (RFC hivecommons/crux#114, S2 + S7 minimum). diff --git a/src/crux/api/runs.go b/src/crux/api/runs.go new file mode 100644 index 0000000000..29360311d3 --- /dev/null +++ b/src/crux/api/runs.go @@ -0,0 +1,122 @@ +package api + +import "time" + +// Contained runs and saved run results (hivecommons/crux#114, S2 and S7): +// +// POST /v1/runs run an untrusted command in a sandbox +// GET /v1/runs/{id} the saved record, whatever the process state +// POST /v1/runs/{id}/cancel cancel; after completion returns the result +// unchanged with already_finished: true +// +// Routes beyond these (restart, listing) are deferred; see +// src/docs/design/crux-work-services.md. +const PathRuns = "/" + Version + "/runs" + +// RunPath is the route of one run's saved record. +func RunPath(id string) string { return PathRuns + "/" + id } + +// RunCancelPath is the route that cancels one run. +func RunCancelPath(id string) string { return RunPath(id) + "/cancel" } + +// The runs routes answer CodeUnsupported (501, shared with the checkout +// routes) when the request asks for something the configured backend cannot +// enforce (a network mode, a profile, a workspace form); the message names it +// and the run never starts with the limit dropped. It differs from +// CodeNotSupported, which is a whole route the server cannot serve (here: a +// server started without a contained-run backend answers POST /v1/runs with +// CodeNotSupported while still serving the saved records). + +// Run statuses, the closed set every record reports. +const ( + RunAccepted = "accepted" + RunRunning = "running" + RunWaiting = "waiting" + RunCompleted = "completed" + RunFailed = "failed" + RunCancelled = "cancelled" + RunUnfinished = "unfinished" + RunUnsupported = "unsupported" +) + +// Run profiles. +const ( + // RunProfileExperiment (the default) runs with network none, no + // credentials and time and output limits. + RunProfileExperiment = "experiment" + // RunProfileAgent is S6's execution half and is not yet accepted + // (CodeUnsupported). + RunProfileAgent = "agent" +) + +// Network modes a run may request. +const ( + RunNetworkNone = "none" + RunNetworkRestricted = "restricted" +) + +// ContainedRunWorktree is the ad-hoc workspace form: the service clones repo and +// pins commit (a full 40-hex SHA) under the run's directory. +type ContainedRunWorktree struct { + Repo string `json:"repo"` + Commit string `json:"commit"` +} + +// ContainedRunRequest is POST /v1/runs. Exactly one of Checkout, Worktree or +// WorkspacePath names the workspace. Checkout (an S1 id) is accepted by the +// wire shape but answers CodeUnsupported until the checkout service lands. +type ContainedRunRequest struct { + Checkout string `json:"checkout,omitempty"` + Worktree *ContainedRunWorktree `json:"worktree,omitempty"` + WorkspacePath string `json:"workspace_path,omitempty"` + Command []string `json:"command"` + EnvAllowlist []string `json:"env_allowlist,omitempty"` + // Network is RunNetworkNone (default) or RunNetworkRestricted. + Network string `json:"network,omitempty"` + // TimeoutMs bounds the command; zero uses the server default. + TimeoutMs int64 `json:"timeout_ms,omitempty"` + // MaxOutputBytes caps each of stdout and stderr in the response; zero + // uses the server default. The full output is kept as an artifact. + MaxOutputBytes int64 `json:"max_output_bytes,omitempty"` + // Profile is RunProfileExperiment (default) or RunProfileAgent. + Profile string `json:"profile,omitempty"` + // IdempotencyKey, when set, makes a repeated request return the + // existing run instead of starting a second one (S7). + IdempotencyKey string `json:"idempotency_key,omitempty"` + // Detach returns as soon as the run is accepted (202) instead of + // waiting for the result; poll GET /v1/runs/{id}. + Detach bool `json:"detach,omitempty"` +} + +// Run is the saved record of one contained run: the response of every runs +// route. Fields that are not yet known are absent; ExitCode in particular is +// present only when the command's exit is known, never guessed. +type ContainedRun struct { + ID string `json:"id"` + Status string `json:"status"` + Owner string `json:"owner,omitempty"` + // Reason explains a failed, unfinished or unsupported status. + Reason string `json:"reason,omitempty"` + CreatedAt time.Time `json:"created_at"` + StartedAt *time.Time `json:"started_at,omitempty"` + FinishedAt *time.Time `json:"finished_at,omitempty"` + + Profile string `json:"profile,omitempty"` + Network string `json:"network,omitempty"` + Backend string `json:"backend,omitempty"` + Command []string `json:"command,omitempty"` + Workdir string `json:"workspace,omitempty"` + ExitCode *int `json:"exit_code,omitempty"` + TimedOut bool `json:"timed_out,omitempty"` + Stdout string `json:"stdout,omitempty"` + Stderr string `json:"stderr,omitempty"` + // StdoutTruncatedAt and StderrTruncatedAt are the byte offsets at which + // the response output was cut; absent when nothing was truncated. The + // full, scrubbed output is the stdout.txt / stderr.txt artifact in the + // run's directory. + StdoutTruncatedAt *int64 `json:"stdout_truncated_at,omitempty"` + StderrTruncatedAt *int64 `json:"stderr_truncated_at,omitempty"` + // AlreadyFinished is set on a cancel response when the run had already + // reached a terminal status; the record is returned unchanged. + AlreadyFinished bool `json:"already_finished,omitempty"` +} diff --git a/src/crux/api/runs_client.go b/src/crux/api/runs_client.go new file mode 100644 index 0000000000..446f05420a --- /dev/null +++ b/src/crux/api/runs_client.go @@ -0,0 +1,27 @@ +package api + +import ( + "context" + "net/http" +) + +// CreateRun is POST /v1/runs: the saved record, final unless req.Detach. +func (c *Client) CreateRun(ctx context.Context, req ContainedRunRequest) (ContainedRun, error) { + var out ContainedRun + err := c.Do(ctx, http.MethodPost, PathRuns, req, &out) + return out, err +} + +// GetRun is GET /v1/runs/{id}. +func (c *Client) GetRun(ctx context.Context, id string) (ContainedRun, error) { + var out ContainedRun + err := c.Do(ctx, http.MethodGet, RunPath(id), nil, &out) + return out, err +} + +// CancelRun is POST /v1/runs/{id}/cancel. +func (c *Client) CancelRun(ctx context.Context, id string) (ContainedRun, error) { + var out ContainedRun + err := c.Do(ctx, http.MethodPost, RunCancelPath(id), nil, &out) + return out, err +} diff --git a/src/crux/api/runs_test.go b/src/crux/api/runs_test.go new file mode 100644 index 0000000000..efe0433393 --- /dev/null +++ b/src/crux/api/runs_test.go @@ -0,0 +1,12 @@ +package api + +import "testing" + +func TestRunPaths(t *testing.T) { + if PathRuns != "/v1/runs" || RunPath("run_a") != "/v1/runs/run_a" || RunCancelPath("run_a") != "/v1/runs/run_a/cancel" { + t.Fatalf("paths: %s %s %s", PathRuns, RunPath("run_a"), RunCancelPath("run_a")) + } + if CodeUnsupported == CodeNotSupported { + t.Fatal("unsupported-limit refusals must be distinguishable from a not-configured server") + } +} diff --git a/src/crux/cli/cli.go b/src/crux/cli/cli.go index b3879de2b5..2ef01ed3d7 100644 --- a/src/crux/cli/cli.go +++ b/src/crux/cli/cli.go @@ -209,6 +209,11 @@ func (e *Env) dispatch(ctx context.Context, opts globalOptions, name string, arg } } return e.local(ctx, opts, name, args) + case "runs": + // The contained-run family lives on a `crux serve`; there is no + // in-process form, so the verb is only reachable with --server. + fmt.Fprintln(e.Stderr, "crux: runs needs --server ADDR (or CRUX_SERVER); contained runs execute on a crux serve") + return ExitUsage case "help", "-h", "--help": fs := flag.NewFlagSet("crux", flag.ContinueOnError) e.usage(fs) @@ -339,6 +344,8 @@ Commands (each maps onto crux.Runtime): runtime that spawned it, not run by hand cline-turn one bounded Cline CLI turn (stdin prompt); --model provider/model, --check for preflight + runs start|status|cancel contained command runs on a crux serve + (--server only; 'crux runs help' for flags) Work services (no runtime; hivecommons/crux#114): checkout add|rm|list pinned checkouts under /checkouts diff --git a/src/crux/cli/remote.go b/src/crux/cli/remote.go index 20882c6a90..accd3e92b1 100644 --- a/src/crux/cli/remote.go +++ b/src/crux/cli/remote.go @@ -53,6 +53,8 @@ func (e *Env) remote(ctx context.Context, opts globalOptions, name string, args return e.remoteStop(ctx, c, args) case "checkout": return e.remoteCheckout(ctx, c, args) + case "runs": + return e.remoteRuns(ctx, c, args) default: fmt.Fprintf(e.Stderr, "crux: %q is not available with --server; it runs where the runtime is\n", name) return ExitUsage diff --git a/src/crux/cli/runs.go b/src/crux/cli/runs.go new file mode 100644 index 0000000000..7b694ae606 --- /dev/null +++ b/src/crux/cli/runs.go @@ -0,0 +1,187 @@ +package cli + +import ( + "context" + "encoding/json" + "fmt" + "strings" + + "github.com/hivecommons/hive/crux/api" +) + +// runsUsage is the `crux runs` synopsis. The verb is `runs`, not `run`, +// because `crux run` already starts an agent from a definition. +const runsUsage = `usage: crux --server ADDR runs start [flags] -- [args…] + crux --server ADDR runs status + crux --server ADDR runs cancel + +start flags: + --repo URL --commit SHA clone URL at SHA into a fresh workspace + --workspace PATH run in an existing directory on the server + --env NAME pass this environment variable through (repeatable) + --network none|restricted network for the command (default none) + --timeout DURATION wall-clock limit (server default when unset) + --max-output BYTES bytes of stdout/stderr returned (server default) + --profile experiment|agent + --idempotency-key KEY return the existing run for a repeated request + --detach return the accepted record instead of waiting + --json print the record as JSON` + +// remoteRuns is `crux --server … runs`: the contained-run family (RFC +// crux#114 S2 + S7). It only exists against a server; the runtime that +// launches sandboxes and keeps run records lives there. +func (e *Env) remoteRuns(ctx context.Context, c *api.Client, args []string) int { + if len(args) == 0 { + fmt.Fprintln(e.Stderr, runsUsage) + return ExitUsage + } + switch args[0] { + case "start": + return e.remoteRunsStart(ctx, c, args[1:]) + case "status": + return e.remoteRunsGet(ctx, c, args[1:]) + case "cancel": + return e.remoteRunsCancel(ctx, c, args[1:]) + case "help", "-h", "--help": + fmt.Fprintln(e.Stderr, runsUsage) + return ExitOK + default: + fmt.Fprintf(e.Stderr, "crux: runs: unknown subcommand %q\n%s\n", args[0], runsUsage) + return ExitUsage + } +} + +// stringList is a repeatable string flag. +type stringList []string + +func (l *stringList) String() string { return strings.Join(*l, ",") } +func (l *stringList) Set(v string) error { *l = append(*l, v); return nil } + +func (e *Env) remoteRunsStart(ctx context.Context, c *api.Client, args []string) int { + fs := e.subFlags("runs start") + var req api.ContainedRunRequest + var wt api.ContainedRunWorktree + var env stringList + fs.StringVar(&wt.Repo, "repo", "", "repository URL to clone") + fs.StringVar(&wt.Commit, "commit", "", "full commit SHA to check out") + fs.StringVar(&req.WorkspacePath, "workspace", "", "existing directory on the server") + fs.Var(&env, "env", "environment variable to pass through (repeatable)") + fs.StringVar(&req.Network, "network", "", "none or restricted") + timeout := fs.Duration("timeout", 0, "wall-clock limit") + fs.Int64Var(&req.MaxOutputBytes, "max-output", 0, "bytes of output returned") + fs.StringVar(&req.Profile, "profile", "", "experiment or agent") + fs.StringVar(&req.IdempotencyKey, "idempotency-key", "", "idempotency key") + fs.BoolVar(&req.Detach, "detach", false, "return the accepted record") + asJSON := fs.Bool("json", false, "print the record as JSON") + if code, ok := parseSub(fs, args); !ok { + return code + } + if fs.NArg() == 0 { + fmt.Fprintln(e.Stderr, runsUsage) + return ExitUsage + } + req.Command = fs.Args() + req.EnvAllowlist = env + req.TimeoutMs = timeout.Milliseconds() + if wt.Repo != "" || wt.Commit != "" { + req.Worktree = &wt + } + run, err := c.CreateRun(ctx, req) + if err != nil { + return e.remoteErr("runs start", "", err) + } + return e.printRun(run, *asJSON) +} + +func (e *Env) remoteRunsGet(ctx context.Context, c *api.Client, args []string) int { + fs := e.subFlags("runs status") + asJSON := fs.Bool("json", false, "print the record as JSON") + if code, ok := parseSub(fs, args); !ok { + return code + } + if fs.NArg() != 1 { + fmt.Fprintln(e.Stderr, "usage: crux --server ADDR runs status [--json] ") + return ExitUsage + } + run, err := c.GetRun(ctx, fs.Arg(0)) + if err != nil { + return e.remoteErr("runs status", fs.Arg(0), err) + } + return e.printRun(run, *asJSON) +} + +func (e *Env) remoteRunsCancel(ctx context.Context, c *api.Client, args []string) int { + fs := e.subFlags("runs cancel") + asJSON := fs.Bool("json", false, "print the record as JSON") + if code, ok := parseSub(fs, args); !ok { + return code + } + if fs.NArg() != 1 { + fmt.Fprintln(e.Stderr, "usage: crux --server ADDR runs cancel [--json] ") + return ExitUsage + } + run, err := c.CancelRun(ctx, fs.Arg(0)) + if err != nil { + return e.remoteErr("runs cancel", fs.Arg(0), err) + } + if run.AlreadyFinished { + fmt.Fprintf(e.Stderr, "crux: run %s had already finished\n", run.ID) + } + return e.printRun(run, *asJSON) +} + +// printRun writes the record: JSON when asked, otherwise a one-line +// summary on stdout with the command's own output following so a caller +// can pipe `crux runs start` like the command itself. The exit code is +// the command's when it is known; an unknown exit is ExitError, never a +// guess. +func (e *Env) printRun(run api.ContainedRun, asJSON bool) int { + if asJSON { + enc := json.NewEncoder(e.Stdout) + enc.SetIndent("", " ") + if err := enc.Encode(run); err != nil { + fmt.Fprintln(e.Stderr, "crux:", err) + return ExitError + } + return runExit(run) + } + line := fmt.Sprintf("%s %s", run.ID, run.Status) + if run.ExitCode != nil { + line += fmt.Sprintf(" exit=%d", *run.ExitCode) + } + if run.TimedOut { + line += " timed_out" + } + if run.Reason != "" { + line += " (" + run.Reason + ")" + } + fmt.Fprintln(e.Stderr, line) + if run.Stdout != "" { + fmt.Fprint(e.Stdout, run.Stdout) + if run.StdoutTruncatedAt != nil { + fmt.Fprintf(e.Stderr, "\n[stdout truncated at byte %d; the full output is kept with the run]\n", *run.StdoutTruncatedAt) + } + } + if run.Stderr != "" { + fmt.Fprint(e.Stderr, run.Stderr) + if run.StderrTruncatedAt != nil { + fmt.Fprintf(e.Stderr, "\n[stderr truncated at byte %d; the full output is kept with the run]\n", *run.StderrTruncatedAt) + } + } + return runExit(run) +} + +// runExit maps a record onto the process exit code. +func runExit(run api.ContainedRun) int { + switch run.Status { + case api.RunCompleted: + if run.ExitCode != nil && *run.ExitCode != 0 { + return ExitError + } + return ExitOK + case api.RunAccepted, api.RunRunning, api.RunWaiting: + return ExitOK + default: + return ExitError + } +} diff --git a/src/crux/cli/runs_test.go b/src/crux/cli/runs_test.go new file mode 100644 index 0000000000..726de5c7cf --- /dev/null +++ b/src/crux/cli/runs_test.go @@ -0,0 +1,252 @@ +package cli + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "path/filepath" + "strings" + "sync" + "testing" + + "github.com/hivecommons/hive/crux/api" + "github.com/hivecommons/hive/crux/config" + "github.com/hivecommons/hive/crux/runs" + "github.com/hivecommons/hive/crux/runstate" +) + +// fakeRunsAPI is the contained-run family of a `crux serve` stand-in. +type fakeRunsAPI struct { + mu sync.Mutex + created []api.ContainedRunRequest + cancelled []string +} + +func (f *fakeRunsAPI) handler() http.Handler { + exit := func(v int) *int { return &v } + cut := int64(5) + records := map[string]api.ContainedRun{ + "run_ok": {ID: "run_ok", Status: api.RunCompleted, ExitCode: exit(0), Stdout: "hello\n", Stderr: "warned\n"}, + "run_fail": {ID: "run_fail", Status: api.RunCompleted, ExitCode: exit(2), Reason: "exit status 2"}, + "run_cut": {ID: "run_cut", Status: api.RunCompleted, ExitCode: exit(0), Stdout: "12345", StdoutTruncatedAt: &cut, Stderr: "abcde", StderrTruncatedAt: &cut}, + "run_lost": {ID: "run_lost", Status: api.RunUnfinished, Reason: "pod vanished"}, + "run_late": {ID: "run_late", Status: api.RunFailed, TimedOut: true, Reason: "timed out", AlreadyFinished: true}, + "run_sleep": {ID: "run_sleep", Status: api.RunRunning}, + } + mux := http.NewServeMux() + mux.HandleFunc("POST "+api.PathRuns, func(w http.ResponseWriter, r *http.Request) { + var req api.ContainedRunRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + w.WriteHeader(http.StatusBadRequest) + _ = json.NewEncoder(w).Encode(api.Error{Error: err.Error(), Code: api.CodeBadRequest}) + return + } + f.mu.Lock() + f.created = append(f.created, req) + f.mu.Unlock() + if req.Profile == api.RunProfileAgent { + w.WriteHeader(http.StatusNotImplemented) + _ = json.NewEncoder(w).Encode(api.Error{Error: "profile agent is not available yet", Code: api.CodeUnsupported}) + return + } + if req.Detach { + w.WriteHeader(http.StatusAccepted) + _ = json.NewEncoder(w).Encode(api.ContainedRun{ID: "run_new", Status: api.RunAccepted}) + return + } + _ = json.NewEncoder(w).Encode(records["run_ok"]) + }) + mux.HandleFunc("GET "+api.PathRuns+"/{id}", func(w http.ResponseWriter, r *http.Request) { + rec, ok := records[r.PathValue("id")] + if !ok { + w.WriteHeader(http.StatusNotFound) + _ = json.NewEncoder(w).Encode(api.Error{Error: "run not found", Code: api.CodeNotFound}) + return + } + _ = json.NewEncoder(w).Encode(rec) + }) + mux.HandleFunc("POST "+api.PathRuns+"/{id}/cancel", func(w http.ResponseWriter, r *http.Request) { + id := r.PathValue("id") + f.mu.Lock() + f.cancelled = append(f.cancelled, id) + f.mu.Unlock() + rec, ok := records[id] + if !ok { + w.WriteHeader(http.StatusNotFound) + _ = json.NewEncoder(w).Encode(api.Error{Error: "run not found", Code: api.CodeNotFound}) + return + } + if rec.Status == api.RunRunning { + rec.Status = api.RunCancelled + } + _ = json.NewEncoder(w).Encode(rec) + }) + return mux +} + +func TestRunsIsRemoteOnly(t *testing.T) { + t.Setenv(config.ServerEnvVar, "") + h := newHarness(t) + if code := h.run("runs", "status", "run_ok"); code != ExitUsage || !strings.Contains(h.stderr.String(), "needs --server") { + t.Fatalf("local runs: %d %s", code, h.stderr) + } + if len(h.rt.Calls()) != 0 { + t.Fatalf("runs touched the local runtime: %v", h.rt.Calls()) + } + h = newHarness(t) + h.run("help") + if !strings.Contains(h.stderr.String(), "runs start|status|cancel") { + t.Fatalf("usage does not list runs:\n%s", h.stderr) + } +} + +func TestRunsStartStatusCancel(t *testing.T) { + t.Setenv(config.ServerEnvVar, "") + t.Setenv(config.ServeTokenEnvVar, "") + f := &fakeRunsAPI{} + ts := httptest.NewServer(f.handler()) + t.Cleanup(ts.Close) + server := ts.URL + + h := newHarness(t) + code := h.run("--server", server, "runs", "start", "--repo", "https://github.com/hivecommons/hive", "--commit", strings.Repeat("a", 40), + "--env", "PATH", "--env", "LANG", "--network", "none", "--timeout", "2s", "--max-output", "64", "--idempotency-key", "k1", "--", "make", "-j2", "test") + if code != ExitOK { + t.Fatalf("start: %d %s", code, h.stderr) + } + if h.stdout.String() != "hello\n" || !strings.Contains(h.stderr.String(), "run_ok completed exit=0") || !strings.Contains(h.stderr.String(), "warned") { + t.Fatalf("start output:\nstdout=%q\nstderr=%q", h.stdout, h.stderr) + } + req := f.created[0] + if req.Worktree == nil || req.Worktree.Repo != "https://github.com/hivecommons/hive" || req.Worktree.Commit != strings.Repeat("a", 40) || + strings.Join(req.Command, " ") != "make -j2 test" || strings.Join(req.EnvAllowlist, ",") != "PATH,LANG" || + req.Network != "none" || req.TimeoutMs != 2000 || req.MaxOutputBytes != 64 || req.IdempotencyKey != "k1" || req.Detach { + t.Fatalf("request = %+v (worktree %+v)", req, req.Worktree) + } + + h = newHarness(t) + if code := h.run("--server", server, "runs", "start", "--workspace", "/srv/ws", "--detach", "--json", "--", "true"); code != ExitOK { + t.Fatalf("detach: %d %s", code, h.stderr) + } + var detached api.ContainedRun + if err := json.Unmarshal([]byte(h.stdout.String()), &detached); err != nil || detached.ID != "run_new" || detached.Status != api.RunAccepted { + t.Fatalf("detach json = %q (%v)", h.stdout, err) + } + if req := f.created[1]; req.WorkspacePath != "/srv/ws" || req.Worktree != nil || !req.Detach { + t.Fatalf("detach request = %+v", req) + } + + h = newHarness(t) + if code := h.run("--server", server, "runs", "start", "--workspace", "/srv/ws", "--profile", "agent", "--", "true"); code != ExitError || !strings.Contains(h.stderr.String(), "not available yet") { + t.Fatalf("unsupported: %d %s", code, h.stderr) + } + + for _, tc := range []struct { + id string + code int + stdout string + stderr string + }{ + {"run_ok", ExitOK, "hello\n", "run_ok completed exit=0"}, + {"run_fail", ExitError, "", "run_fail completed exit=2 (exit status 2)"}, + {"run_cut", ExitOK, "12345", "stdout truncated at byte 5"}, + {"run_lost", ExitError, "", "run_lost unfinished (pod vanished)"}, + {"run_sleep", ExitOK, "", "run_sleep running"}, + {"run_missing", ExitError, "", "run not found"}, + } { + h = newHarness(t) + if code := h.run("--server", server, "runs", "status", tc.id); code != tc.code || h.stdout.String() != tc.stdout || !strings.Contains(h.stderr.String(), tc.stderr) { + t.Fatalf("status %s: %d stdout=%q stderr=%q", tc.id, code, h.stdout, h.stderr) + } + } + h = newHarness(t) + if code := h.run("--server", server, "runs", "status", "--json", "run_cut"); code != ExitOK || !strings.Contains(h.stdout.String(), `"stderr_truncated_at": 5`) { + t.Fatalf("status json: %d %s", code, h.stdout) + } + + h = newHarness(t) + if code := h.run("--server", server, "runs", "cancel", "run_sleep"); code != ExitError || !strings.Contains(h.stderr.String(), "run_sleep cancelled") { + t.Fatalf("cancel: %d %s", code, h.stderr) + } + h = newHarness(t) + if code := h.run("--server", server, "runs", "cancel", "--json", "run_late"); code != ExitError || !strings.Contains(h.stderr.String(), "had already finished") || !strings.Contains(h.stdout.String(), `"timed_out": true`) { + t.Fatalf("late cancel: %d stdout=%s stderr=%s", code, h.stdout, h.stderr) + } + if strings.Join(f.cancelled, ",") != "run_sleep,run_late" { + t.Fatalf("cancelled = %v", f.cancelled) + } +} + +func TestRunsUsage(t *testing.T) { + t.Setenv(config.ServerEnvVar, "") + t.Setenv(config.ServeTokenEnvVar, "") + ts := httptest.NewServer((&fakeRunsAPI{}).handler()) + t.Cleanup(ts.Close) + for _, tc := range []struct { + args []string + code int + stderr string + }{ + {[]string{"runs"}, ExitUsage, "usage: crux --server ADDR runs start"}, + {[]string{"runs", "help"}, ExitOK, "usage: crux --server ADDR runs start"}, + {[]string{"runs", "bogus"}, ExitUsage, `unknown subcommand "bogus"`}, + {[]string{"runs", "start"}, ExitUsage, "usage: crux --server ADDR runs start"}, + {[]string{"runs", "start", "-h"}, ExitOK, "-repo"}, + {[]string{"runs", "start", "--bogus", "x"}, ExitUsage, "bogus"}, + {[]string{"runs", "status"}, ExitUsage, "usage: crux --server ADDR runs status"}, + {[]string{"runs", "status", "-bogus"}, ExitUsage, "bogus"}, + {[]string{"runs", "cancel", "a", "b"}, ExitUsage, "usage: crux --server ADDR runs cancel"}, + {[]string{"runs", "cancel", "-bogus"}, ExitUsage, "bogus"}, + } { + h := newHarness(t) + if code := h.run(append([]string{"--server", ts.URL}, tc.args...)...); code != tc.code || !strings.Contains(h.stderr.String(), tc.stderr) { + t.Fatalf("%v: %d %s", tc.args, code, h.stderr) + } + } +} + +func TestOpenRunsReconcilesAndLoadsTheBackend(t *testing.T) { + t.Setenv(runsConfigEnvVar, "") + h := newHarness(t) + workDir := t.TempDir() + store, err := runstate.Open(workDir) + if err != nil { + t.Fatal(err) + } + stale, _, err := store.Create(runstate.Record{}) + if err != nil { + t.Fatal(err) + } + svc, err := h.env.openRuns(workDir, "") + if err != nil { + t.Fatal(err) + } + if svc.Configured() { + t.Fatal("configured without a config file") + } + if run, err := svc.Get(stale.ID); err != nil || run.Status != api.RunUnfinished { + t.Fatalf("stale record = %+v, %v", run, err) + } + if !strings.Contains(h.stderr.String(), stale.ID+" was in flight") { + t.Fatalf("reconcile not reported: %s", h.stderr) + } + + cfgPath := writeFile(t, filepath.Join(t.TempDir(), "runs.json"), `{"image":"img","workspace_roots":["/srv"]}`) + svc, err = h.env.openRuns(workDir, cfgPath) + if err != nil || !svc.Configured() || svc.Backend.Name != runs.BackendPodman { + t.Fatalf("configured = %v, %v", svc, err) + } + t.Setenv(runsConfigEnvVar, cfgPath) + if svc, err = h.env.openRuns(workDir, ""); err != nil || !svc.Configured() { + t.Fatalf("env config = %v, %v", svc, err) + } + if _, err := h.env.openRuns(workDir, writeFile(t, filepath.Join(t.TempDir(), "bad.json"), `{"backend":"job","image":"img"}`)); err == nil { + t.Fatal("bad config accepted") + } + if _, err := h.env.openRuns(writeFile(t, filepath.Join(t.TempDir(), "file"), ""), cfgPath); err == nil { + t.Fatal("work dir under a file accepted") + } + if code := h.run("serve", "--runs-config", filepath.Join(t.TempDir(), "missing.json")); code != ExitUsage || !strings.Contains(h.stderr.String(), "crux: runs:") { + t.Fatalf("serve with a missing runs config: %d %s", code, h.stderr) + } +} diff --git a/src/crux/cli/serve.go b/src/crux/cli/serve.go index 104848eccf..f188cecc03 100644 --- a/src/crux/cli/serve.go +++ b/src/crux/cli/serve.go @@ -12,6 +12,8 @@ import ( "github.com/hivecommons/hive/crux/checkout" "github.com/hivecommons/hive/crux/config" "github.com/hivecommons/hive/crux/lanes" + "github.com/hivecommons/hive/crux/runs" + "github.com/hivecommons/hive/crux/runstate" "github.com/hivecommons/hive/crux/serve" "github.com/nats-io/nats.go" ) @@ -40,12 +42,14 @@ func (e *Env) cmdServe(ctx context.Context, opts globalOptions, args []string) i fs.StringVar(&checkoutDir, "checkout-dir", "", "where pinned checkouts live (default $"+config.CheckoutDirEnvVar+", then /"+checkout.DirName+")") fs.StringVar(&operatorName, "operator-name", "", "name of the principal behind the control token on work-services routes (default $"+config.ServeOperatorNameEnvVar+", then "+serve.DefaultOperatorName+")") fs.StringVar(&laneConfig, "lanes-config", "", "opt-in durable lane configuration file (default $CRUX_LANES_CONFIG)") + var runsConfig string + fs.StringVar(&runsConfig, "runs-config", "", "opt-in contained-run backend configuration file (default $"+runsConfigEnvVar+")") fs.StringVar(&hiveAddr, "hive-callback", "", "hive capability endpoint to dial (default $"+callback.SocketEnvVar+"; none when unset)") if code, ok := parseSub(fs, args); !ok { return code } if fs.NArg() != 0 { - fmt.Fprintln(e.Stderr, "usage: crux serve [-socket PATH] [-listen HOST:PORT] [-token T] [-hive-callback ADDR] [-lanes-config PATH] [-operator-name NAME] [-checkout-allow PREFIXES] [-checkout-dir DIR]") + fmt.Fprintln(e.Stderr, "usage: crux serve [-socket PATH] [-listen HOST:PORT] [-token T] [-hive-callback ADDR] [-lanes-config PATH] [-operator-name NAME] [-checkout-allow PREFIXES] [-checkout-dir DIR] [-runs-config PATH]") return ExitUsage } if cfg.Token == "" { @@ -117,6 +121,12 @@ func (e *Env) cmdServe(ctx context.Context, opts globalOptions, args []string) i defer broker.Close() srv.Lanes, srv.LaneAuth = broker, auth } + runSvc, err := e.openRuns(opts.workDir, runsConfig) + if err != nil { + fmt.Fprintln(e.Stderr, "crux: runs:", err) + return ExitUsage + } + srv.Runs = runSvc if hive != nil { hive.install(ctx, rt, srv, e.Stdout) hive.watchPolicy(ctx, rt, e.Stdout, e.Stderr) @@ -148,3 +158,42 @@ func (e *Env) laneBrokerFailure(err error) int { fmt.Fprintf(e.Stderr, "crux: lanes: cannot initialize durable broker (%s)\n", category) return ExitError } + +// runsConfigEnvVar names the contained-run configuration file when +// --runs-config is not given. +const runsConfigEnvVar = "CRUX_RUNS_CONFIG" + +// openRuns builds the contained-run service. Saved records under +// /runs are always served, so a server with no backend configured +// still answers `crux runs status`; the backend comes from the config file +// when one is named. Records left in flight by an earlier process are +// marked unfinished here, before the first request can observe them. +func (e *Env) openRuns(workDir, path string) (*runs.Service, error) { + if path == "" { + path = os.Getenv(runsConfigEnvVar) + } + var cfg runs.Config + var backend *runs.Backend + if path != "" { + var err error + if cfg, err = runs.LoadConfig(path); err != nil { + return nil, err + } + if backend, err = runs.BackendFromConfig(cfg); err != nil { + return nil, err + } + } + store, err := runstate.Open(workDir) + if err != nil { + return nil, err + } + svc := runs.New(store, backend, cfg) + reconciled, err := svc.Reconcile() + if err != nil { + return nil, err + } + for _, rec := range reconciled { + fmt.Fprintf(e.Stderr, "crux: runs: %s was in flight at the last shutdown; marked unfinished\n", rec.ID) + } + return svc, nil +} diff --git a/src/crux/cli/verbs_test.go b/src/crux/cli/verbs_test.go index 712e98ced8..aad7ebbbc3 100644 --- a/src/crux/cli/verbs_test.go +++ b/src/crux/cli/verbs_test.go @@ -14,8 +14,9 @@ import ( ) // nonRuntimeVerbs are dispatch cases that do not operate on a runtime: serve -// exposes one, help prints usage, cline-turn executes one pane-side worker. -var nonRuntimeVerbs = map[string]bool{"serve": true, "help": true, "-h": true, "--help": true, "cline-turn": true, "checkout": true} +// exposes one, help prints usage, cline-turn executes one pane-side worker, +// checkout and runs drive work services on a server. +var nonRuntimeVerbs = map[string]bool{"serve": true, "help": true, "-h": true, "--help": true, "cline-turn": true, "checkout": true, "runs": true} func TestVerbsHaveRuntimeMethods(t *testing.T) { rt := reflect.TypeOf((*crux.Runtime)(nil)).Elem() diff --git a/src/crux/runs/backend.go b/src/crux/runs/backend.go new file mode 100644 index 0000000000..8b2aea2a57 --- /dev/null +++ b/src/crux/runs/backend.go @@ -0,0 +1,118 @@ +package runs + +import ( + "context" + "fmt" + "os/exec" + "strings" + + "github.com/hivecommons/hive/crux/config" + "github.com/hivecommons/hive/crux/kubejob" + "github.com/hivecommons/hive/crux/sandbox" +) + +// Capabilities is what a backend can enforce. A request for anything it +// cannot is refused before the run starts. +type Capabilities struct { + NetworkNone bool + NetworkRestricted bool +} + +// Backend pairs a sandbox.Launcher with its capabilities and the teardown +// that guarantees a cancelled or timed-out run leaves nothing behind. +type Backend struct { + Name string + Launcher sandbox.Launcher + Capabilities Capabilities + // Teardown, when set, is run after a cancel or timeout with the launch + // name. The podman launcher's client process dies with the context but + // the container it started does not, so podman removes it here; the + // Job launcher deletes its Job itself (propagationPolicy Background) + // and leaves Teardown nil. + Teardown func(ctx context.Context, name string) error +} + +// Supports reports whether the backend can enforce network mode mode; the +// returned string names what is missing when it cannot. +func (b Backend) Supports(network string) (string, bool) { + switch network { + case sandbox.NetworkModeNone: + if !b.Capabilities.NetworkNone { + return fmt.Sprintf("backend %s cannot enforce network %q", b.Name, network), false + } + case sandbox.NetworkModeRestricted: + if !b.Capabilities.NetworkRestricted { + return fmt.Sprintf("backend %s cannot enforce network %q", b.Name, network), false + } + default: + return fmt.Sprintf("unknown network mode %q", network), false + } + return "", true +} + +// BackendFromConfig selects and builds the backend cfg names. +func BackendFromConfig(cfg Config) (*Backend, error) { + switch cfg.Backend { + case BackendPodman: + bin := cfg.PodmanBinary + if bin == "" { + bin = sandbox.PodmanBinary + } + return &Backend{ + Name: BackendPodman, + Launcher: sandbox.PodmanLauncher{Binary: bin}, + Capabilities: Capabilities{NetworkNone: true, NetworkRestricted: cfg.NetworkRestricted}, + Teardown: podmanTeardown(bin), + }, nil + case BackendJob: + if cfg.Job == nil { + return nil, fmt.Errorf("%w: backend %q needs a job block", ErrConfig, BackendJob) + } + opts := jobOptions(*cfg.Job) + if err := opts.Validate(); err != nil { + return nil, fmt.Errorf("%w: %v", ErrConfig, err) + } + return &Backend{ + Name: BackendJob, + Launcher: &kubejob.Launcher{Options: opts}, + // The Job manifest carries no network mode; only an operator's + // NetworkPolicy can enforce one, and only "none" is claimable. + Capabilities: Capabilities{NetworkNone: cfg.JobNetworkPolicy}, + }, nil + } + return nil, fmt.Errorf("%w: unknown backend %q", ErrConfig, cfg.Backend) +} + +// podmanTeardown removes the named container whether or not it is still +// running; --ignore makes an already-removed container a success. +func podmanTeardown(bin string) func(context.Context, string) error { + return func(ctx context.Context, name string) error { + out, err := exec.CommandContext(ctx, bin, "rm", "--force", "--ignore", name).CombinedOutput() + if err != nil { + return fmt.Errorf("podman rm %s: %w: %s", name, err, strings.TrimSpace(string(out))) + } + return nil + } +} + +// jobOptions mirrors the agent sandbox's config-to-launcher mapping +// (agent.JobOptionsFromConfig) without importing the agent manager. +func jobOptions(job config.SandboxJobConfig) kubejob.Options { + opts := kubejob.Options{ + WorkspaceClaim: job.WorkspaceClaim, + WorkspaceClaimMount: job.WorkspaceClaimMount, + NodeSelector: job.NodeSelector, + ServiceAccount: job.ServiceAccount, + SchedulerName: job.SchedulerName, + EnvFromSecrets: job.EnvFromSecrets, + TTLSeconds: job.TTLSeconds, + Resources: kubejob.Resources{Limits: job.Resources.Limits, Requests: job.Resources.Requests}, + } + for _, t := range job.Tolerations { + opts.Tolerations = append(opts.Tolerations, kubejob.Toleration{Key: t.Key, Operator: t.Operator, Value: t.Value, Effect: t.Effect}) + } + for _, v := range job.Volumes { + opts.ExtraVolumes = append(opts.ExtraVolumes, kubejob.VolumeMount{Name: v.Name, Claim: v.Claim, MountPath: v.MountPath, ReadOnly: v.ReadOnly}) + } + return opts +} diff --git a/src/crux/runs/config.go b/src/crux/runs/config.go new file mode 100644 index 0000000000..92040ba151 --- /dev/null +++ b/src/crux/runs/config.go @@ -0,0 +1,159 @@ +// Package runs is the contained-run service of the Crux work services (S2 +// in hivecommons/crux#114): run a command the caller does not trust in a +// sandbox with no network and no credentials, bounded in time and output, +// and report the real exit code β€” or, when the exit is not knowable, say +// so. Records live in runstate (S7) so a caller that lost its connection +// can ask again by ID. +// +// sandbox.Launcher is the execution seam; Backend pairs a launcher with +// what it can enforce. A request for a limit the selected backend cannot +// enforce is refused with an *UnsupportedError, never run weakened. +package runs + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "strings" + "time" + + "github.com/hivecommons/hive/crux/config" +) + +// Backend names, Config.Backend. +const ( + BackendPodman = "podman" + BackendJob = "job" +) + +// Limits and defaults. +const ( + DefaultTimeout = 10 * time.Minute + DefaultMaxTimeout = 2 * time.Hour + DefaultMaxOutputBytes = 1 << 20 + // DefaultMaxOutputCap bounds what a caller may ask for; the full output + // is kept as an artifact regardless, so the cap protects the response + // and the record, not the evidence. + DefaultMaxOutputCap = 64 << 20 + maxConfigBytes = 1 << 20 + // TeardownTimeout bounds the container removal after a cancel or + // timeout, run on its own context because the run's is already dead. + TeardownTimeout = 30 * time.Second +) + +// ErrConfig wraps every configuration error. +var ErrConfig = errors.New("runs: invalid configuration") + +// Config is the operator's choice of backend and bounds, loaded by +// LoadConfig from the JSON file `crux serve --runs-config` names. +// Selection is configuration, not per request. +type Config struct { + // Backend is BackendPodman (default) or BackendJob. + Backend string `json:"backend,omitempty"` + // Image is the container image every run uses. Required. + Image string `json:"image"` + // PodmanBinary overrides the podman executable (tests). + PodmanBinary string `json:"podman_binary,omitempty"` + // Job holds the Kubernetes Job settings when Backend is BackendJob; + // the same shape as the agent sandbox's job block. + Job *config.SandboxJobConfig `json:"job,omitempty"` + // JobNetworkPolicy is the operator's assertion that a NetworkPolicy in + // the Job namespace denies egress for pods labelled + // app.kubernetes.io/managed-by=hive-kubejob. The Job launcher cannot + // set a network mode itself, so without this assertion a request for + // network "none" on the job backend is refused. + JobNetworkPolicy bool `json:"job_network_policy,omitempty"` + // NetworkRestricted is the operator's assertion that a container + // network named "restricted" exists on the podman host; without it a + // request for network "restricted" is refused. + NetworkRestricted bool `json:"network_restricted,omitempty"` + // RepoAllowlist is the set of repository URL prefixes the ad-hoc + // worktree form may clone. Empty refuses every worktree request. + RepoAllowlist []string `json:"repo_allowlist,omitempty"` + // WorkspaceRoots is the set of host directories a plain workspace_path + // may lie under. Empty refuses every workspace_path request. + WorkspaceRoots []string `json:"workspace_roots,omitempty"` + // DefaultTimeoutMs and MaxTimeoutMs bound timeout_ms; zero takes the + // package defaults. + DefaultTimeoutMs int64 `json:"default_timeout_ms,omitempty"` + MaxTimeoutMs int64 `json:"max_timeout_ms,omitempty"` + // DefaultMaxOutputBytes and MaxOutputBytesCap bound max_output_bytes; + // zero takes the package defaults. + DefaultMaxOutputBytes int64 `json:"default_max_output_bytes,omitempty"` + MaxOutputBytesCap int64 `json:"max_output_bytes_cap,omitempty"` +} + +// LoadConfig reads and validates the JSON configuration at path. +func LoadConfig(path string) (Config, error) { + f, err := os.Open(path) + if err != nil { + return Config{}, fmt.Errorf("%w: %v", ErrConfig, err) + } + defer f.Close() + return loadConfig(f) +} + +func loadConfig(r io.Reader) (Config, error) { + data, err := io.ReadAll(io.LimitReader(r, maxConfigBytes+1)) + if err != nil || len(data) > maxConfigBytes { + return Config{}, fmt.Errorf("%w: configuration size", ErrConfig) + } + d := json.NewDecoder(bytes.NewReader(data)) + d.DisallowUnknownFields() + var cfg Config + if err := d.Decode(&cfg); err != nil { + return Config{}, fmt.Errorf("%w: %v", ErrConfig, err) + } + cfg.Defaults() + return cfg, cfg.Validate() +} + +// Defaults fills the zero fields. +func (c *Config) Defaults() { + if c.Backend == "" { + c.Backend = BackendPodman + } + if c.DefaultTimeoutMs <= 0 { + c.DefaultTimeoutMs = DefaultTimeout.Milliseconds() + } + if c.MaxTimeoutMs <= 0 { + c.MaxTimeoutMs = DefaultMaxTimeout.Milliseconds() + } + if c.DefaultMaxOutputBytes <= 0 { + c.DefaultMaxOutputBytes = DefaultMaxOutputBytes + } + if c.MaxOutputBytesCap <= 0 { + c.MaxOutputBytesCap = DefaultMaxOutputCap + } +} + +// Validate reports the errors that would otherwise surface as a failed run. +func (c Config) Validate() error { + if strings.TrimSpace(c.Image) == "" { + return fmt.Errorf("%w: image is required", ErrConfig) + } + switch c.Backend { + case BackendPodman: + case BackendJob: + if c.Job == nil { + return fmt.Errorf("%w: backend %q needs a job block", ErrConfig, BackendJob) + } + default: + return fmt.Errorf("%w: backend must be %q or %q", ErrConfig, BackendPodman, BackendJob) + } + if c.DefaultTimeoutMs > c.MaxTimeoutMs { + return fmt.Errorf("%w: default_timeout_ms exceeds max_timeout_ms", ErrConfig) + } + if c.DefaultMaxOutputBytes > c.MaxOutputBytesCap { + return fmt.Errorf("%w: default_max_output_bytes exceeds max_output_bytes_cap", ErrConfig) + } + for _, root := range c.WorkspaceRoots { + if !strings.HasPrefix(root, "/") { + return fmt.Errorf("%w: workspace_roots entries must be absolute: %q", ErrConfig, root) + } + } + return nil +} diff --git a/src/crux/runs/config_test.go b/src/crux/runs/config_test.go new file mode 100644 index 0000000000..0298fecbc7 --- /dev/null +++ b/src/crux/runs/config_test.go @@ -0,0 +1,244 @@ +package runs + +import ( + "context" + "errors" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/hivecommons/hive/crux/api" + "github.com/hivecommons/hive/crux/config" + "github.com/hivecommons/hive/crux/kubejob" + "github.com/hivecommons/hive/crux/sandbox" +) + +func TestLoadConfig(t *testing.T) { + cases := []struct { + name string + body string + err string + }{ + {"podman defaults", `{"image":"img"}`, ""}, + {"job", `{"backend":"job","image":"img","job":{"workspace_claim":"pvc"}}`, ""}, + {"unknown field", `{"image":"img","bogus":1}`, "bogus"}, + {"no image", `{"backend":"podman"}`, "image is required"}, + {"bad backend", `{"backend":"docker","image":"img"}`, "backend must be"}, + {"job without block", `{"backend":"job","image":"img"}`, "needs a job block"}, + {"timeout order", `{"image":"img","default_timeout_ms":5,"max_timeout_ms":1}`, "default_timeout_ms exceeds"}, + {"output order", `{"image":"img","default_max_output_bytes":5,"max_output_bytes_cap":1}`, "default_max_output_bytes exceeds"}, + {"relative root", `{"image":"img","workspace_roots":["ws"]}`, "must be absolute"}, + {"too big", `{"image":"` + strings.Repeat("x", maxConfigBytes) + `"}`, "configuration size"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + cfg, err := loadConfig(strings.NewReader(tc.body)) + if tc.err == "" { + if err != nil { + t.Fatal(err) + } + if cfg.DefaultTimeoutMs != DefaultTimeout.Milliseconds() || cfg.MaxOutputBytesCap != DefaultMaxOutputCap { + t.Fatalf("defaults not applied: %+v", cfg) + } + return + } + if err == nil || !errors.Is(err, ErrConfig) || !strings.Contains(err.Error(), tc.err) { + t.Fatalf("err = %v, want %q", err, tc.err) + } + }) + } + path := filepath.Join(t.TempDir(), "runs.json") + if err := os.WriteFile(path, []byte(`{"image":"img","repo_allowlist":["https://github.com/"]}`), 0o600); err != nil { + t.Fatal(err) + } + cfg, err := LoadConfig(path) + if err != nil || cfg.Backend != BackendPodman || len(cfg.RepoAllowlist) != 1 { + t.Fatalf("LoadConfig = %+v, %v", cfg, err) + } + if _, err := LoadConfig(filepath.Join(t.TempDir(), "missing.json")); !errors.Is(err, ErrConfig) { + t.Fatalf("missing file = %v", err) + } +} + +func TestBackendFromConfig(t *testing.T) { + podman, err := BackendFromConfig(Config{Backend: BackendPodman, Image: "img"}) + if err != nil { + t.Fatal(err) + } + if podman.Name != BackendPodman || podman.Teardown == nil { + t.Fatalf("podman = %+v", podman) + } + if l, ok := podman.Launcher.(sandbox.PodmanLauncher); !ok || l.Binary != sandbox.PodmanBinary { + t.Fatalf("launcher = %#v", podman.Launcher) + } + if _, ok := podman.Supports(api.RunNetworkNone); !ok { + t.Fatal("podman cannot do network none") + } + if what, ok := podman.Supports(api.RunNetworkRestricted); ok || !strings.Contains(what, `"restricted"`) { + t.Fatalf("restricted without the flag: %q %v", what, ok) + } + if what, ok := podman.Supports("bridge"); ok || !strings.Contains(what, "unknown network mode") { + t.Fatalf("unknown mode: %q %v", what, ok) + } + restricted, _ := BackendFromConfig(Config{Backend: BackendPodman, Image: "img", NetworkRestricted: true, PodmanBinary: "/opt/podman"}) + if _, ok := restricted.Supports(api.RunNetworkRestricted); !ok { + t.Fatal("restricted flag ignored") + } + if l := restricted.Launcher.(sandbox.PodmanLauncher); l.Binary != "/opt/podman" { + t.Fatalf("binary override = %q", l.Binary) + } + + job := &config.SandboxJobConfig{WorkspaceClaim: "pvc", TTLSeconds: 60} + job.Tolerations = []config.SandboxJobToleration{{Key: "k", Operator: "Exists", Effect: "NoSchedule"}} + job.Volumes = []config.SandboxJobVolume{{Name: "cache", Claim: "c", MountPath: "/cache", ReadOnly: true}} + kube, err := BackendFromConfig(Config{Backend: BackendJob, Image: "img", Job: job}) + if err != nil { + t.Fatal(err) + } + if kube.Teardown != nil || kube.Name != BackendJob { + t.Fatalf("job backend = %+v", kube) + } + if what, ok := kube.Supports(api.RunNetworkNone); ok || !strings.Contains(what, `cannot enforce network "none"`) { + t.Fatalf("job none without a policy: %q %v", what, ok) + } + opts := kube.Launcher.(*kubejob.Launcher).Options + if opts.WorkspaceClaim != "pvc" || opts.TTLSeconds != 60 || len(opts.Tolerations) != 1 || opts.Tolerations[0].Key != "k" || + len(opts.ExtraVolumes) != 1 || opts.ExtraVolumes[0].MountPath != "/cache" || !opts.ExtraVolumes[0].ReadOnly { + t.Fatalf("job options = %+v", opts) + } + policy, _ := BackendFromConfig(Config{Backend: BackendJob, Image: "img", Job: job, JobNetworkPolicy: true}) + if _, ok := policy.Supports(api.RunNetworkNone); !ok { + t.Fatal("job_network_policy ignored") + } + if _, ok := policy.Supports(api.RunNetworkRestricted); ok { + t.Fatal("job claims restricted") + } + + for name, cfg := range map[string]Config{ + "job without block": {Backend: BackendJob, Image: "img"}, + "job invalid": {Backend: BackendJob, Image: "img", Job: &config.SandboxJobConfig{}}, + "unknown": {Backend: "docker", Image: "img"}, + } { + if _, err := BackendFromConfig(cfg); !errors.Is(err, ErrConfig) { + t.Errorf("%s: err = %v", name, err) + } + } +} + +func TestPodmanTeardownReportsFailures(t *testing.T) { + for _, bin := range []string{"true", "false"} { + if _, err := exec.LookPath(bin); err != nil { + t.Skipf("%s not on PATH", bin) + } + } + if err := podmanTeardown("true")(context.Background(), "crux-run-x"); err != nil { + t.Fatalf("true: %v", err) + } + err := podmanTeardown("false")(context.Background(), "crux-run-x") + if err == nil || !strings.Contains(err.Error(), "podman rm crux-run-x") { + t.Fatalf("false: %v", err) + } +} + +func TestGitProvisionerWithARecordedRunner(t *testing.T) { + dir := t.TempDir() + var calls [][]string + runner := func(_ context.Context, cwd, name string, args ...string) ([]byte, error) { + calls = append(calls, append([]string{filepath.Base(cwd), name}, args...)) + if args[0] == "rev-parse" { + return []byte(testSHA + "\n"), nil + } + return nil, nil + } + wt := api.ContainedRunWorktree{Repo: "https://github.com/hivecommons/hive", Commit: testSHA} + path, err := GitProvisioner{Run: runner}.Provision(context.Background(), dir, wt) + if err != nil || path != filepath.Join(dir, WorkspaceDirName) { + t.Fatalf("Provision = %q, %v", path, err) + } + want := [][]string{ + {filepath.Base(dir), "git", "clone", "--no-checkout", "--", wt.Repo, path}, + {WorkspaceDirName, "git", "checkout", "--detach", testSHA}, + {WorkspaceDirName, "git", "rev-parse", "HEAD"}, + } + if len(calls) != len(want) { + t.Fatalf("calls = %v", calls) + } + for i := range want { + if strings.Join(calls[i], " ") != strings.Join(want[i], " ") { + t.Fatalf("call %d = %v, want %v", i, calls[i], want[i]) + } + } + + failing := func(step string, out string) Runner { + return func(_ context.Context, _, _ string, args ...string) ([]byte, error) { + if args[0] == step { + return []byte(out), errors.New("exit 128") + } + if args[0] == "rev-parse" { + return []byte("deadbeef\n"), nil + } + return nil, nil + } + } + for _, tc := range []struct{ step, want string }{ + {"clone", "git clone: exit 128: not found"}, + {"checkout", "git checkout " + testSHA + ": exit 128: not found"}, + {"rev-parse", "git rev-parse HEAD: exit 128"}, + {"none", "checkout pinned " + testSHA + " but HEAD is deadbeef"}, + } { + if _, err := (GitProvisioner{Run: failing(tc.step, "not found")}).Provision(context.Background(), t.TempDir(), wt); err == nil || err.Error() != tc.want { + t.Errorf("%s: err = %v, want %q", tc.step, err, tc.want) + } + } + if _, err := (GitProvisioner{Run: runner}).Provision(context.Background(), filepath.Join(writeTempFile(t), "sub"), wt); err == nil { + t.Fatal("mkdir under a file succeeded") + } +} + +func TestGitProvisionerClonesALocalRepository(t *testing.T) { + if _, err := exec.LookPath("git"); err != nil { + t.Skip("git not on PATH") + } + src := t.TempDir() + git := func(args ...string) string { + t.Helper() + cmd := exec.Command("git", args...) + cmd.Dir = src + cmd.Env = append(os.Environ(), "GIT_AUTHOR_NAME=t", "GIT_AUTHOR_EMAIL=t@x", "GIT_COMMITTER_NAME=t", "GIT_COMMITTER_EMAIL=t@x") + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("git %v: %v: %s", args, err, out) + } + return strings.TrimSpace(string(out)) + } + git("init", "-q", "-b", "main") + if err := os.WriteFile(filepath.Join(src, "f"), []byte("1"), 0o644); err != nil { + t.Fatal(err) + } + git("add", "f") + git("commit", "-q", "-m", "one") + sha := git("rev-parse", "HEAD") + + dir := t.TempDir() + path, err := GitProvisioner{}.Provision(context.Background(), dir, api.ContainedRunWorktree{Repo: src, Commit: sha}) + if err != nil { + t.Fatal(err) + } + if b, err := os.ReadFile(filepath.Join(path, "f")); err != nil || string(b) != "1" { + t.Fatalf("checked-out file = %q, %v", b, err) + } + if _, err := (GitProvisioner{}).Provision(context.Background(), t.TempDir(), api.ContainedRunWorktree{Repo: src, Commit: testSHA}); err == nil || !strings.Contains(err.Error(), "git checkout") { + t.Fatalf("unknown commit = %v", err) + } +} + +func writeTempFile(t *testing.T) string { + t.Helper() + path := filepath.Join(t.TempDir(), "file") + if err := os.WriteFile(path, nil, 0o600); err != nil { + t.Fatal(err) + } + return path +} diff --git a/src/crux/runs/provision.go b/src/crux/runs/provision.go new file mode 100644 index 0000000000..9c8780a8a7 --- /dev/null +++ b/src/crux/runs/provision.go @@ -0,0 +1,78 @@ +package runs + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + + "github.com/hivecommons/hive/crux/api" + "github.com/hivecommons/hive/crux/sandbox" +) + +// gitEnvAllowlist is what the clone may see of the service's environment. +var gitEnvAllowlist = []string{"PATH", "HOME", "LANG", "LC_ALL", "TMPDIR", "SSL_CERT_FILE", "SSL_CERT_DIR"} + +// WorkspaceDirName is the workspace subdirectory of a run's directory when +// the ad-hoc worktree form provisions one. +const WorkspaceDirName = "workspace" + +// Provisioner materialises the ad-hoc worktree form under dir and returns +// the workspace path. It is the stand-in for the checkout service (S1): +// when that lands, the service delegates to it and worktrees get an owner, +// a size cap and a TTL. +type Provisioner interface { + Provision(ctx context.Context, dir string, wt api.ContainedRunWorktree) (string, error) +} + +// Runner executes one command in dir and returns its combined output; the +// default runs it, tests substitute a script. +type Runner func(ctx context.Context, dir, name string, args ...string) ([]byte, error) + +// GitProvisioner clones the repository with git and pins the commit. +type GitProvisioner struct { + Run Runner +} + +// Provision clones wt.Repo into dir/workspace with every ref, detaches at +// wt.Commit and verifies HEAD is that commit. No credential reaches git: +// the environment is the service's and the repository must be public. +func (p GitProvisioner) Provision(ctx context.Context, dir string, wt api.ContainedRunWorktree) (string, error) { + path := filepath.Join(dir, WorkspaceDirName) + if err := os.MkdirAll(dir, 0o770); err != nil { + return "", err + } + run := p.Run + if run == nil { + run = execRunner + } + if out, err := run(ctx, dir, "git", "clone", "--no-checkout", "--", wt.Repo, path); err != nil { + return "", fmt.Errorf("git clone: %w: %s", err, strings.TrimSpace(string(out))) + } + if out, err := run(ctx, path, "git", "checkout", "--detach", wt.Commit); err != nil { + _ = os.RemoveAll(path) + return "", fmt.Errorf("git checkout %s: %w: %s", wt.Commit, err, strings.TrimSpace(string(out))) + } + head, err := run(ctx, path, "git", "rev-parse", "HEAD") + if err != nil { + _ = os.RemoveAll(path) + return "", fmt.Errorf("git rev-parse HEAD: %w", err) + } + if got := strings.TrimSpace(string(head)); got != wt.Commit { + _ = os.RemoveAll(path) + return "", fmt.Errorf("checkout pinned %s but HEAD is %s", wt.Commit, got) + } + return path, nil +} + +func execRunner(ctx context.Context, dir, name string, args ...string) ([]byte, error) { + cmd := exec.CommandContext(ctx, name, args...) + cmd.Dir = dir + // Never prompt and never carry a credential: the ad-hoc form is for + // public repositories on the allowlist, so git gets the sandbox's + // credential-free environment plus a no-prompt switch. + cmd.Env = append(sandbox.SanitizedEnv(os.Environ(), gitEnvAllowlist, nil), "GIT_TERMINAL_PROMPT=0") + return cmd.CombinedOutput() +} diff --git a/src/crux/runs/service.go b/src/crux/runs/service.go new file mode 100644 index 0000000000..29db74e01b --- /dev/null +++ b/src/crux/runs/service.go @@ -0,0 +1,494 @@ +package runs + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "regexp" + "strings" + "sync" + "time" + + "github.com/hivecommons/hive/crux/api" + "github.com/hivecommons/hive/crux/logscrub" + "github.com/hivecommons/hive/crux/runstate" + "github.com/hivecommons/hive/crux/sandbox" +) + +// Artifact names inside a run's directory: the full scrubbed output. +const ( + StdoutArtifact = "stdout.txt" + StderrArtifact = "stderr.txt" + // launchNamePrefix names the container/Job after the run so a teardown + // can find it. + launchNamePrefix = "crux-run-" + // ReconcileReason is recorded on runs that were in flight when the + // serving process stopped. + ReconcileReason = "crux serve restarted while the run was in flight; the exit is not knowable" + // exitUnknown is the launcher convention for "exit not read". + exitUnknown = -1 +) + +// Typed errors. The serve routes map each onto an api.Code*. +var ( + // ErrNotConfigured: the server has no backend; records are still readable. + ErrNotConfigured = errors.New("runs: no contained-run backend is configured on this server") + // ErrNotFound: unknown run ID. + ErrNotFound = runstate.ErrNotFound +) + +// RequestError is a malformed request (400). +type RequestError struct{ Msg string } + +func (e *RequestError) Error() string { return e.Msg } + +// ForbiddenError is a workspace the configuration does not allow (403). +type ForbiddenError struct{ Msg string } + +func (e *ForbiddenError) Error() string { return e.Msg } + +// UnsupportedError names a requested feature or limit the selected backend +// cannot enforce (501). The run never starts. +type UnsupportedError struct{ What string } + +func (e *UnsupportedError) Error() string { return e.What } + +var fullSHA = regexp.MustCompile(`^[0-9a-f]{40}$`) + +// detail is the service's payload inside the runstate record. +type detail struct { + Profile string `json:"profile"` + Network string `json:"network"` + Backend string `json:"backend"` + Command []string `json:"command"` + Workspace string `json:"workspace,omitempty"` + ExitCode *int `json:"exit_code,omitempty"` + TimedOut bool `json:"timed_out,omitempty"` + Stdout string `json:"stdout,omitempty"` + Stderr string `json:"stderr,omitempty"` + StdoutTruncatedAt *int64 `json:"stdout_truncated_at,omitempty"` + StderrTruncatedAt *int64 `json:"stderr_truncated_at,omitempty"` +} + +// plan is a validated request. +type plan struct { + worktree *api.ContainedRunWorktree + workspace string + command []string + allowlist []string + network string + timeout time.Duration + maxOutput int64 + profile string +} + +type active struct { + cancel context.CancelFunc + done chan struct{} + cancelled bool +} + +// Service accepts, executes, cancels and reports contained runs. +type Service struct { + Store *runstate.Store + Backend *Backend + Config Config + // Provisioner materialises the ad-hoc worktree form; nil uses git. + Provisioner Provisioner + // Environ supplies the base environment the allowlist filters; nil uses + // os.Environ. + Environ func() []string + // Scrub redacts output before it is persisted; nil uses logscrub. + Scrub func(string) string + + mu sync.Mutex + active map[string]*active +} + +// New returns a Service over store. backend nil means not configured: the +// saved records stay readable and every create answers ErrNotConfigured. +func New(store *runstate.Store, backend *Backend, cfg Config) *Service { + cfg.Defaults() + return &Service{Store: store, Backend: backend, Config: cfg, active: map[string]*active{}} +} + +// Configured reports whether runs can be executed here. +func (s *Service) Configured() bool { return s.Backend != nil } + +// Reconcile marks every run left accepted or running by a previous process +// unfinished. Call it once at serve start, before accepting requests. +func (s *Service) Reconcile() ([]runstate.Record, error) { + return s.Store.Reconcile(ReconcileReason) +} + +// Create validates req, saves the accepted record and starts the run under +// launchCtx (which must outlive the request). created is false when +// req.IdempotencyKey matched an existing run, whose record is returned. +func (s *Service) Create(launchCtx context.Context, owner string, req api.ContainedRunRequest) (api.ContainedRun, bool, error) { + if !s.Configured() { + return api.ContainedRun{}, false, ErrNotConfigured + } + p, err := s.validate(req) + if err != nil { + return api.ContainedRun{}, false, err + } + d := detail{Profile: p.profile, Network: p.network, Backend: s.Backend.Name, Command: p.command, Workspace: p.workspace} + raw, err := json.Marshal(d) + if err != nil { + return api.ContainedRun{}, false, err + } + rec, created, err := s.Store.Create(runstate.Record{ + Kind: runstate.KindRun, + Owner: owner, + IdempotencyKey: runstate.DeriveIdempotencyKey(owner, req.IdempotencyKey), + Detail: raw, + }) + if err != nil { + return api.ContainedRun{}, false, err + } + if !created { + return toWire(rec), false, nil + } + ctx, cancel := context.WithCancel(launchCtx) + a := &active{cancel: cancel, done: make(chan struct{})} + s.mu.Lock() + s.active[rec.ID] = a + s.mu.Unlock() + go s.execute(ctx, rec.ID, p, a) + return toWire(rec), true, nil +} + +// Get returns the saved record whatever the process state. +func (s *Service) Get(id string) (api.ContainedRun, error) { + rec, err := s.Store.Get(id) + if err != nil { + return api.ContainedRun{}, err + } + return toWire(rec), nil +} + +// Wait blocks until the run reaches a terminal status or ctx ends, then +// returns the saved record. +func (s *Service) Wait(ctx context.Context, id string) (api.ContainedRun, error) { + s.mu.Lock() + a := s.active[id] + s.mu.Unlock() + if a != nil { + select { + case <-a.done: + case <-ctx.Done(): + return api.ContainedRun{}, ctx.Err() + } + } + return s.Get(id) +} + +// Cancel stops a run in flight and waits for its record to settle. A run +// that already finished is returned unchanged with AlreadyFinished set. +func (s *Service) Cancel(ctx context.Context, id string) (api.ContainedRun, error) { + rec, err := s.Store.Get(id) + if err != nil { + return api.ContainedRun{}, err + } + if runstate.Terminal(rec.Status) { + out := toWire(rec) + out.AlreadyFinished = true + return out, nil + } + s.mu.Lock() + a := s.active[id] + if a != nil { + a.cancelled = true + a.cancel() + } + s.mu.Unlock() + if a == nil { + // Accepted or running with no goroutine: not possible after + // Reconcile ran at start; report the record as it is. + return toWire(rec), nil + } + return s.Wait(ctx, id) +} + +func (s *Service) validate(req api.ContainedRunRequest) (plan, error) { + var p plan + if len(req.Command) == 0 || strings.TrimSpace(req.Command[0]) == "" { + return p, &RequestError{"command is required"} + } + p.command = req.Command + switch p.profile = req.Profile; p.profile { + case "": + p.profile = api.RunProfileExperiment + case api.RunProfileExperiment: + case api.RunProfileAgent: + return p, &UnsupportedError{fmt.Sprintf("profile %q is not available yet (schema-checked agent steps, S6)", req.Profile)} + default: + return p, &RequestError{fmt.Sprintf("profile must be %q or %q", api.RunProfileExperiment, api.RunProfileAgent)} + } + switch p.network = req.Network; p.network { + case "": + p.network = api.RunNetworkNone + case api.RunNetworkNone, api.RunNetworkRestricted: + default: + return p, &RequestError{fmt.Sprintf("network must be %q or %q", api.RunNetworkNone, api.RunNetworkRestricted)} + } + if what, ok := s.Backend.Supports(p.network); !ok { + return p, &UnsupportedError{what} + } + for _, name := range req.EnvAllowlist { + if sandbox.IsCredentialName(name) { + return p, &RequestError{fmt.Sprintf("env_allowlist names credential %q; the experiment profile carries no credentials", name)} + } + } + p.allowlist = req.EnvAllowlist + p.timeout = time.Duration(s.Config.DefaultTimeoutMs) * time.Millisecond + if req.TimeoutMs > 0 { + if req.TimeoutMs > s.Config.MaxTimeoutMs { + return p, &RequestError{fmt.Sprintf("timeout_ms %d exceeds this server's cap of %d", req.TimeoutMs, s.Config.MaxTimeoutMs)} + } + p.timeout = time.Duration(req.TimeoutMs) * time.Millisecond + } + p.maxOutput = s.Config.DefaultMaxOutputBytes + if req.MaxOutputBytes > 0 { + if req.MaxOutputBytes > s.Config.MaxOutputBytesCap { + return p, &RequestError{fmt.Sprintf("max_output_bytes %d exceeds this server's cap of %d", req.MaxOutputBytes, s.Config.MaxOutputBytesCap)} + } + p.maxOutput = req.MaxOutputBytes + } + return p, s.validateWorkspace(req, &p) +} + +func (s *Service) validateWorkspace(req api.ContainedRunRequest, p *plan) error { + sources := 0 + for _, set := range []bool{req.Checkout != "", req.Worktree != nil, req.WorkspacePath != ""} { + if set { + sources++ + } + } + switch { + case sources == 0: + return &RequestError{"one of worktree or workspace_path is required"} + case sources > 1: + return &RequestError{"checkout, worktree and workspace_path are mutually exclusive"} + case req.Checkout != "": + return &UnsupportedError{"checkout ids need the checkout service (S1), which is not available yet; use worktree {repo, commit} or workspace_path"} + case req.Worktree != nil: + wt := *req.Worktree + if strings.TrimSpace(wt.Repo) == "" { + return &RequestError{"worktree.repo is required"} + } + if !fullSHA.MatchString(wt.Commit) { + return &RequestError{"worktree.commit must be a full 40-hex commit SHA; a tag or branch name cannot pin the run"} + } + if !repoAllowed(wt.Repo, s.Config.RepoAllowlist) { + return &ForbiddenError{fmt.Sprintf("repository %q is not on this server's repo_allowlist", wt.Repo)} + } + p.worktree = &wt + return nil + } + path := filepath.Clean(req.WorkspacePath) + if !filepath.IsAbs(path) { + return &RequestError{"workspace_path must be absolute"} + } + if !pathUnder(path, s.Config.WorkspaceRoots) { + return &ForbiddenError{fmt.Sprintf("workspace_path %q is not under this server's workspace_roots", path)} + } + if st, err := os.Stat(path); err != nil || !st.IsDir() { + return &RequestError{fmt.Sprintf("workspace_path %q is not a directory", path)} + } + p.workspace = path + return nil +} + +// repoAllowed reports whether repo starts with one of the allowlisted +// URL prefixes. +func repoAllowed(repo string, prefixes []string) bool { + for _, p := range prefixes { + if p = strings.TrimSpace(p); p != "" && strings.HasPrefix(repo, p) { + return true + } + } + return false +} + +// pathUnder reports whether path is one of roots or lies beneath one. +func pathUnder(path string, roots []string) bool { + for _, root := range roots { + root = filepath.Clean(strings.TrimSpace(root)) + if root == "" || root == "." { + continue + } + if path == root || strings.HasPrefix(path, strings.TrimSuffix(root, "/")+"/") { + return true + } + } + return false +} + +// execute runs one accepted run to a terminal status. Every exit path +// writes the record; nothing is left accepted or running by this process. +func (s *Service) execute(ctx context.Context, id string, p plan, a *active) { + defer func() { + s.mu.Lock() + delete(s.active, id) + s.mu.Unlock() + close(a.done) + }() + finish := func(status, reason string, fill func(*detail)) { + _, _ = s.Store.Update(id, func(rec *runstate.Record) error { + var d detail + _ = json.Unmarshal(rec.Detail, &d) + if fill != nil { + fill(&d) + } + raw, err := json.Marshal(d) + if err != nil { + return err + } + rec.Status, rec.Reason, rec.Detail = status, reason, raw + return nil + }) + } + workspace := p.workspace + if p.worktree != nil { + prov := s.Provisioner + if prov == nil { + prov = GitProvisioner{} + } + path, err := prov.Provision(ctx, s.Store.Path(id), *p.worktree) + if err != nil { + if ctx.Err() != nil { + finish(runstate.StatusCancelled, "cancelled while provisioning the worktree", nil) + } else { + finish(runstate.StatusFailed, "provisioning the worktree: "+s.scrub(err.Error()), nil) + } + return + } + workspace = path + } + if _, err := s.Store.Update(id, func(rec *runstate.Record) error { + var d detail + _ = json.Unmarshal(rec.Detail, &d) + d.Workspace = workspace + raw, err := json.Marshal(d) + rec.Status, rec.Detail = runstate.StatusRunning, raw + return err + }); err != nil { + finish(runstate.StatusFailed, "recording the start: "+err.Error(), nil) + return + } + + runCtx, cancelTimeout := context.WithTimeout(ctx, p.timeout) + defer cancelTimeout() + name := launchNamePrefix + id + spec := sandbox.LaunchSpec{ + Image: s.Config.Image, + Name: name, + Workspace: workspace, + Command: p.command, + Env: s.allowlistedEnv(p.allowlist), + EnvAllowlist: p.allowlist, + NetworkMode: p.network, + } + res, err := s.Backend.Launcher.Run(runCtx, spec) + // A launcher that returned without error finished the command; a + // cancel or deadline that landed after that does not unmake the result. + cancelled := err != nil && ctx.Err() != nil + timedOut := err != nil && !cancelled && errors.Is(runCtx.Err(), context.DeadlineExceeded) + status, reason, exit := classify(res, err, cancelled, timedOut) + if (cancelled || timedOut) && s.Backend.Teardown != nil { + tctx, cancel := context.WithTimeout(context.Background(), TeardownTimeout) + if terr := s.Backend.Teardown(tctx, name); terr != nil { + reason += "; teardown: " + s.scrub(terr.Error()) + } + cancel() + } + stdout, stdoutCut := s.persistOutput(id, StdoutArtifact, res.Stdout, p.maxOutput) + stderr, stderrCut := s.persistOutput(id, StderrArtifact, res.Stderr, p.maxOutput) + finish(status, reason, func(d *detail) { + d.ExitCode, d.TimedOut = exit, timedOut + d.Stdout, d.StdoutTruncatedAt = stdout, stdoutCut + d.Stderr, d.StderrTruncatedAt = stderr, stderrCut + }) +} + +// classify turns the launcher's answer into a status. The exit code is +// reported only when the launcher read one: a zero Result beside an error +// means the launcher refused before anything ran, and the launchers' +// exitUnknown means the process ended without a readable exit. Neither is +// ever turned into a guessed code. +func classify(res sandbox.Result, err error, cancelled, timedOut bool) (status, reason string, exit *int) { + launched := err == nil || res != (sandbox.Result{}) + if launched && res.ExitCode != exitUnknown { + code := res.ExitCode + exit = &code + } + switch { + case cancelled: + return runstate.StatusCancelled, "cancelled by the caller", exit + case timedOut: + return runstate.StatusFailed, "timed out", exit + case err == nil: + return runstate.StatusCompleted, "", exit + case exit != nil: + // The command ran and its exit was read; the error is the backend's + // own description (a Job that reports Failed on a non-zero exit). + return runstate.StatusCompleted, logscrub.ScrubString(err.Error()), exit + case !launched: + return runstate.StatusFailed, logscrub.ScrubString(err.Error()), nil + } + return runstate.StatusUnfinished, logscrub.ScrubString(err.Error()), nil +} + +// persistOutput scrubs raw, writes the whole of it as an artifact and +// returns the response copy, cut at max with the offset when it was. +func (s *Service) persistOutput(id, name, raw string, max int64) (string, *int64) { + text := s.scrub(raw) + _ = os.WriteFile(filepath.Join(s.Store.Path(id), name), []byte(text), 0o660) + if int64(len(text)) <= max { + return text, nil + } + cut := max + return text[:max], &cut +} + +func (s *Service) scrub(v string) string { + if s.Scrub != nil { + return s.Scrub(v) + } + return logscrub.ScrubString(v) +} + +// allowlistedEnv is the explicit environment the container receives: the +// allowlisted names present in the service's environment, credentials +// excluded by sandbox.SanitizedEnv. The podman launcher forwards only +// explicit entries into the container, so the allowlist alone is not +// enough. +func (s *Service) allowlistedEnv(allowlist []string) map[string]string { + environ := os.Environ + if s.Environ != nil { + environ = s.Environ + } + out := map[string]string{} + for _, entry := range sandbox.SanitizedEnv(environ(), allowlist, nil) { + if k, v, ok := strings.Cut(entry, "="); ok { + out[k] = v + } + } + return out +} + +func toWire(rec runstate.Record) api.ContainedRun { + var d detail + _ = json.Unmarshal(rec.Detail, &d) + return api.ContainedRun{ + ID: rec.ID, Status: rec.Status, Owner: rec.Owner, Reason: rec.Reason, + CreatedAt: rec.CreatedAt, StartedAt: rec.StartedAt, FinishedAt: rec.FinishedAt, + Profile: d.Profile, Network: d.Network, Backend: d.Backend, Command: d.Command, Workdir: d.Workspace, + ExitCode: d.ExitCode, TimedOut: d.TimedOut, Stdout: d.Stdout, Stderr: d.Stderr, + StdoutTruncatedAt: d.StdoutTruncatedAt, StderrTruncatedAt: d.StderrTruncatedAt, + } +} diff --git a/src/crux/runs/service_test.go b/src/crux/runs/service_test.go new file mode 100644 index 0000000000..9f2d6ef609 --- /dev/null +++ b/src/crux/runs/service_test.go @@ -0,0 +1,458 @@ +package runs + +import ( + "context" + "errors" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/hivecommons/hive/crux/api" + "github.com/hivecommons/hive/crux/runstate" + "github.com/hivecommons/hive/crux/sandbox" +) + +// fakeLauncher drives every classify path without a container runtime. +type fakeLauncher struct { + run func(ctx context.Context, spec sandbox.LaunchSpec) (sandbox.Result, error) + specs chan sandbox.LaunchSpec +} + +func newFakeLauncher(run func(ctx context.Context, spec sandbox.LaunchSpec) (sandbox.Result, error)) *fakeLauncher { + return &fakeLauncher{run: run, specs: make(chan sandbox.LaunchSpec, 8)} +} + +func (f *fakeLauncher) Run(ctx context.Context, spec sandbox.LaunchSpec) (sandbox.Result, error) { + f.specs <- spec + return f.run(ctx, spec) +} + +// blocking returns a launcher that waits for ctx and reports its error. +func blocking() *fakeLauncher { + return newFakeLauncher(func(ctx context.Context, _ sandbox.LaunchSpec) (sandbox.Result, error) { + <-ctx.Done() + return sandbox.Result{ExitCode: -1}, ctx.Err() + }) +} + +type fixture struct { + t *testing.T + svc *Service + launcher *fakeLauncher + torn chan string + root string +} + +const testSHA = "0123456789abcdef0123456789abcdef01234567" + +func newFixture(t *testing.T, launcher *fakeLauncher) *fixture { + t.Helper() + workDir := t.TempDir() + store, err := runstate.Open(workDir) + if err != nil { + t.Fatal(err) + } + root := filepath.Join(t.TempDir(), "ws") + if err := os.MkdirAll(filepath.Join(root, "proj"), 0o755); err != nil { + t.Fatal(err) + } + f := &fixture{t: t, launcher: launcher, torn: make(chan string, 8), root: root} + backend := &Backend{ + Name: "fake", + Launcher: launcher, + Capabilities: Capabilities{NetworkNone: true}, + Teardown: func(_ context.Context, name string) error { f.torn <- name; return nil }, + } + cfg := Config{ + Image: "img:1", + RepoAllowlist: []string{"https://github.com/hivecommons/"}, + WorkspaceRoots: []string{root}, + MaxTimeoutMs: time.Minute.Milliseconds(), + } + f.svc = New(store, backend, cfg) + f.svc.Environ = func() []string { return []string{"KEEP=yes", "GITHUB_TOKEN=secret", "DROP=no"} } + return f +} + +func (f *fixture) workspace() string { return filepath.Join(f.root, "proj") } + +func (f *fixture) start(req api.ContainedRunRequest) api.ContainedRun { + f.t.Helper() + run, created, err := f.svc.Create(context.Background(), "operator", req) + if err != nil || !created { + f.t.Fatalf("Create: %v created=%v", err, created) + } + if run.Status != api.RunAccepted || run.ID == "" { + f.t.Fatalf("accepted record = %+v", run) + } + return run +} + +func (f *fixture) wait(id string) api.ContainedRun { + f.t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + run, err := f.svc.Wait(ctx, id) + if err != nil { + f.t.Fatal(err) + } + return run +} + +func (f *fixture) spec() sandbox.LaunchSpec { + f.t.Helper() + select { + case s := <-f.launcher.specs: + return s + case <-time.After(10 * time.Second): + f.t.Fatal("launcher was never called") + } + return sandbox.LaunchSpec{} +} + +func TestCompletedRunCarriesOutputAndSanitizedEnv(t *testing.T) { + f := newFixture(t, newFakeLauncher(func(_ context.Context, _ sandbox.LaunchSpec) (sandbox.Result, error) { + return sandbox.Result{Stdout: "hello\n", Stderr: "warn\n"}, nil + })) + run := f.start(api.ContainedRunRequest{ + WorkspacePath: f.workspace(), Command: []string{"echo", "hello"}, + EnvAllowlist: []string{"KEEP", "MISSING"}, + }) + spec := f.spec() + if spec.Image != "img:1" || spec.Name != "crux-run-"+run.ID || spec.Workspace != f.workspace() || spec.NetworkMode != "none" { + t.Fatalf("spec = %+v", spec) + } + if spec.Env["KEEP"] != "yes" || len(spec.Env) != 1 { + t.Fatalf("env = %v", spec.Env) + } + done := f.wait(run.ID) + if done.Status != api.RunCompleted || done.ExitCode == nil || *done.ExitCode != 0 || done.Stdout != "hello\n" || done.Stderr != "warn\n" { + t.Fatalf("completed = %+v", done) + } + if done.Profile != api.RunProfileExperiment || done.Network != api.RunNetworkNone || done.Backend != "fake" || done.Workdir != f.workspace() { + t.Fatalf("plan not recorded: %+v", done) + } + if done.StartedAt == nil || done.FinishedAt == nil || done.Owner != "operator" { + t.Fatalf("timestamps/owner = %+v", done) + } + if out, _ := os.ReadFile(filepath.Join(f.svc.Store.Path(run.ID), StdoutArtifact)); string(out) != "hello\n" { + t.Fatalf("stdout artifact = %q", out) + } + if got, err := f.svc.Get(run.ID); err != nil || got.Status != api.RunCompleted { + t.Fatalf("Get = %+v, %v", got, err) + } + if len(f.torn) != 0 { + t.Fatal("teardown ran for a completed run") + } +} + +func TestNonZeroExitIsCompletedWithTheCode(t *testing.T) { + f := newFixture(t, newFakeLauncher(func(_ context.Context, _ sandbox.LaunchSpec) (sandbox.Result, error) { + return sandbox.Result{Stderr: "boom", ExitCode: 3}, errors.New("podman: exit status 3") + })) + run := f.wait(f.start(api.ContainedRunRequest{WorkspacePath: f.workspace(), Command: []string{"false"}}).ID) + if run.Status != api.RunCompleted || run.ExitCode == nil || *run.ExitCode != 3 || run.Reason != "podman: exit status 3" { + t.Fatalf("non-zero = %+v", run) + } +} + +func TestLauncherRefusalIsFailedWithoutAnExitCode(t *testing.T) { + f := newFixture(t, newFakeLauncher(func(_ context.Context, _ sandbox.LaunchSpec) (sandbox.Result, error) { + return sandbox.Result{}, errors.New("image pull failed") + })) + run := f.wait(f.start(api.ContainedRunRequest{WorkspacePath: f.workspace(), Command: []string{"x"}}).ID) + if run.Status != api.RunFailed || run.ExitCode != nil || run.Reason != "image pull failed" { + t.Fatalf("refusal = %+v", run) + } +} + +func TestUnknownExitIsUnfinishedNeverGuessed(t *testing.T) { + f := newFixture(t, newFakeLauncher(func(_ context.Context, _ sandbox.LaunchSpec) (sandbox.Result, error) { + return sandbox.Result{Stdout: "partial", ExitCode: -1}, errors.New("pod vanished") + })) + run := f.wait(f.start(api.ContainedRunRequest{WorkspacePath: f.workspace(), Command: []string{"x"}}).ID) + if run.Status != api.RunUnfinished || run.ExitCode != nil || run.Stdout != "partial" || run.Reason != "pod vanished" { + t.Fatalf("unknown exit = %+v", run) + } +} + +func TestTimeoutFailsAndTearsDown(t *testing.T) { + f := newFixture(t, blocking()) + run := f.start(api.ContainedRunRequest{WorkspacePath: f.workspace(), Command: []string{"sleep"}, TimeoutMs: 20}) + done := f.wait(run.ID) + if done.Status != api.RunFailed || !done.TimedOut || done.ExitCode != nil || done.Reason != "timed out" { + t.Fatalf("timeout = %+v", done) + } + if name := <-f.torn; name != "crux-run-"+run.ID { + t.Fatalf("teardown name = %q", name) + } +} + +func TestCancelTearsDownAndLateCancelReportsAlreadyFinished(t *testing.T) { + f := newFixture(t, blocking()) + run := f.start(api.ContainedRunRequest{WorkspacePath: f.workspace(), Command: []string{"sleep"}}) + f.spec() + f.svc.Backend.Teardown = func(context.Context, string) error { return errors.New("rm: token=abc123 gone") } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + cancelled, err := f.svc.Cancel(ctx, run.ID) + if err != nil { + t.Fatal(err) + } + if cancelled.Status != api.RunCancelled || cancelled.AlreadyFinished || cancelled.ExitCode != nil { + t.Fatalf("cancel = %+v", cancelled) + } + if !strings.HasPrefix(cancelled.Reason, "cancelled by the caller; teardown: ") { + t.Fatalf("teardown error not recorded: %q", cancelled.Reason) + } + late, err := f.svc.Cancel(ctx, run.ID) + if err != nil || !late.AlreadyFinished || late.Status != api.RunCancelled { + t.Fatalf("late cancel = %+v, %v", late, err) + } + if _, err := f.svc.Cancel(ctx, "run_missing"); !errors.Is(err, ErrNotFound) { + t.Fatalf("cancel unknown = %v", err) + } + if _, err := f.svc.Get("run_missing"); !errors.Is(err, ErrNotFound) { + t.Fatalf("get unknown = %v", err) + } +} + +func TestCancelWithoutAGoroutineReportsTheRecord(t *testing.T) { + f := newFixture(t, blocking()) + rec, _, err := f.svc.Store.Create(runstate.Record{}) + if err != nil { + t.Fatal(err) + } + run, err := f.svc.Cancel(context.Background(), rec.ID) + if err != nil || run.Status != api.RunAccepted || run.AlreadyFinished { + t.Fatalf("orphan cancel = %+v, %v", run, err) + } + changed, err := f.svc.Reconcile() + if err != nil || len(changed) != 1 || changed[0].Status != runstate.StatusUnfinished || changed[0].Reason != ReconcileReason { + t.Fatalf("Reconcile = %+v, %v", changed, err) + } +} + +func TestWaitHonoursItsContext(t *testing.T) { + f := newFixture(t, blocking()) + run := f.start(api.ContainedRunRequest{WorkspacePath: f.workspace(), Command: []string{"sleep"}}) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := f.svc.Wait(ctx, run.ID); !errors.Is(err, context.Canceled) { + t.Fatalf("Wait = %v", err) + } + f.svc.Cancel(context.Background(), run.ID) +} + +func TestOutputIsScrubbedAndTruncatedWithTheOffsetRecorded(t *testing.T) { + long := strings.Repeat("a", 50) + "secret" + strings.Repeat("b", 50) + f := newFixture(t, newFakeLauncher(func(_ context.Context, _ sandbox.LaunchSpec) (sandbox.Result, error) { + return sandbox.Result{Stdout: long, Stderr: "short"}, nil + })) + f.svc.Scrub = func(v string) string { return strings.ReplaceAll(v, "secret", "[x]") } + run := f.wait(f.start(api.ContainedRunRequest{WorkspacePath: f.workspace(), Command: []string{"x"}, MaxOutputBytes: 10}).ID) + if run.Stdout != strings.Repeat("a", 10) || run.StdoutTruncatedAt == nil || *run.StdoutTruncatedAt != 10 { + t.Fatalf("truncated stdout = %q at %v", run.Stdout, run.StdoutTruncatedAt) + } + if run.Stderr != "short" || run.StderrTruncatedAt != nil { + t.Fatalf("stderr = %q at %v", run.Stderr, run.StderrTruncatedAt) + } + full, err := os.ReadFile(filepath.Join(f.svc.Store.Path(run.ID), StdoutArtifact)) + if err != nil { + t.Fatal(err) + } + if want := strings.ReplaceAll(long, "secret", "[x]"); string(full) != want { + t.Fatalf("artifact = %q", full) + } +} + +func TestDefaultScrubRedactsCredentials(t *testing.T) { + f := newFixture(t, newFakeLauncher(func(_ context.Context, _ sandbox.LaunchSpec) (sandbox.Result, error) { + return sandbox.Result{Stdout: "token ghp_abcdefghijklmnopqrstuvwxyz0123456789 ok"}, nil + })) + run := f.wait(f.start(api.ContainedRunRequest{WorkspacePath: f.workspace(), Command: []string{"x"}}).ID) + if strings.Contains(run.Stdout, "ghp_abcdefghijklmnopqrstuvwxyz0123456789") { + t.Fatalf("token survived scrubbing: %q", run.Stdout) + } +} + +func TestIdempotencyKeyReturnsTheExistingRun(t *testing.T) { + f := newFixture(t, newFakeLauncher(func(_ context.Context, _ sandbox.LaunchSpec) (sandbox.Result, error) { + return sandbox.Result{Stdout: "once"}, nil + })) + req := api.ContainedRunRequest{WorkspacePath: f.workspace(), Command: []string{"x"}, IdempotencyKey: "k1"} + first := f.wait(f.start(req).ID) + again, created, err := f.svc.Create(context.Background(), "operator", req) + if err != nil || created || again.ID != first.ID || again.Status != api.RunCompleted || again.Stdout != "once" { + t.Fatalf("replay = %+v created=%v err=%v", again, created, err) + } + other, created, err := f.svc.Create(context.Background(), "someone-else", req) + if err != nil || !created || other.ID == first.ID { + t.Fatalf("other owner shared the record: %+v created=%v err=%v", other, created, err) + } + f.wait(other.ID) + if len(f.launcher.specs) != 2 { + t.Fatalf("launches = %d, want 2", len(f.launcher.specs)) + } +} + +func TestWorktreeIsProvisionedBeforeLaunch(t *testing.T) { + f := newFixture(t, newFakeLauncher(func(_ context.Context, _ sandbox.LaunchSpec) (sandbox.Result, error) { + return sandbox.Result{}, nil + })) + var got api.ContainedRunWorktree + f.svc.Provisioner = provisionFunc(func(_ context.Context, dir string, wt api.ContainedRunWorktree) (string, error) { + got = wt + return filepath.Join(dir, "workspace"), nil + }) + wt := &api.ContainedRunWorktree{Repo: "https://github.com/hivecommons/hive", Commit: testSHA} + run := f.start(api.ContainedRunRequest{Worktree: wt, Command: []string{"make"}}) + spec := f.spec() + done := f.wait(run.ID) + if got != *wt || spec.Workspace != filepath.Join(f.svc.Store.Path(run.ID), "workspace") || done.Workdir != spec.Workspace { + t.Fatalf("worktree = %+v spec.Workspace=%q done=%+v", got, spec.Workspace, done) + } + if done.Status != api.RunCompleted { + t.Fatalf("status = %s", done.Status) + } +} + +func TestProvisionFailureIsFailedOrCancelled(t *testing.T) { + f := newFixture(t, blocking()) + f.svc.Provisioner = provisionFunc(func(context.Context, string, api.ContainedRunWorktree) (string, error) { + return "", errors.New("clone refused") + }) + wt := &api.ContainedRunWorktree{Repo: "https://github.com/hivecommons/hive", Commit: testSHA} + run := f.wait(f.start(api.ContainedRunRequest{Worktree: wt, Command: []string{"make"}}).ID) + if run.Status != api.RunFailed || run.Reason != "provisioning the worktree: clone refused" { + t.Fatalf("failed provision = %+v", run) + } + + f.svc.Provisioner = provisionFunc(func(ctx context.Context, _ string, _ api.ContainedRunWorktree) (string, error) { + <-ctx.Done() + return "", ctx.Err() + }) + run = f.start(api.ContainedRunRequest{Worktree: wt, Command: []string{"make"}}) + cancelled, err := f.svc.Cancel(context.Background(), run.ID) + if err != nil || cancelled.Status != api.RunCancelled || cancelled.Reason != "cancelled while provisioning the worktree" { + t.Fatalf("cancelled provision = %+v, %v", cancelled, err) + } + if len(f.launcher.specs) != 0 { + t.Fatal("launcher ran without a workspace") + } +} + +type provisionFunc func(ctx context.Context, dir string, wt api.ContainedRunWorktree) (string, error) + +func (p provisionFunc) Provision(ctx context.Context, dir string, wt api.ContainedRunWorktree) (string, error) { + return p(ctx, dir, wt) +} + +func TestValidationErrors(t *testing.T) { + f := newFixture(t, blocking()) + ws := f.workspace() + wt := &api.ContainedRunWorktree{Repo: "https://github.com/hivecommons/hive", Commit: testSHA} + cases := []struct { + name string + req api.ContainedRunRequest + want any + msg string + }{ + {"no command", api.ContainedRunRequest{WorkspacePath: ws}, &RequestError{}, "command is required"}, + {"blank command", api.ContainedRunRequest{WorkspacePath: ws, Command: []string{" "}}, &RequestError{}, "command is required"}, + {"agent profile", api.ContainedRunRequest{WorkspacePath: ws, Command: []string{"x"}, Profile: "agent"}, &UnsupportedError{}, "S6"}, + {"bad profile", api.ContainedRunRequest{WorkspacePath: ws, Command: []string{"x"}, Profile: "root"}, &RequestError{}, "profile must be"}, + {"bad network", api.ContainedRunRequest{WorkspacePath: ws, Command: []string{"x"}, Network: "host"}, &RequestError{}, "network must be"}, + {"restricted unsupported", api.ContainedRunRequest{WorkspacePath: ws, Command: []string{"x"}, Network: "restricted"}, &UnsupportedError{}, `cannot enforce network "restricted"`}, + {"credential env", api.ContainedRunRequest{WorkspacePath: ws, Command: []string{"x"}, EnvAllowlist: []string{"GITHUB_TOKEN"}}, &RequestError{}, "credential"}, + {"timeout cap", api.ContainedRunRequest{WorkspacePath: ws, Command: []string{"x"}, TimeoutMs: time.Hour.Milliseconds()}, &RequestError{}, "timeout_ms"}, + {"output cap", api.ContainedRunRequest{WorkspacePath: ws, Command: []string{"x"}, MaxOutputBytes: DefaultMaxOutputCap + 1}, &RequestError{}, "max_output_bytes"}, + {"no source", api.ContainedRunRequest{Command: []string{"x"}}, &RequestError{}, "one of worktree or workspace_path"}, + {"two sources", api.ContainedRunRequest{Command: []string{"x"}, WorkspacePath: ws, Worktree: wt}, &RequestError{}, "mutually exclusive"}, + {"checkout id", api.ContainedRunRequest{Command: []string{"x"}, Checkout: "chk_1"}, &UnsupportedError{}, "S1"}, + {"worktree repo", api.ContainedRunRequest{Command: []string{"x"}, Worktree: &api.ContainedRunWorktree{Commit: testSHA}}, &RequestError{}, "worktree.repo"}, + {"worktree short sha", api.ContainedRunRequest{Command: []string{"x"}, Worktree: &api.ContainedRunWorktree{Repo: wt.Repo, Commit: "main"}}, &RequestError{}, "40-hex"}, + {"worktree forbidden", api.ContainedRunRequest{Command: []string{"x"}, Worktree: &api.ContainedRunWorktree{Repo: "https://example.com/r", Commit: testSHA}}, &ForbiddenError{}, "repo_allowlist"}, + {"relative workspace", api.ContainedRunRequest{Command: []string{"x"}, WorkspacePath: "proj"}, &RequestError{}, "absolute"}, + {"workspace outside roots", api.ContainedRunRequest{Command: []string{"x"}, WorkspacePath: filepath.Dir(f.root)}, &ForbiddenError{}, "workspace_roots"}, + {"workspace sibling prefix", api.ContainedRunRequest{Command: []string{"x"}, WorkspacePath: f.root + "2"}, &ForbiddenError{}, "workspace_roots"}, + {"workspace missing", api.ContainedRunRequest{Command: []string{"x"}, WorkspacePath: filepath.Join(ws, "nope")}, &RequestError{}, "not a directory"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + _, created, err := f.svc.Create(context.Background(), "operator", tc.req) + if created || err == nil { + t.Fatalf("Create accepted: created=%v err=%v", created, err) + } + var ok bool + switch tc.want.(type) { + case *RequestError: + var e *RequestError + ok = errors.As(err, &e) + case *ForbiddenError: + var e *ForbiddenError + ok = errors.As(err, &e) + case *UnsupportedError: + var e *UnsupportedError + ok = errors.As(err, &e) + } + if !ok || !strings.Contains(err.Error(), tc.msg) { + t.Fatalf("err = %T %v, want %T containing %q", err, err, tc.want, tc.msg) + } + }) + } + if len(f.launcher.specs) != 0 { + t.Fatal("a refused request reached the launcher") + } +} + +func TestNotConfiguredServesRecordsButRefusesCreates(t *testing.T) { + store, err := runstate.Open(t.TempDir()) + if err != nil { + t.Fatal(err) + } + svc := New(store, nil, Config{}) + if svc.Configured() { + t.Fatal("Configured without a backend") + } + if _, _, err := svc.Create(context.Background(), "operator", api.ContainedRunRequest{Command: []string{"x"}}); !errors.Is(err, ErrNotConfigured) { + t.Fatalf("Create = %v", err) + } + rec, _, _ := store.Create(runstate.Record{}) + if run, err := svc.Get(rec.ID); err != nil || run.ID != rec.ID { + t.Fatalf("Get = %+v, %v", run, err) + } +} + +func TestClassifyTable(t *testing.T) { + unknown := sandbox.Result{ExitCode: -1} + boom := errors.New("boom") + cases := []struct { + name string + res sandbox.Result + err error + cancelled bool + timedOut bool + status string + wantExit *int + wantReason string + }{ + {"ok", sandbox.Result{}, nil, false, false, runstate.StatusCompleted, ptr(0), ""}, + {"non-zero", sandbox.Result{ExitCode: 2}, boom, false, false, runstate.StatusCompleted, ptr(2), "boom"}, + {"refused", sandbox.Result{}, boom, false, false, runstate.StatusFailed, nil, "boom"}, + {"unknown", unknown, boom, false, false, runstate.StatusUnfinished, nil, "boom"}, + {"cancelled", unknown, context.Canceled, true, false, runstate.StatusCancelled, nil, "cancelled by the caller"}, + {"cancelled with exit", sandbox.Result{ExitCode: 137}, boom, true, false, runstate.StatusCancelled, ptr(137), "cancelled by the caller"}, + {"timed out", unknown, context.DeadlineExceeded, false, true, runstate.StatusFailed, nil, "timed out"}, + } + for _, tc := range cases { + status, reason, exit := classify(tc.res, tc.err, tc.cancelled, tc.timedOut) + if status != tc.status || reason != tc.wantReason || (exit == nil) != (tc.wantExit == nil) || (exit != nil && *exit != *tc.wantExit) { + t.Errorf("%s: got %s %q %v", tc.name, status, reason, exit) + } + } +} + +func ptr(v int) *int { return &v } diff --git a/src/crux/runstate/runstate.go b/src/crux/runstate/runstate.go new file mode 100644 index 0000000000..97771f2e95 --- /dev/null +++ b/src/crux/runstate/runstate.go @@ -0,0 +1,362 @@ +// Package runstate is the saved-run-results store of the Crux work services +// (S7 in hivecommons/crux#114): a stable, caller-visible ID assigned at +// accept time, an explicit status set, one JSON record per item under +// WorkDir()/runs//record.json and an append-only runs/index.jsonl +// history, mirroring the sandbox-runs conventions in sandbox_run.go. +// +// The store never guesses. A record that was accepted or running when the +// serving process stopped is reconciled to StatusUnfinished on the next +// start (Reconcile); the caller decides whether to re-run. An idempotency +// key supplied by the caller makes a repeated create return the existing +// record instead of a second one. +package runstate + +import ( + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + "sync" + "time" +) + +// Statuses. The set is closed: a driver maps these onto its own meaning; +// crux only ever reports what happened to the step. +const ( + StatusAccepted = "accepted" + StatusRunning = "running" + StatusWaiting = "waiting" + StatusCompleted = "completed" + StatusFailed = "failed" + StatusCancelled = "cancelled" + StatusUnfinished = "unfinished" + StatusUnsupported = "unsupported" +) + +const ( + // DirName is the per-concern directory under the work dir. + DirName = "runs" + // RecordName is the per-item record file inside the item's directory. + RecordName = "record.json" + // IndexName is the append-only status history, one IndexEntry per line. + IndexName = "index.jsonl" + // KindRun is a contained run (S2); S6 steps add their own kind later. + KindRun = "run" + + idRandomBytes = 6 + maxIDLen = 64 + dirMode = 0o770 + fileMode = 0o660 +) + +// ErrNotFound is returned by Get and Update for an unknown ID. +var ErrNotFound = errors.New("runstate: record not found") + +// Record is the saved state of one item. Detail carries the owning service's +// payload (for S2 the command result) and is opaque to this package. +type Record struct { + ID string `json:"id"` + Kind string `json:"kind"` + Status string `json:"status"` + Owner string `json:"owner,omitempty"` + IdempotencyKey string `json:"idempotency_key,omitempty"` + CreatedAt time.Time `json:"created_at"` + StartedAt *time.Time `json:"started_at,omitempty"` + FinishedAt *time.Time `json:"finished_at,omitempty"` + Reason string `json:"reason,omitempty"` + Detail json.RawMessage `json:"detail,omitempty"` +} + +// IndexEntry is one line of the append-only history: every status the item +// passed through, in order. +type IndexEntry struct { + At time.Time `json:"at"` + ID string `json:"id"` + Kind string `json:"kind"` + Status string `json:"status"` + Reason string `json:"reason,omitempty"` +} + +// Terminal reports whether status is final: nothing moves out of it. +func Terminal(status string) bool { + switch status { + case StatusCompleted, StatusFailed, StatusCancelled, StatusUnfinished, StatusUnsupported: + return true + } + return false +} + +// Valid reports whether status is one of the closed set. +func Valid(status string) bool { + switch status { + case StatusAccepted, StatusRunning, StatusWaiting: + return true + } + return Terminal(status) +} + +// Dir is the runs directory under workDir. +func Dir(workDir string) string { return filepath.Join(workDir, DirName) } + +// DeriveIdempotencyKey scopes a caller-supplied key to its owner so two +// callers using the same key cannot read each other's records. The result +// is what Record.IdempotencyKey stores; an empty key derives to "". +func DeriveIdempotencyKey(owner, key string) string { + if strings.TrimSpace(key) == "" { + return "" + } + h := sha256.New() + for _, p := range []string{owner, key} { + h.Write([]byte(p)) + h.Write([]byte{0}) + } + return hex.EncodeToString(h.Sum(nil)) +} + +// Store is the on-disk record store for one work dir. It is safe for +// concurrent use within one process; a work dir is served by one process. +type Store struct { + dir string + + mu sync.Mutex + keys map[string]string // derived idempotency key -> id + // Now and NewID are overridable for tests. + Now func() time.Time + NewID func() string +} + +// Open loads the store at Dir(workDir), creating the directory and +// rebuilding the idempotency index from the records already on disk. +func Open(workDir string) (*Store, error) { + s := &Store{dir: Dir(workDir), keys: map[string]string{}, Now: time.Now, NewID: newID} + if err := os.MkdirAll(s.dir, dirMode); err != nil { + return nil, fmt.Errorf("runstate: %w", err) + } + recs, err := s.List() + if err != nil { + return nil, err + } + for _, r := range recs { + if r.IdempotencyKey != "" { + s.keys[r.IdempotencyKey] = r.ID + } + } + return s, nil +} + +// Path is the directory of one item: its record and the owning service's +// artifacts live there. +func (s *Store) Path(id string) string { return filepath.Join(s.dir, id) } + +// Create saves rec as a new item in StatusAccepted and returns it with its +// ID and timestamps filled in. When rec.IdempotencyKey names an existing +// item, that item's current record is returned with created=false and +// nothing is written. +func (s *Store) Create(rec Record) (Record, bool, error) { + s.mu.Lock() + defer s.mu.Unlock() + if rec.IdempotencyKey != "" { + if id, ok := s.keys[rec.IdempotencyKey]; ok { + existing, err := s.read(id) + return existing, false, err + } + } + if rec.Kind == "" { + rec.Kind = KindRun + } + rec.ID = s.NewID() + rec.Status = StatusAccepted + rec.CreatedAt = s.Now().UTC() + rec.StartedAt, rec.FinishedAt = nil, nil + if err := os.MkdirAll(s.Path(rec.ID), dirMode); err != nil { + return Record{}, false, fmt.Errorf("runstate: %w", err) + } + if err := s.write(rec); err != nil { + return Record{}, false, err + } + if rec.IdempotencyKey != "" { + s.keys[rec.IdempotencyKey] = rec.ID + } + return rec, true, nil +} + +// Get returns the saved record whatever the process state. +func (s *Store) Get(id string) (Record, error) { + s.mu.Lock() + defer s.mu.Unlock() + return s.read(id) +} + +// Update applies fn to the saved record and writes the result. fn may change +// Status (validated against the closed set), Reason and Detail; a terminal +// record is never moved to another status. Timestamps are maintained here: +// StartedAt on the move to running, FinishedAt on the move to a terminal +// status. +func (s *Store) Update(id string, fn func(*Record) error) (Record, error) { + s.mu.Lock() + defer s.mu.Unlock() + rec, err := s.read(id) + if err != nil { + return Record{}, err + } + before := rec.Status + if err := fn(&rec); err != nil { + return Record{}, err + } + rec.ID = id + if !Valid(rec.Status) { + return Record{}, fmt.Errorf("runstate: invalid status %q", rec.Status) + } + if Terminal(before) && rec.Status != before { + return Record{}, fmt.Errorf("runstate: %s is already %s", id, before) + } + if rec.Status != before { + now := s.Now().UTC() + if rec.Status == StatusRunning && rec.StartedAt == nil { + rec.StartedAt = &now + } + if Terminal(rec.Status) && rec.FinishedAt == nil { + rec.FinishedAt = &now + } + } + if err := s.write(rec); err != nil { + return Record{}, err + } + return rec, nil +} + +// List returns every saved record, oldest first. +func (s *Store) List() ([]Record, error) { + entries, err := os.ReadDir(s.dir) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, fmt.Errorf("runstate: %w", err) + } + var out []Record + for _, e := range entries { + if !e.IsDir() { + continue + } + rec, err := s.read(e.Name()) + if err != nil { + if errors.Is(err, ErrNotFound) { + continue // a directory with artifacts but no record yet + } + return nil, err + } + out = append(out, rec) + } + sort.Slice(out, func(i, j int) bool { + if out[i].CreatedAt.Equal(out[j].CreatedAt) { + return out[i].ID < out[j].ID + } + return out[i].CreatedAt.Before(out[j].CreatedAt) + }) + return out, nil +} + +// Reconcile moves every accepted or running record to StatusUnfinished with +// reason and returns the records it changed. It runs at `crux serve` start: +// the process that owned those items is gone, so their exit is not knowable +// and they are never restarted silently. +func (s *Store) Reconcile(reason string) ([]Record, error) { + recs, err := s.List() + if err != nil { + return nil, err + } + var changed []Record + for _, r := range recs { + if r.Status != StatusAccepted && r.Status != StatusRunning { + continue + } + rec, err := s.Update(r.ID, func(rec *Record) error { + rec.Status = StatusUnfinished + rec.Reason = reason + return nil + }) + if err != nil { + return changed, err + } + changed = append(changed, rec) + } + return changed, nil +} + +func (s *Store) read(id string) (Record, error) { + if !validID(id) { + return Record{}, ErrNotFound + } + data, err := os.ReadFile(filepath.Join(s.Path(id), RecordName)) + if err != nil { + if os.IsNotExist(err) { + return Record{}, ErrNotFound + } + return Record{}, fmt.Errorf("runstate: %w", err) + } + var rec Record + if err := json.Unmarshal(data, &rec); err != nil { + return Record{}, fmt.Errorf("runstate: %s: %w", id, err) + } + return rec, nil +} + +// write replaces the record atomically and appends the status to the index. +func (s *Store) write(rec Record) error { + data, err := json.Marshal(rec) + if err != nil { + return fmt.Errorf("runstate: %w", err) + } + path := filepath.Join(s.Path(rec.ID), RecordName) + tmp := path + ".tmp" + if err := os.WriteFile(tmp, data, fileMode); err != nil { + return fmt.Errorf("runstate: %w", err) + } + if err := os.Rename(tmp, path); err != nil { + return fmt.Errorf("runstate: %w", err) + } + line, err := json.Marshal(IndexEntry{At: s.Now().UTC(), ID: rec.ID, Kind: rec.Kind, Status: rec.Status, Reason: rec.Reason}) + if err != nil { + return fmt.Errorf("runstate: %w", err) + } + f, err := os.OpenFile(filepath.Join(s.dir, IndexName), os.O_CREATE|os.O_APPEND|os.O_WRONLY, fileMode) + if err != nil { + return fmt.Errorf("runstate: %w", err) + } + _, werr := f.Write(append(line, '\n')) + if cerr := f.Close(); werr == nil { + werr = cerr + } + if werr != nil { + return fmt.Errorf("runstate: %w", werr) + } + return nil +} + +// validID keeps a caller-supplied ID from escaping the runs directory. +func validID(id string) bool { + if id == "" || len(id) > maxIDLen { + return false + } + for _, r := range id { + if !(r >= 'a' && r <= 'z' || r >= '0' && r <= '9' || r == '_' || r == '-') { + return false + } + } + return true +} + +func newID() string { + var b [idRandomBytes]byte + if _, err := rand.Read(b[:]); err != nil { + panic(fmt.Sprintf("runstate: crypto/rand: %v", err)) + } + return "run_" + hex.EncodeToString(b[:]) +} diff --git a/src/crux/runstate/runstate_test.go b/src/crux/runstate/runstate_test.go new file mode 100644 index 0000000000..7ecdfe7e55 --- /dev/null +++ b/src/crux/runstate/runstate_test.go @@ -0,0 +1,381 @@ +package runstate + +import ( + "bufio" + "encoding/json" + "errors" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +func newStore(t *testing.T) (*Store, string) { + t.Helper() + dir := t.TempDir() + s, err := Open(dir) + if err != nil { + t.Fatal(err) + } + ids := 0 + s.NewID = func() string { ids++; return "run_" + strings.Repeat("0", 11) + string(rune('a'+ids-1)) } + now := time.Date(2026, 11, 1, 12, 0, 0, 0, time.UTC) + s.Now = func() time.Time { now = now.Add(time.Second); return now } + return s, dir +} + +func TestCreateGetAndIndex(t *testing.T) { + s, dir := newStore(t) + rec, created, err := s.Create(Record{Owner: "operator", Detail: json.RawMessage(`{"x":1}`)}) + if err != nil || !created { + t.Fatalf("Create: %v created=%v", err, created) + } + if rec.ID == "" || rec.Status != StatusAccepted || rec.Kind != KindRun || rec.CreatedAt.IsZero() { + t.Fatalf("record not filled in: %+v", rec) + } + got, err := s.Get(rec.ID) + if err != nil { + t.Fatal(err) + } + if got.ID != rec.ID || got.Owner != "operator" || string(got.Detail) != `{"x":1}` { + t.Fatalf("Get = %+v", got) + } + if _, err := os.Stat(filepath.Join(Dir(dir), rec.ID, RecordName)); err != nil { + t.Fatal(err) + } + if _, err := s.Get("nope"); !errors.Is(err, ErrNotFound) { + t.Fatalf("Get missing = %v", err) + } + if _, err := s.Get("../escape"); !errors.Is(err, ErrNotFound) { + t.Fatalf("Get invalid id = %v", err) + } + entries := readIndex(t, dir) + if len(entries) != 1 || entries[0].ID != rec.ID || entries[0].Status != StatusAccepted { + t.Fatalf("index = %+v", entries) + } +} + +func TestUpdateTimestampsAndTerminalGuard(t *testing.T) { + s, dir := newStore(t) + rec, _, _ := s.Create(Record{}) + run, err := s.Update(rec.ID, func(r *Record) error { r.Status = StatusRunning; return nil }) + if err != nil { + t.Fatal(err) + } + if run.StartedAt == nil || run.FinishedAt != nil { + t.Fatalf("running timestamps = %+v", run) + } + done, err := s.Update(rec.ID, func(r *Record) error { + r.Status = StatusCompleted + r.Reason = "ok" + r.ID = "forged" + return nil + }) + if err != nil { + t.Fatal(err) + } + if done.ID != rec.ID || done.FinishedAt == nil || !done.FinishedAt.After(*done.StartedAt) || done.Reason != "ok" { + t.Fatalf("completed = %+v", done) + } + if _, err := s.Update(rec.ID, func(r *Record) error { r.Status = StatusRunning; return nil }); err == nil || !strings.Contains(err.Error(), "already completed") { + t.Fatalf("leaving terminal: %v", err) + } + // Reason may still be amended on a terminal record. + if _, err := s.Update(rec.ID, func(r *Record) error { r.Reason = "ok; late note"; return nil }); err != nil { + t.Fatal(err) + } + if _, err := s.Update(rec.ID, func(r *Record) error { r.Status = "bogus"; return nil }); err == nil || !strings.Contains(err.Error(), "invalid status") { + t.Fatalf("invalid status: %v", err) + } + boom := errors.New("boom") + if _, err := s.Update(rec.ID, func(r *Record) error { return boom }); !errors.Is(err, boom) { + t.Fatalf("fn error: %v", err) + } + if _, err := s.Update("missing", func(r *Record) error { return nil }); !errors.Is(err, ErrNotFound) { + t.Fatalf("missing: %v", err) + } + statuses := []string{} + for _, e := range readIndex(t, dir) { + statuses = append(statuses, e.Status) + } + if want := "accepted running completed completed"; strings.Join(statuses, " ") != want { + t.Fatalf("index statuses = %v", statuses) + } +} + +func TestIdempotencyAcrossReopen(t *testing.T) { + s, dir := newStore(t) + key := DeriveIdempotencyKey("operator", "k1") + if key == "" || key == DeriveIdempotencyKey("other", "k1") || DeriveIdempotencyKey("operator", " ") != "" { + t.Fatalf("DeriveIdempotencyKey scoping broken") + } + first, created, err := s.Create(Record{IdempotencyKey: key}) + if err != nil || !created { + t.Fatal(err, created) + } + if _, err := s.Update(first.ID, func(r *Record) error { r.Status = StatusFailed; return nil }); err != nil { + t.Fatal(err) + } + again, created, err := s.Create(Record{IdempotencyKey: key}) + if err != nil || created || again.ID != first.ID || again.Status != StatusFailed { + t.Fatalf("replay = %+v created=%v err=%v", again, created, err) + } + other, created, _ := s.Create(Record{IdempotencyKey: DeriveIdempotencyKey("operator", "k2")}) + if !created || other.ID == first.ID { + t.Fatalf("different key reused record") + } + reopened, err := Open(dir) + if err != nil { + t.Fatal(err) + } + replay, created, err := reopened.Create(Record{IdempotencyKey: key}) + if err != nil || created || replay.ID != first.ID { + t.Fatalf("after reopen = %+v created=%v err=%v", replay, created, err) + } +} + +func TestListAndReconcile(t *testing.T) { + s, dir := newStore(t) + a, _, _ := s.Create(Record{}) + b, _, _ := s.Create(Record{}) + c, _, _ := s.Create(Record{}) + s.Update(b.ID, func(r *Record) error { r.Status = StatusRunning; return nil }) + s.Update(c.ID, func(r *Record) error { r.Status = StatusCompleted; return nil }) + // A directory with artifacts but no record is skipped, not an error. + if err := os.MkdirAll(filepath.Join(Dir(dir), "run_orphan"), 0o770); err != nil { + t.Fatal(err) + } + list, err := s.List() + if err != nil { + t.Fatal(err) + } + if len(list) != 3 || list[0].ID != a.ID || list[1].ID != b.ID || list[2].ID != c.ID { + t.Fatalf("List = %+v", list) + } + changed, err := s.Reconcile("restarted") + if err != nil { + t.Fatal(err) + } + if len(changed) != 2 || changed[0].ID != a.ID || changed[1].ID != b.ID { + t.Fatalf("Reconcile changed %+v", changed) + } + for _, id := range []string{a.ID, b.ID} { + rec, _ := s.Get(id) + if rec.Status != StatusUnfinished || rec.Reason != "restarted" || rec.FinishedAt == nil { + t.Fatalf("%s = %+v", id, rec) + } + } + if rec, _ := s.Get(c.ID); rec.Status != StatusCompleted { + t.Fatalf("completed record touched: %+v", rec) + } + if again, _ := s.Reconcile("x"); len(again) != 0 { + t.Fatalf("second reconcile changed %+v", again) + } + empty := &Store{dir: filepath.Join(t.TempDir(), "none"), Now: time.Now, NewID: newID} + if list, err := empty.List(); err != nil || list != nil { + t.Fatalf("missing dir List = %v, %v", list, err) + } +} + +func TestCorruptRecordIsAnError(t *testing.T) { + s, dir := newStore(t) + rec, _, _ := s.Create(Record{}) + if err := os.WriteFile(filepath.Join(Dir(dir), rec.ID, RecordName), []byte("{"), 0o660); err != nil { + t.Fatal(err) + } + if _, err := s.Get(rec.ID); err == nil || errors.Is(err, ErrNotFound) { + t.Fatalf("corrupt Get = %v", err) + } + if _, err := s.List(); err == nil { + t.Fatal("corrupt List succeeded") + } + if _, err := Open(dir); err == nil { + t.Fatal("Open over a corrupt record succeeded") + } +} + +func TestOpenFailsOnUnwritableWorkDir(t *testing.T) { + file := filepath.Join(t.TempDir(), "file") + if err := os.WriteFile(file, nil, 0o600); err != nil { + t.Fatal(err) + } + if _, err := Open(file); err == nil { + t.Fatal("Open under a file succeeded") + } +} + +func TestStatusSets(t *testing.T) { + for _, st := range []string{StatusCompleted, StatusFailed, StatusCancelled, StatusUnfinished, StatusUnsupported} { + if !Terminal(st) || !Valid(st) { + t.Errorf("%s should be terminal and valid", st) + } + } + for _, st := range []string{StatusAccepted, StatusRunning, StatusWaiting} { + if Terminal(st) || !Valid(st) { + t.Errorf("%s should be valid and not terminal", st) + } + } + if Valid("") || Valid("done") { + t.Error("unknown statuses accepted") + } + if !validID(newID()) || validID("") || validID(strings.Repeat("a", maxIDLen+1)) || validID("A") { + t.Error("validID") + } +} + +func readIndex(t *testing.T, workDir string) []IndexEntry { + t.Helper() + f, err := os.Open(filepath.Join(Dir(workDir), IndexName)) + if err != nil { + t.Fatal(err) + } + defer f.Close() + var out []IndexEntry + sc := bufio.NewScanner(f) + for sc.Scan() { + var e IndexEntry + if err := json.Unmarshal(sc.Bytes(), &e); err != nil { + t.Fatal(err) + } + out = append(out, e) + } + return out +} + +func TestUpdateRejectsUnknownStatus(t *testing.T) { + s, _ := newStore(t) + rec, _, _ := s.Create(Record{}) + if _, err := s.Update(rec.ID, func(r *Record) error { r.Status = "paused"; return nil }); err == nil || !strings.Contains(err.Error(), "invalid status") { + t.Fatalf("Update with unknown status = %v", err) + } + if got, _ := s.Get(rec.ID); got.Status != StatusAccepted { + t.Fatalf("rejected update was written: %+v", got) + } +} + +func TestInvalidIDsAreNotFound(t *testing.T) { + s, _ := newStore(t) + for _, id := range []string{"Run_ABC", "run/x", "run x", strings.Repeat("a", maxIDLen+1)} { + if _, err := s.Get(id); !errors.Is(err, ErrNotFound) { + t.Errorf("Get(%q) = %v, want ErrNotFound", id, err) + } + } +} + +// Filesystem obstacles in the item's directory surface as errors rather +// than a half-written record: the item dir is a file, the temp file is a +// directory, the record is a non-empty directory, the index is a directory. +func TestCreateReportsWriteObstacles(t *testing.T) { + cases := map[string]func(t *testing.T, dir, id string){ + "item dir is a file": func(t *testing.T, dir, id string) { + if err := os.WriteFile(filepath.Join(Dir(dir), id), nil, 0o600); err != nil { + t.Fatal(err) + } + }, + "temp file is a directory": func(t *testing.T, dir, id string) { + if err := os.MkdirAll(filepath.Join(Dir(dir), id, RecordName+".tmp"), 0o770); err != nil { + t.Fatal(err) + } + }, + "record is a non-empty directory": func(t *testing.T, dir, id string) { + if err := os.MkdirAll(filepath.Join(Dir(dir), id, RecordName, "child"), 0o770); err != nil { + t.Fatal(err) + } + }, + "index is a directory": func(t *testing.T, dir, id string) { + if err := os.MkdirAll(filepath.Join(Dir(dir), IndexName), 0o770); err != nil { + t.Fatal(err) + } + }, + } + for name, obstruct := range cases { + t.Run(name, func(t *testing.T) { + s, dir := newStore(t) + obstruct(t, dir, s.NewID()) + s.NewID = func() string { return "run_" + strings.Repeat("0", 11) + "a" } + if _, _, err := s.Create(Record{}); err == nil { + t.Fatal("Create succeeded through the obstacle") + } + }) + } +} + +func TestUnreadableRecordIsAnErrorNotMissing(t *testing.T) { + s, dir := newStore(t) + rec, _, _ := s.Create(Record{}) + // A record that is a directory cannot be read; it is not ErrNotFound. + if err := os.Remove(filepath.Join(Dir(dir), rec.ID, RecordName)); err != nil { + t.Fatal(err) + } + if err := os.Mkdir(filepath.Join(Dir(dir), rec.ID, RecordName), 0o770); err != nil { + t.Fatal(err) + } + if _, err := s.Get(rec.ID); err == nil || errors.Is(err, ErrNotFound) { + t.Fatalf("Get = %v, want a read error", err) + } + if _, err := s.List(); err == nil || errors.Is(err, ErrNotFound) { + t.Fatalf("List = %v, want a read error", err) + } + if _, err := s.Reconcile("restart"); err == nil { + t.Fatal("Reconcile succeeded over an unreadable record") + } +} + +func TestReconcileReportsWriteFailure(t *testing.T) { + s, dir := newStore(t) + rec, _, _ := s.Create(Record{}) + if err := os.Mkdir(filepath.Join(Dir(dir), rec.ID, RecordName+".tmp"), 0o770); err != nil { + t.Fatal(err) + } + changed, err := s.Reconcile("restart") + if err == nil || len(changed) != 0 { + t.Fatalf("Reconcile = %+v, %v; want a write error", changed, err) + } + if got, _ := s.Get(rec.ID); got.Status != StatusAccepted { + t.Fatalf("record moved despite the failed write: %+v", got) + } +} + +func TestListFailsWhenDirIsAFile(t *testing.T) { + file := filepath.Join(t.TempDir(), "runs") + if err := os.WriteFile(file, nil, 0o600); err != nil { + t.Fatal(err) + } + s := &Store{dir: file, keys: map[string]string{}, Now: time.Now, NewID: newID} + if _, err := s.List(); err == nil || errors.Is(err, ErrNotFound) { + t.Fatalf("List over a file = %v", err) + } +} + +func TestListBreaksCreatedAtTiesByID(t *testing.T) { + s, _ := newStore(t) + fixed := time.Date(2026, 11, 1, 12, 0, 0, 0, time.UTC) + s.Now = func() time.Time { return fixed } + ids := []string{"run_00000000000c", "run_00000000000a", "run_00000000000b"} + next := 0 + s.NewID = func() string { id := ids[next]; next++; return id } + for range ids { + if _, _, err := s.Create(Record{}); err != nil { + t.Fatal(err) + } + } + list, err := s.List() + if err != nil { + t.Fatal(err) + } + var got []string + for _, r := range list { + got = append(got, r.ID) + } + if strings.Join(got, ",") != "run_00000000000a,run_00000000000b,run_00000000000c" { + t.Fatalf("List order = %v", got) + } +} + +func TestNewIDShape(t *testing.T) { + a, b := newID(), newID() + if a == b || !validID(a) || !strings.HasPrefix(a, "run_") { + t.Fatalf("newID = %q, %q", a, b) + } +} diff --git a/src/crux/serve/runs.go b/src/crux/serve/runs.go new file mode 100644 index 0000000000..13933037e6 --- /dev/null +++ b/src/crux/serve/runs.go @@ -0,0 +1,121 @@ +package serve + +import ( + "context" + "errors" + "net/http" + + "github.com/hivecommons/hive/crux/api" + "github.com/hivecommons/hive/crux/runs" +) + +// runsOwner is the principal a contained run is recorded under. +// +// TODO(S0): switch to CallerFrom(r.Context()) once the caller-principal +// change lands; until then every authenticated caller is the operator, +// which is exactly what the bearer-token check already establishes. +func runsOwner(r *http.Request) string { + return "operator" +} + +// registerRuns mounts the contained-run family (RFC crux#114 S2 + S7). +// The routes are always present so a server without a backend answers +// with a clear not_supported instead of a bare 404. +func (s *Server) registerRuns(mux *http.ServeMux) { + mux.HandleFunc("POST "+api.PathRuns, s.handleRunsCreate) + mux.HandleFunc("GET "+api.PathRuns+"/{id}", s.handleRunsGet) + mux.HandleFunc("POST "+api.PathRuns+"/{id}/cancel", s.handleRunsCancel) +} + +// runsReady reports whether the run service exists; when it does not, a +// 501 has been written. A service without a backend still serves saved +// records, so only Create checks Configured separately. +func (s *Server) runsReady(w http.ResponseWriter) bool { + if s.Runs == nil { + s.writeError(w, http.StatusNotImplemented, api.CodeNotSupported, "contained runs are not enabled on this server") + return false + } + return true +} + +func (s *Server) handleRunsCreate(w http.ResponseWriter, r *http.Request) { + if !s.runsReady(w) { + return + } + var req api.ContainedRunRequest + if !s.readJSON(w, r, &req) { + return + } + run, created, err := s.Runs.Create(s.LaunchContext, runsOwner(r), req) + if err != nil { + s.writeRunsError(w, err) + return + } + if !created { + // Idempotency hit: the earlier record, whatever state it is in. + s.writeJSON(w, http.StatusOK, run) + return + } + if req.Detach { + s.writeJSON(w, http.StatusAccepted, run) + return + } + // The run itself is bound to LaunchContext; only the wait is bound to + // the request, so a caller that goes away leaves the run in flight. + run, err = s.Runs.Wait(r.Context(), run.ID) + if err != nil { + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return + } + s.writeRunsError(w, err) + return + } + s.writeJSON(w, http.StatusOK, run) +} + +func (s *Server) handleRunsGet(w http.ResponseWriter, r *http.Request) { + if !s.runsReady(w) { + return + } + run, err := s.Runs.Get(r.PathValue("id")) + if err != nil { + s.writeRunsError(w, err) + return + } + s.writeJSON(w, http.StatusOK, run) +} + +func (s *Server) handleRunsCancel(w http.ResponseWriter, r *http.Request) { + if !s.runsReady(w) { + return + } + run, err := s.Runs.Cancel(r.Context(), r.PathValue("id")) + if err != nil { + s.writeRunsError(w, err) + return + } + s.writeJSON(w, http.StatusOK, run) +} + +// writeRunsError maps the run service's typed errors onto wire codes. The +// refusals keep their own codes so a caller can tell "you asked for +// something wrong" from "this backend cannot enforce that". +func (s *Server) writeRunsError(w http.ResponseWriter, err error) { + var reqErr *runs.RequestError + var forbidden *runs.ForbiddenError + var unsupported *runs.UnsupportedError + switch { + case errors.Is(err, runs.ErrNotConfigured): + s.writeError(w, http.StatusNotImplemented, api.CodeNotSupported, err.Error()) + case errors.Is(err, runs.ErrNotFound): + s.writeError(w, http.StatusNotFound, api.CodeNotFound, "run not found") + case errors.As(err, &reqErr): + s.writeError(w, http.StatusBadRequest, api.CodeBadRequest, err.Error()) + case errors.As(err, &forbidden): + s.writeError(w, http.StatusForbidden, api.CodeForbidden, err.Error()) + case errors.As(err, &unsupported): + s.writeError(w, http.StatusNotImplemented, api.CodeUnsupported, err.Error()) + default: + s.writeError(w, http.StatusInternalServerError, api.CodeRuntime, err.Error()) + } +} diff --git a/src/crux/serve/runs_test.go b/src/crux/serve/runs_test.go new file mode 100644 index 0000000000..bedfb47ed2 --- /dev/null +++ b/src/crux/serve/runs_test.go @@ -0,0 +1,131 @@ +package serve + +import ( + "context" + "net/http" + "os" + "path/filepath" + "testing" + "time" + + "github.com/hivecommons/hive/crux/api" + "github.com/hivecommons/hive/crux/runs" + "github.com/hivecommons/hive/crux/runstate" + "github.com/hivecommons/hive/crux/sandbox" +) + +type runsLauncher func(ctx context.Context, spec sandbox.LaunchSpec) (sandbox.Result, error) + +func (l runsLauncher) Run(ctx context.Context, spec sandbox.LaunchSpec) (sandbox.Result, error) { + return l(ctx, spec) +} + +// newRunsServer wires a run service with a fake launcher onto a family +// server; the workspace root holds one directory, "proj". +func newRunsServer(t *testing.T, launcher runsLauncher) (*famServer, string) { + t.Helper() + f := newFamServer(t) + store, err := runstate.Open(t.TempDir()) + if err != nil { + t.Fatal(err) + } + root := t.TempDir() + ws := filepath.Join(root, "proj") + if err := os.MkdirAll(ws, 0o755); err != nil { + t.Fatal(err) + } + backend := &runs.Backend{Name: "fake", Launcher: launcher, Capabilities: runs.Capabilities{NetworkNone: true}} + f.srv.Runs = runs.New(store, backend, runs.Config{Image: "img", WorkspaceRoots: []string{root}}) + return f, ws +} + +func TestRunsRoutesAnswerNotSupportedWithoutAService(t *testing.T) { + f := newFamServer(t) + f.expectCode(http.MethodPost, api.PathRuns, api.ContainedRunRequest{Command: []string{"x"}}, http.StatusNotImplemented, api.CodeNotSupported) + f.expectCode(http.MethodGet, api.RunPath("run_x"), nil, http.StatusNotImplemented, api.CodeNotSupported) + f.expectCode(http.MethodPost, api.RunCancelPath("run_x"), nil, http.StatusNotImplemented, api.CodeNotSupported) +} + +func TestRunsWithoutABackendStillServesRecords(t *testing.T) { + f := newFamServer(t) + store, err := runstate.Open(t.TempDir()) + if err != nil { + t.Fatal(err) + } + f.srv.Runs = runs.New(store, nil, runs.Config{}) + rec, _, err := store.Create(runstate.Record{Owner: "operator"}) + if err != nil { + t.Fatal(err) + } + f.expectCode(http.MethodPost, api.PathRuns, api.ContainedRunRequest{Command: []string{"x"}}, http.StatusNotImplemented, api.CodeNotSupported) + var run api.ContainedRun + f.expect(http.MethodGet, api.RunPath(rec.ID), nil, http.StatusOK, &run) + if run.ID != rec.ID || run.Status != api.RunAccepted { + t.Fatalf("GET = %+v", run) + } +} + +func TestRunsCreateWaitsAndReturnsTheRecord(t *testing.T) { + f, ws := newRunsServer(t, func(_ context.Context, spec sandbox.LaunchSpec) (sandbox.Result, error) { + return sandbox.Result{Stdout: "ran " + spec.Command[0]}, nil + }) + req := api.ContainedRunRequest{WorkspacePath: ws, Command: []string{"make", "test"}, IdempotencyKey: "k"} + var run api.ContainedRun + f.expect(http.MethodPost, api.PathRuns, req, http.StatusOK, &run) + if run.Status != api.RunCompleted || run.Stdout != "ran make" || run.ExitCode == nil || *run.ExitCode != 0 || run.Owner != "operator" { + t.Fatalf("POST = %+v", run) + } + var again api.ContainedRun + f.expect(http.MethodPost, api.PathRuns, req, http.StatusOK, &again) + if again.ID != run.ID { + t.Fatalf("idempotent replay returned %s, want %s", again.ID, run.ID) + } + var got api.ContainedRun + f.expect(http.MethodGet, api.RunPath(run.ID), nil, http.StatusOK, &got) + if got.ID != run.ID || got.Status != api.RunCompleted { + t.Fatalf("GET = %+v", got) + } + var late api.ContainedRun + f.expect(http.MethodPost, api.RunCancelPath(run.ID), nil, http.StatusOK, &late) + if !late.AlreadyFinished || late.Status != api.RunCompleted { + t.Fatalf("late cancel = %+v", late) + } +} + +func TestRunsDetachAndCancel(t *testing.T) { + started := make(chan struct{}, 1) + f, ws := newRunsServer(t, func(ctx context.Context, _ sandbox.LaunchSpec) (sandbox.Result, error) { + started <- struct{}{} + <-ctx.Done() + return sandbox.Result{ExitCode: -1}, ctx.Err() + }) + var run api.ContainedRun + f.expect(http.MethodPost, api.PathRuns, api.ContainedRunRequest{WorkspacePath: ws, Command: []string{"sleep"}, Detach: true}, http.StatusAccepted, &run) + if run.Status != api.RunAccepted || run.ID == "" { + t.Fatalf("detached = %+v", run) + } + select { + case <-started: + case <-time.After(10 * time.Second): + t.Fatal("launcher never started") + } + var cancelled api.ContainedRun + f.expect(http.MethodPost, api.RunCancelPath(run.ID), nil, http.StatusOK, &cancelled) + if cancelled.Status != api.RunCancelled || cancelled.AlreadyFinished || cancelled.ExitCode != nil { + t.Fatalf("cancel = %+v", cancelled) + } +} + +func TestRunsErrorCodes(t *testing.T) { + f, ws := newRunsServer(t, func(context.Context, sandbox.LaunchSpec) (sandbox.Result, error) { + return sandbox.Result{}, nil + }) + f.expectCode(http.MethodPost, api.PathRuns, "{not json", http.StatusBadRequest, api.CodeBadRequest) + f.expectCode(http.MethodPost, api.PathRuns, api.ContainedRunRequest{WorkspacePath: ws}, http.StatusBadRequest, api.CodeBadRequest) + f.expectCode(http.MethodPost, api.PathRuns, api.ContainedRunRequest{WorkspacePath: ws, Command: []string{"x"}, Profile: api.RunProfileAgent}, http.StatusNotImplemented, api.CodeUnsupported) + f.expectCode(http.MethodPost, api.PathRuns, api.ContainedRunRequest{WorkspacePath: ws, Command: []string{"x"}, Network: api.RunNetworkRestricted}, http.StatusNotImplemented, api.CodeUnsupported) + f.expectCode(http.MethodPost, api.PathRuns, api.ContainedRunRequest{Checkout: "chk_1", Command: []string{"x"}}, http.StatusNotImplemented, api.CodeUnsupported) + f.expectCode(http.MethodPost, api.PathRuns, api.ContainedRunRequest{WorkspacePath: "/elsewhere", Command: []string{"x"}}, http.StatusForbidden, api.CodeForbidden) + f.expectCode(http.MethodGet, api.RunPath("run_missing"), nil, http.StatusNotFound, api.CodeNotFound) + f.expectCode(http.MethodPost, api.RunCancelPath("run_missing"), nil, http.StatusNotFound, api.CodeNotFound) +} diff --git a/src/crux/serve/serve.go b/src/crux/serve/serve.go index eed2b429ca..e5a75cfcd8 100644 --- a/src/crux/serve/serve.go +++ b/src/crux/serve/serve.go @@ -38,6 +38,7 @@ import ( "github.com/hivecommons/hive/crux/checkout" "github.com/hivecommons/hive/crux/config" "github.com/hivecommons/hive/crux/lanes" + "github.com/hivecommons/hive/crux/runs" ) const ( @@ -109,6 +110,10 @@ type Server struct { // (api.PathCheckouts); nil answers those routes 501. Checkouts *checkout.Service + // Runs, when set, serves the contained-run family (RFC crux#114 S2 + + // S7); nil answers those routes with not_supported. + Runs *runs.Service + now func() time.Time started time.Time @@ -166,6 +171,7 @@ func (s *Server) Handler() http.Handler { mux.HandleFunc("GET "+api.PathEvents, s.handleEvents) s.registerFamilies(mux) s.registerCheckouts(mux) + s.registerRuns(mux) return s.laneHandler(s.authenticate(mux)) } From eac7f886d692fe35799d67a64d8c6467900d4bea Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Sat, 7 Nov 2026 17:58:14 +0000 Subject: [PATCH 2/7] =?UTF-8?q?=F0=9F=90=9B=20fix(ci):=20re-validate=20pol?= =?UTF-8?q?icy=20checks=20before=20replaying=20a=20native-queue-policy=20v?= =?UTF-8?q?erdict=20(#16988)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(ci): re-validate policy checks before replaying a native-queue-policy verdict Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> * fix(ci): fall through to the full wait when a replayed policy provider is pending replayLastVerdict re-validates POLICY_CHECKS before replaying a prior success, but assertEvidence throws PendingEvidence for a missing or still-running provider and the workflow's replay step only tolerates transient API errors. A cosmetic label flip during a provider re-run therefore failed native-queue-policy outright with no retry. Catch PendingEvidence and return null so the caller runs the normal polling wait; add queued/in-progress/missing provider tests. Hive-Fix-Attempt: 1/6 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: ci-maintainer <280983584+hivecommons-hive[bot]@users.noreply.github.com> * fix(ci): fall through on DCO UNKNOWN during replay; refresh replay docs Address review findings on #16988: - replayLastVerdict now returns null for ProviderUnavailable too, so the full wait re-raises it once and renders the runbook job summary instead of a bare setFailed from the label-flip step. - Re-flow the replayLastVerdict doc comment to ~80 columns. - native-merge-queue.md: replay re-validates the five policy checks on the head; red fails, pending/missing/UNKNOWN falls through. - Test: UNKNOWN DCO on replay falls through. Hive-Fix-Attempt: 2/6 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: ci-maintainer <280983584+hivecommons-hive[bot]@users.noreply.github.com> --------- Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Signed-off-by: ci-maintainer <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/scripts/native-queue-policy.cjs | 23 +++++++++-- .github/scripts/native-queue-policy.test.cjs | 40 +++++++++++++++++++- src/docs/native-merge-queue.md | 15 +++++--- 3 files changed, 67 insertions(+), 11 deletions(-) diff --git a/.github/scripts/native-queue-policy.cjs b/.github/scripts/native-queue-policy.cjs index 5baeb710e0..48b4bc15a9 100644 --- a/.github/scripts/native-queue-policy.cjs +++ b/.github/scripts/native-queue-policy.cjs @@ -237,10 +237,12 @@ function isCosmeticLabelFlip({ eventName, payload }) { // A cosmetic flip skips the evidence wait but must not mint a fresh passing // required check: replay the latest completed `success` run of this check on // the head (rejecting one not from this workflow) and still read live PR -// eligibility. Returns null when there is nothing to replay β€” no completed -// run, or the latest one did not succeed (a failed, cancelled or timed-out -// wait) β€” so the caller falls through to the full evidence wait instead of -// pinning that earlier verdict onto every later stage-label flip. +// eligibility and the POLICY_CHECKS evidence. Returns null when there is +// nothing to replay β€” no completed run, or the latest one did not succeed (a +// failed, cancelled or timed-out wait), or a policy provider is missing, +// still running or reporting UNKNOWN β€” so the caller falls through to the +// full evidence wait instead of pinning that earlier verdict onto every later +// stage-label flip. async function replayLastVerdict({ github, context, currentCheckRunId }) { const repo = context.repo; const event = context.payload.pull_request; @@ -259,6 +261,19 @@ async function replayLastVerdict({ github, context, currentCheckRunId }) { // a run of this workflow file on this event may be replayed. await assertActionsProvenance(github, repo, pr.number, pr.head.sha, prior, POLICY_CHECK_SOURCE, new Map()); if (prior.conclusion !== 'success') return null; + // The prior success is only valid while the providers it validated are + // still green: one re-run red on this head must not be papered over. A + // provider that is missing or still running is not a verdict either way; + // fall through so the caller runs the full polling wait instead of failing. + // A DCO enumeration UNKNOWN also falls through: the full wait re-raises it + // once and renders the runbook job summary, so operators see the same + // surface whichever event fired. + try { + await assertEvidence(github, repo, pr.number, pr.head.sha, POLICY_CHECKS); + } catch (error) { + if (error instanceof PendingEvidence || error instanceof ProviderUnavailable) return null; + throw error; + } return `PR #${pr.number}: replayed ${POLICY_CHECK_NAME} run ${prior.id} (success) on ${pr.head.sha}`; } diff --git a/.github/scripts/native-queue-policy.test.cjs b/.github/scripts/native-queue-policy.test.cjs index 7a08dfec2f..c3d3e50f48 100644 --- a/.github/scripts/native-queue-policy.test.cjs +++ b/.github/scripts/native-queue-policy.test.cjs @@ -409,11 +409,16 @@ function replayFixture(priorRuns, labels = [], workflowOverride = {}) { check_suite: { id: 5000 + r.id }, details_url: `https://github.com/acme/widget/actions/runs/${3000 + r.id}/job/1`, ...r, })); - f.github.rest.actions.getWorkflowRun = async ({ run_id: id }) => ({ data: { + const baseGetWorkflowRun = f.github.rest.actions.getWorkflowRun; + const basePaginate = f.github.paginate; + f.github.rest.actions.getWorkflowRun = async ({ run_id: id }) => (id < 3000 ? baseGetWorkflowRun({ run_id: id }) : { data: { id, check_suite_id: id + 2000, path: '.github/workflows/native-queue-policy.yml', event: 'pull_request_target', head_sha: head, ...workflowOverride, } }); f.github.paginate = async (method, args) => { + // Policy provider evidence comes from the base fixture (all green). + if (method === 'jobs' && args.run_id < 3000) return basePaginate(method, args); + if (method === 'checks' && !args.check_name) return basePaginate(method, args); if (method === 'jobs') { const run = runs.find(r => 3000 + r.id === args.run_id); if (!run) return []; @@ -440,6 +445,39 @@ test('cosmetic flip replays the latest completed success on the same head', asyn assert.equal(f.state.listed[0].ref, sha(11)); }); +test('cosmetic flip does not replay a success when a policy provider re-ran red on the head', async () => { + for (const name of Object.keys(POLICY_CHECKS)) { + const f = replayFixture([{ id: 7, status: 'completed', conclusion: 'success' }]); + f.state.runs.get(sha(11)).find(r => r.name === name).conclusion = 'failure'; + await assert.rejects(replayLastVerdict(f), new RegExp(`${name}: failure`)); + } +}); + +test('cosmetic flip falls through to the full wait while a policy provider is re-running or missing', async () => { + // PendingEvidence is not a verdict: returning null sends the caller to the + // polling wait instead of failing the check with no retry. + for (const name of Object.keys(POLICY_CHECKS)) { + for (const change of [{ status: 'queued', conclusion: null }, { status: 'in_progress', conclusion: null }]) { + const f = replayFixture([{ id: 7, status: 'completed', conclusion: 'success' }]); + Object.assign(f.state.runs.get(sha(11)).find(r => r.name === name), change); + assert.equal(await replayLastVerdict(f), null); + } + const f = replayFixture([{ id: 7, status: 'completed', conclusion: 'success' }]); + const runs = f.state.runs.get(sha(11)); + runs.splice(runs.findIndex(r => r.name === name), 1); + assert.equal(await replayLastVerdict(f), null); + } +}); + +test('cosmetic flip falls through to the full wait when the DCO provider reports UNKNOWN', async () => { + // The full wait re-raises ProviderUnavailable once and renders the runbook + // job summary; failing here would skip that operator-facing surface. + const f = replayFixture([{ id: 7, status: 'completed', conclusion: 'success' }]); + Object.assign(f.state.runs.get(sha(11)).find(r => r.name === 'DCO'), + { conclusion: 'failure', output: { summary: unavailableSummary } }); + assert.equal(await replayLastVerdict(f), null); +}); + test('cosmetic flip has nothing to replay without a prior success and falls through', async () => { // A failed, cancelled (timed-out) or absent prior run must not be pinned // onto every later stage-label flip: null sends the caller to the full wait. diff --git a/src/docs/native-merge-queue.md b/src/docs/native-merge-queue.md index ebd97271be..be526e74c3 100644 --- a/src/docs/native-merge-queue.md +++ b/src/docs/native-merge-queue.md @@ -220,13 +220,16 @@ metadata, not a fabricated last-member PR event. in-progress); the queued run re-reads mutable PR state after the prior run finishes, preserving fail-closed semantics. Labels the policy never reads (`size/*`, `dco-signoff`, stage labels, ...) skip the wait: the job replays - the latest completed `native-queue-policy` `success` on the same head and - still checks the live PR for holds, draft and head moves. When there is no + the latest completed `native-queue-policy` `success` on the same head, + still checks the live PR for holds, draft and head moves, and re-validates + the five policy checks on that head β€” a provider re-run red since the + original verdict fails the job instead of being replayed. When there is no prior success to replay (never evaluated, or the last wait failed or timed - out) or GitHub returns a 5xx, the job falls through to the full wait so a - stale red verdict is never pinned onto later cosmetic flips. `hold`, - `needs-human`, `release-in-progress` and `do-not-merge*` flips always get - the full wait. + out), a policy provider is still pending or missing on the head, the DCO + provider reports UNKNOWN, or GitHub returns a 5xx, the job falls through to + the full wait so a stale verdict is never pinned onto later cosmetic flips. + `hold`, `needs-human`, `release-in-progress` and `do-not-merge*` flips + always get the full wait. - On a group it executes the event's trusted `base_sha` validator. It reads the live queue prefix ending at the event's exact synthetic head, verifies contiguous membership and the base/head commit chain, and checks From 2ecfb719470c4596d47ae52a67669c4af5eb266d Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Sat, 7 Nov 2026 18:11:19 +0000 Subject: [PATCH 3/7] =?UTF-8?q?=F0=9F=90=9B=20fix(kubejob):=20do=20not=20m?= =?UTF-8?q?ount=20the=20SA=20token=20into=20contained=20runs=20and=20never?= =?UTF-8?q?=20guess=20exit=20code=200=20(#17023)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(kubejob): drop SA token mount, harden securityContext, keep unknown exit code honest Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> * fix(kubejob): drop runAsNonRoot β€” image USER is non-numeric Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> * fix(kubejob): refresh ctx per collection step; let verify pass on unknown exit code A cancel that lands during pod log collection left podExitCode on a dead context, so a readable Complete pod reported -1 and TestRunSuccessDeletesJobEvenIfContextExpiresDuringLogCollection failed. Every collection/cleanup request now re-checks ctx and falls back to a fresh request-bounded context. The empty exitCodeUnknown branch (SA9003) is folded into a comment. SandboxVerifyResult.Passed now accepts kubejob.ExitCodeUnknown when the launcher returned no error and did not time out, so a Complete verify Job whose pod is gone no longer withholds the PR β€” matching the publish gate and the PR description. Changelog notes the -1 transcript/dashboard rendering. Hive-Fix-Attempt: 1/6 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --------- Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../fixed-kubejob-sa-token-exit-code.md | 1 + src/crux/agent/sandbox_executor.go | 17 ++++- src/crux/agent/sandbox_kick_log.go | 9 ++- src/crux/agent/sandbox_kick_log_test.go | 37 ++++++++++ src/crux/kubejob/kubejob.go | 70 ++++++++++++------- src/crux/kubejob/kubejob_test.go | 10 ++- 6 files changed, 111 insertions(+), 33 deletions(-) create mode 100644 changelog.d/fixed-kubejob-sa-token-exit-code.md diff --git a/changelog.d/fixed-kubejob-sa-token-exit-code.md b/changelog.d/fixed-kubejob-sa-token-exit-code.md new file mode 100644 index 0000000000..62b6efa971 --- /dev/null +++ b/changelog.d/fixed-kubejob-sa-token-exit-code.md @@ -0,0 +1 @@ +- Sandbox Job pods no longer receive the ServiceAccount token and now run with a restrictive security context, and an unreadable completed pod no longer reports a guessed exit code 0 ([#17021](https://github.com/hivecommons/hive/issues/17021), [#17022](https://github.com/hivecommons/hive/issues/17022)). Previously the kubejob manifest left `automountServiceAccountToken` at its default and set no `securityContext`, so a contained run could read the cluster token; it now sets `automountServiceAccountToken: false`, `allowPrivilegeEscalation: false`, drops all capabilities and uses the `RuntimeDefault` seccomp profile. When a Job completes but its pod can no longer be read, the result keeps the unknown exit code (`-1`) instead of claiming 0; the kick log shows `exit_code: unknown`, the transcript's trailing status line and the dashboard's job terminal show the code as `-1`, and neither the publish gate nor `publish.verify` withholds the PR for it. diff --git a/src/crux/agent/sandbox_executor.go b/src/crux/agent/sandbox_executor.go index cc1eeefe9e..3e5671cf6b 100644 --- a/src/crux/agent/sandbox_executor.go +++ b/src/crux/agent/sandbox_executor.go @@ -17,6 +17,7 @@ import ( "github.com/hivecommons/hive/crux/config" "github.com/hivecommons/hive/crux/effects" "github.com/hivecommons/hive/crux/internal/textutil" + "github.com/hivecommons/hive/crux/kubejob" "github.com/hivecommons/hive/crux/logscrub" "github.com/hivecommons/hive/crux/outputschema" "github.com/hivecommons/hive/crux/pushbroker" @@ -129,8 +130,16 @@ type SandboxVerifyResult struct { TimedOut bool } -// Passed reports whether the command ran to completion with exit 0. -func (v SandboxVerifyResult) Passed() bool { return v.Err == "" && v.ExitCode == 0 && !v.TimedOut } +// Passed reports whether the command ran to completion with exit 0. A +// launcher that returned no error but could not read the exit code back +// (a Complete Job whose pod is already gone) counts as passed too: the +// Job's own success condition is the fact, the missing code is not a failure. +func (v SandboxVerifyResult) Passed() bool { + if v.Err != "" || v.TimedOut { + return false + } + return v.ExitCode == 0 || v.ExitCode == kubejob.ExitCodeUnknown +} type SandboxExecutor struct { Launcher sandbox.Launcher @@ -590,7 +599,9 @@ func sandboxPublishWithheld(res SandboxKickResult) string { } return reason } - if res.Sandbox.ExitCode != 0 { + // A completed Job whose pod could not be read has no exit code; that is + // not a failure, and the guessed 0 it used to carry was not a fact either. + if code := res.Sandbox.ExitCode; code != 0 && code != kubejob.ExitCodeUnknown { return fmt.Sprintf("sandbox exited %d", res.Sandbox.ExitCode) } return "" diff --git a/src/crux/agent/sandbox_kick_log.go b/src/crux/agent/sandbox_kick_log.go index 56aeede587..39833f57f3 100644 --- a/src/crux/agent/sandbox_kick_log.go +++ b/src/crux/agent/sandbox_kick_log.go @@ -4,8 +4,11 @@ import ( "fmt" "os" "path/filepath" + "strconv" "strings" "time" + + "github.com/hivecommons/hive/crux/kubejob" ) const ( @@ -25,7 +28,11 @@ const ( // text from the agent. func sandboxKickLogBody(runtime string, res SandboxKickResult, runErr error) string { var b strings.Builder - fmt.Fprintf(&b, "runtime: %s\nexit_code: %d\ncommits: %d\n", runtime, res.Sandbox.ExitCode, res.CommitCount) + exitCode := strconv.Itoa(res.Sandbox.ExitCode) + if res.Sandbox.ExitCode == kubejob.ExitCodeUnknown { + exitCode = "unknown" + } + fmt.Fprintf(&b, "runtime: %s\nexit_code: %s\ncommits: %d\n", runtime, exitCode, res.CommitCount) if res.Branch != "" { fmt.Fprintf(&b, "branch: %s\n", res.Branch) } diff --git a/src/crux/agent/sandbox_kick_log_test.go b/src/crux/agent/sandbox_kick_log_test.go index da308ac8a7..1ac931e14c 100644 --- a/src/crux/agent/sandbox_kick_log_test.go +++ b/src/crux/agent/sandbox_kick_log_test.go @@ -10,6 +10,7 @@ import ( "github.com/hivecommons/hive/crux" "github.com/hivecommons/hive/crux/config" + "github.com/hivecommons/hive/crux/kubejob" "github.com/hivecommons/hive/crux/outputschema" "github.com/hivecommons/hive/crux/sandbox" ) @@ -146,3 +147,39 @@ func TestBuildSandboxRunCarriesPRWithheld(t *testing.T) { t.Errorf("PRWithheld = %q (len %d), want the bounded reason", run.PRWithheld, len([]rune(run.PRWithheld))) } } + +func TestSandboxKickLogBodyUnknownExitCode(t *testing.T) { + res := SandboxKickResult{Sandbox: sandbox.Result{ExitCode: kubejob.ExitCodeUnknown}} + body := sandboxKickLogBody("job", res, nil) + if !strings.Contains(body, "exit_code: unknown\n") { + t.Errorf("unreadable-pod run must not render a guessed exit code:\n%s", body) + } +} + +func TestSandboxPublishWithheldIgnoresUnknownExitCode(t *testing.T) { + unknown := SandboxKickResult{Sandbox: sandbox.Result{ExitCode: kubejob.ExitCodeUnknown}} + if got := sandboxPublishWithheld(unknown); got != "" { + t.Errorf("completed-without-code must not withhold the PR, got %q", got) + } + failed := SandboxKickResult{Sandbox: sandbox.Result{ExitCode: 3}} + if got := sandboxPublishWithheld(failed); got != "sandbox exited 3" { + t.Errorf("got %q, want sandbox exited 3", got) + } +} + +func TestSandboxVerifyResultPassedUnknownExitCode(t *testing.T) { + // A Complete verify Job whose pod could not be read back carries no exit + // code and no error; that must not withhold the PR. + if !(SandboxVerifyResult{ExitCode: kubejob.ExitCodeUnknown}).Passed() { + t.Error("nil-error verify with unknown exit code must pass") + } + if (SandboxVerifyResult{ExitCode: kubejob.ExitCodeUnknown, Err: "boom"}).Passed() { + t.Error("unknown exit code with an error must not pass") + } + if (SandboxVerifyResult{ExitCode: kubejob.ExitCodeUnknown, TimedOut: true}).Passed() { + t.Error("unknown exit code after a timeout must not pass") + } + if (SandboxVerifyResult{ExitCode: 1}).Passed() { + t.Error("exit 1 must not pass") + } +} diff --git a/src/crux/kubejob/kubejob.go b/src/crux/kubejob/kubejob.go index d930cadfdd..42d4b6a9fd 100644 --- a/src/crux/kubejob/kubejob.go +++ b/src/crux/kubejob/kubejob.go @@ -225,45 +225,50 @@ func (l *Launcher) Run(ctx context.Context, spec sandbox.LaunchSpec) (sandbox.Re done, waitErr := l.wait(ctx, jobPath) // Collect whatever the pod produced BEFORE deciding anything: a timed-out // or failed kick with no transcript is the worst outcome for an operator. - // After a timeout the caller's ctx is already dead, so the collection and - // the cleanup below run on a fresh, request-bounded context. - collectCtx := ctx - if ctx.Err() != nil { - var cancel context.CancelFunc - collectCtx, cancel = context.WithTimeout(context.Background(), requestTimeout) - defer cancel() - } - pod, podErr := l.findPod(collectCtx, ns, name) + // The caller's ctx may be dead already (kick timeout) or die between any + // two of these requests (cancel during log collection), so every step + // re-checks it and falls back to a fresh, request-bounded context. + // Otherwise a readable, Complete pod would report exitCodeUnknown merely + // because ctx was cancelled after the log was read. + var cancels []context.CancelFunc + defer func() { + for _, c := range cancels { + c() + } + }() + live := func() context.Context { + if ctx.Err() == nil { + return ctx + } + fresh, cancel := context.WithTimeout(context.Background(), requestTimeout) + cancels = append(cancels, cancel) + return fresh + } + pod, podErr := l.findPod(live(), ns, name) if podErr == nil && pod != "" { - res.Stdout = l.podLog(collectCtx, ns, pod) - res.ExitCode = l.podExitCode(collectCtx, ns, pod) + res.Stdout = l.podLog(live(), ns, pod) + res.ExitCode = l.podExitCode(live(), ns, pod) } else { res.ExitCode = exitCodeUnknown } if waitErr != nil { // Context ended (kick timeout or cancel). The Job would otherwise // keep running the workload on the accelerator node; stop it. - _ = l.delete(collectCtx, jobPath) + _ = l.delete(live(), jobPath) return res, waitErr } if !done.succeeded { res.Stderr = done.reason return res, fmt.Errorf("kubejob: job %s failed: %s", name, done.reason) } - if res.ExitCode == exitCodeUnknown { - // The Job reports Complete, so the container exited 0 even though the - // pod could not be read back (already GC'd, or RBAC on pods missing). - res.ExitCode = 0 - } - // Collection may have expired the context even on success; use a fresh one - // for cleanup to ensure the finished Job is immediately removed. - deleteCtx := ctx - if ctx.Err() != nil { - var cancel context.CancelFunc - deleteCtx, cancel = context.WithTimeout(context.Background(), requestTimeout) - defer cancel() - } - _ = l.delete(deleteCtx, jobPath) + // When the Job reports Complete but the pod could not be read back + // (already GC'd, or RBAC on pods missing), res.ExitCode stays + // exitCodeUnknown rather than a guessed 0; callers treat a nil error with + // an unknown code as completed-without-code. + // + // Collection may have expired the context even on success; the fresh + // context ensures the finished Job is still removed immediately. + _ = l.delete(live(), jobPath) return res, nil } @@ -454,6 +459,13 @@ func (l *Launcher) manifest(ns, name, subPath string, spec sandbox.LaunchSpec) m } container := map[string]any{ + // The workspace PVC and the agent's home need to stay writable, so the + // root filesystem is deliberately not made read-only. + "securityContext": map[string]any{ + "allowPrivilegeEscalation": false, + "capabilities": map[string]any{"drop": []string{"ALL"}}, + "seccompProfile": map[string]string{"type": "RuntimeDefault"}, + }, "name": containerName, "image": spec.Image, "workingDir": workdir, @@ -471,6 +483,12 @@ func (l *Launcher) manifest(ns, name, subPath string, spec sandbox.LaunchSpec) m "restartPolicy": restartNever, "containers": []map[string]any{container}, "volumes": volumes, + // Nothing in the kick pod calls the Kubernetes API, so the SA token is + // never mounted into a contained run. + "automountServiceAccountToken": false, + "securityContext": map[string]any{ + "seccompProfile": map[string]string{"type": "RuntimeDefault"}, + }, } if len(l.Options.NodeSelector) > 0 { podSpec["nodeSelector"] = l.Options.NodeSelector diff --git a/src/crux/kubejob/kubejob_test.go b/src/crux/kubejob/kubejob_test.go index bd885d22d0..51f0488e07 100644 --- a/src/crux/kubejob/kubejob_test.go +++ b/src/crux/kubejob/kubejob_test.go @@ -219,7 +219,7 @@ func TestRunContextTimeoutDeletesTheJob(t *testing.T) { } } -func TestRunCompleteWithNoPodStillSucceeds(t *testing.T) { +func TestRunCompleteWithNoPodKeepsExitCodeUnknown(t *testing.T) { api := newFakeAPI() api.statuses = []string{statusComplete} api.noPod = true @@ -230,8 +230,8 @@ func TestRunCompleteWithNoPodStillSucceeds(t *testing.T) { if err != nil { t.Fatalf("Run: %v", err) } - if res.ExitCode != 0 || res.Stdout != "" { - t.Errorf("result = %+v, want exit 0 (Complete) and no log", res) + if res.ExitCode != exitCodeUnknown || res.Stdout != "" { + t.Errorf("result = %+v, want unknown exit code (Complete, pod unreadable) and no log", res) } } @@ -359,6 +359,10 @@ func TestManifestShape(t *testing.T) { `"tolerations":[{"effect":"NoSchedule","key":"spyre","operator":"Exists"}]`, `"limits":{"ibm.com/spyre_pf":"2"}`, `"serviceAccountName":"hive"`, + `"automountServiceAccountToken":false`, + `"allowPrivilegeEscalation":false`, + `"capabilities":{"drop":["ALL"]}`, + `"seccompProfile":{"type":"RuntimeDefault"}`, `"envFrom":[{"secretRef":{"name":"artifact-registry"}}]`, `"claimName":"hive-data"`, `"subPath":"agents/sandbox/deps-123"`, From 92cd2af2650e0fc7bdbedb5feebb9f7d7c0f5e8b Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Sat, 7 Nov 2026 13:23:27 -0500 Subject: [PATCH 4/7] =?UTF-8?q?=F0=9F=93=96=20docs:=20proxy=20override=20r?= =?UTF-8?q?eflects=20#16785/#16895=20behaviour=20(#17034)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Since #16785 the proxy's destination guard covers plain-HTTP forwards, uninspected CONNECT tunnels and transparent-TLS passthrough, and HIVE_PROXY_ALLOW_PRIVATE_FORWARD=true lifts it for all three legs. env-vars.md and security-model.md still described the guard and its override as plain-HTTP-only; bring both in line with the code. Signed-off-by: Andrew Anderson --- .../changed-16785-proxy-override-docs.md | 1 + src/docs/env-vars.md | 2 +- src/docs/security-model.md | 28 +++++++++++-------- 3 files changed, 19 insertions(+), 12 deletions(-) create mode 100644 changelog.d/changed-16785-proxy-override-docs.md diff --git a/changelog.d/changed-16785-proxy-override-docs.md b/changelog.d/changed-16785-proxy-override-docs.md new file mode 100644 index 0000000000..3386ec5d5f --- /dev/null +++ b/changelog.d/changed-16785-proxy-override-docs.md @@ -0,0 +1 @@ +- `src/docs/env-vars.md` and `src/docs/security-model.md` now describe `HIVE_PROXY_ALLOW_PRIVATE_FORWARD` as it behaves since [#16785](https://github.com/hivecommons/hive/pull/16785): the destination guard covers plain-HTTP forwards, uninspected CONNECT tunnels and transparent-TLS passthrough, and the override lifts it for all three legs at once. The docs previously described the guard and its override as plain-HTTP-only. diff --git a/src/docs/env-vars.md b/src/docs/env-vars.md index a7836b8463..3abd2bb8fa 100644 --- a/src/docs/env-vars.md +++ b/src/docs/env-vars.md @@ -218,7 +218,7 @@ new value at the same time. | `HIVE_STATIC_DIR` | No | `/opt/hive/proxy/public` | Static asset directory for the Node proxy. | | `HIVE_PROXY_EGRESS_MARK` | No | `0x1112` | Packet mark exempted from the MITM egress redirect. | | `HIVE_PROXY_ADVISORY_OK` | No | `false` | Allows the spoke to start when the forced-proxy egress redirect cannot be installed (no `CAP_NET_ADMIN`/iptables). Enforcement becomes advisory-only β€” agents can bypass the proxy. Also gates whether the Go proxy trusts a self-asserted `Proxy-Authorization` header as agent identity when its UID map is unavailable (N7, #3841) β€” off by default, an unidentified caller is treated as `ADVISORY` (writes blocked) rather than whatever name it claims. See [security-model.md](security-model.md#5-network-and-credentials). | -| `HIVE_PROXY_ALLOW_PRIVATE_FORWARD` | No | `false` | Lifts the plain-HTTP forward destination guard ([#14082](https://github.com/hivecommons/hive/issues/14082)). Off by default: a non-CONNECT request through the proxy is dialled only when the resolved address is globally routable, so an agent cannot use the proxy to reach loopback services beside it, the link-local cloud metadata endpoint (`169.254.169.254`), RFC 1918/CGNAT cluster addresses or multicast, and the refusal is a `403` naming the class. Only the exact string `true` turns it on, for deployments that genuinely egress to a private network over plain HTTP (a LAN mirror on a native install). Loopback refusals stay at `DEBUG` and are not counted as violations, because agents are already handed `NO_PROXY` for loopback (#14076). | +| `HIVE_PROXY_ALLOW_PRIVATE_FORWARD` | No | `false` | Lifts the proxy's destination guard ([#14082](https://github.com/hivecommons/hive/issues/14082), [#16785](https://github.com/hivecommons/hive/pull/16785)). Off by default: a plain-HTTP (non-CONNECT) forward, a CONNECT to a host the proxy does not inspect, and a transparent-TLS passthrough are each dialled only when the resolved address is globally routable, so an agent cannot use the proxy to reach loopback services beside it, the link-local cloud metadata endpoint (`169.254.169.254`), RFC 1918/CGNAT cluster addresses or multicast, and the refusal is a `403` naming the class. Only the exact string `true` turns it on, and it lifts the guard for all three legs at once, for deployments that genuinely egress to a private network through the proxy (a LAN mirror on a native install); the proxy logs a boot-time warning when it is set. Loopback refusals stay at `DEBUG` and are not counted as violations, because agents are already handed `NO_PROXY` for loopback (#14076). | | `HIVE_PROXY_INJECT_GH_AUTH` | No | unset (off on every hive, including hosted) | Exact `true` enables proxy-side App-token injection. **Requires GitHub App auth. Refuses startup (exit 19) with `HIVE_PROXY_ADVISORY_OK=true`.** Other spellings do not enable it. Read [behavior and exceptions](#proxy-credential-injection). | | `HIVE_DEPLOYMENT_RUNTIME` | No | none (config `deployment.runtime` fallback; hub-registered Kubernetes hives are auto-detected) | Declares the deployment runtime (`kubernetes`, `podman-quadlet`, or `docker-compose`) so the dashboard can offer standalone upgrades. Unset/unrecognized means `unknown` and the upgrade button stays hidden. See [dashboard-standalone-upgrades.md](dashboard-standalone-upgrades.md). | | `HIVE_DEPLOYMENT_PODMAN_MODE` | Only when runtime is `podman-quadlet` | none (config `deployment.podman_mode` fallback) | `rootless` (`systemctl --user`) or `rootful` (system manager). Without an explicit mode the Podman/Quadlet upgrade path stays disabled. | diff --git a/src/docs/security-model.md b/src/docs/security-model.md index 51128bc668..63002704d2 100644 --- a/src/docs/security-model.md +++ b/src/docs/security-model.md @@ -196,17 +196,23 @@ process cannot forge. Only `api.github.com` is inspected; other hosts are tunnelled without inspection. See [net-admin-requirement.md](net-admin-requirement.md). -**Plain-HTTP forwards.** The redirect covers port 443 only, so plain HTTP -reaches the proxy through its explicit proxy port. Those forwards are -identified like any other request and dialled only when the resolved address -is globally routable: loopback services beside the proxy, the link-local -metadata endpoint, RFC 1918/CGNAT cluster addresses and multicast are refused -with `403` and counted as violations (loopback excepted β€” agents are handed -`NO_PROXY` for it, so it is logged as the client misconfiguration it is). The -check runs at dial time on the resolved address, so a hostname pointed inward -is refused on the same evidence as a literal IP. Set -`HIVE_PROXY_ALLOW_PRIVATE_FORWARD=true` only where a private-network plain-HTTP -destination is intended. +**Plain-HTTP forwards and uninspected tunnels.** The redirect covers port 443 +only, so plain HTTP reaches the proxy through its explicit proxy port. Those +forwards are identified like any other request and dialled only when the +resolved address is globally routable: loopback services beside the proxy, the +link-local metadata endpoint, RFC 1918/CGNAT cluster addresses and multicast +are refused with `403` and counted as violations (loopback excepted β€” agents +are handed `NO_PROXY` for it, so it is logged as the client misconfiguration it +is). The same policy applies to the tunnels the proxy does not inspect: a +CONNECT to a host other than the inspected API and the transparent-TLS +passthrough for such a host are dialled through the same guarded dialer, so +`CONNECT 169.254.169.254:80` or an SNI naming a cluster address is refused on +the same terms rather than tunnelled from the proxy's own uid. The check runs +at dial time on the resolved address, so a hostname pointed inward is refused +on the same evidence as a literal IP. Set +`HIVE_PROXY_ALLOW_PRIVATE_FORWARD=true` only where a private-network +destination is intended; it lifts the guard for all three legs at once, and the +proxy warns at boot when it is set. **Hub-to-spoke channel.** Heartbeats carry a per-hive bearer key derived from the hub master secret and are compared in constant time. The payload is From 7a92935643b4870e990ec339ca7823e66c4e8b6d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 7 Nov 2026 13:25:39 -0500 Subject: [PATCH 5/7] =?UTF-8?q?=F0=9F=8C=B1=20build(deps):=20bump=20ws=20f?= =?UTF-8?q?rom=208.21.3=20to=208.22.1=20in=20/src/proxy=20(#16567)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🌱 build(deps): bump ws from 8.21.3 to 8.22.1 in /src/proxy Bumps [ws](https://github.com/websockets/ws) from 8.21.3 to 8.22.1. - [Release notes](https://github.com/websockets/ws/releases) - [Commits](https://github.com/websockets/ws/compare/8.21.3...8.22.1) --- updated-dependencies: - dependency-name: ws dependency-version: 8.22.1 dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * changelog: fragment for #16567 Generated by changelog-fragment-guard from the PR title (#5675). Edit the wording freely, or replace this file with the no-changelog label if the change is not user-visible. Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Signed-off-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --------- Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- changelog.d/changed-16567.md | 1 + src/proxy/package-lock.json | 8 ++++---- src/proxy/package.json | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) create mode 100644 changelog.d/changed-16567.md diff --git a/changelog.d/changed-16567.md b/changelog.d/changed-16567.md new file mode 100644 index 0000000000..110a6a8f61 --- /dev/null +++ b/changelog.d/changed-16567.md @@ -0,0 +1 @@ +- build(deps): bump ws from 8.21.3 to 8.22.1 in /src/proxy (#16567) diff --git a/src/proxy/package-lock.json b/src/proxy/package-lock.json index 86be927a62..f28302412e 100644 --- a/src/proxy/package-lock.json +++ b/src/proxy/package-lock.json @@ -12,7 +12,7 @@ "http-proxy-middleware": "^4.2.1" }, "devDependencies": { - "ws": "^8.21.3" + "ws": "^8.22.1" } }, "node_modules/accepts": { @@ -965,9 +965,9 @@ "license": "ISC" }, "node_modules/ws": { - "version": "8.21.3", - "resolved": "https://registry.npmjs.org/ws/-/ws-8.21.3.tgz", - "integrity": "sha512-201TZ/kPWxoPr/OKWjquZR1SWKXcvxdH+e1xrx89b3YbmzLMFCLfnaG1HFIgWzJOEWZ7MvpK++odZufgYR50Rw==", + "version": "8.22.1", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.22.1.tgz", + "integrity": "sha512-oyL6OPYme/CmiFvNuGzeoxCJ4nOrZE2O1FwM9MWM1AsGri6nLXB2pg+lNA8aLeXMSav0J6eIVS1j6vUPzUtK9g==", "dev": true, "license": "MIT", "engines": { diff --git a/src/proxy/package.json b/src/proxy/package.json index 96b1a08725..7b0937a24e 100644 --- a/src/proxy/package.json +++ b/src/proxy/package.json @@ -14,6 +14,6 @@ "http-proxy-middleware": "^4.2.1" }, "devDependencies": { - "ws": "^8.21.3" + "ws": "^8.22.1" } } From 912b2eb502237e73f36542aec9dc28ad96ce5597 Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Sat, 7 Nov 2026 13:44:13 -0500 Subject: [PATCH 6/7] =?UTF-8?q?=F0=9F=93=96=20docs:=20how=20Hive=20injects?= =?UTF-8?q?=20knowledge=20and=20memory=20into=20agents=20(#17042)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Answers the 2026-11-05 community-meeting question nobody could answer: which context reaches an agent and from where. Separates the three layers (Hive knowledge primer, CLI conversation, CLI persistent memory), traces the kick-primer path (Primer.PrimeScoped -> primeKnowledge -> # Relevant Knowledge) with its scoping rules and size caps, explains why repo AGENTS.md is injected by the scheduler rather than read natively, lists which per-agent home directories are fleet-shared, and tabulates what survives a kick, /clear, pod restart or a new agent. In-flight memory isolation work (#17031, #17032, #17033, PR #17039) is linked as known gaps rather than described as current behaviour. Signed-off-by: Andrew Anderson --- src/docs/agent-knowledge-and-memory.md | 215 +++++++++++++++++++++++++ src/docs/agents-md.md | 4 + src/docs/documentation-map.md | 1 + src/docs/knowledge-curator.md | 3 +- src/docs/operator-reference/README.md | 2 +- 5 files changed, 223 insertions(+), 2 deletions(-) create mode 100644 src/docs/agent-knowledge-and-memory.md diff --git a/src/docs/agent-knowledge-and-memory.md b/src/docs/agent-knowledge-and-memory.md new file mode 100644 index 0000000000..5bc6bd08a3 --- /dev/null +++ b/src/docs/agent-knowledge-and-memory.md @@ -0,0 +1,215 @@ +# How Hive injects knowledge and memory into agents + +> Written to answer the 2026-11-05 community-meeting question "how does +> Hive's memory/knowledge injection work today?". It describes `v5` as of +> this page's merge; the [Known gaps](#known-gaps--in-progress) section links +> the work that will change the memory half. + +Hive agents receive context from **three distinct layers**, and only one of +them is Hive's own: + +| Layer | Who owns it | Where it lives | Hive's role | +|---|---|---|---| +| **Knowledge** β€” curated facts about the repos and the fleet | Hive (`pkg/knowledge`) | File vaults under `/data/knowledge/…`, optional HTTP wiki layers | Selects a few facts per kick and **pushes** them into the kick prompt; also serves them for **pull** via API/MCP/`hivectl` | +| **Conversation** β€” the current chat in the CLI pane | The CLI | In the CLI process | Resets it before every kick (`/clear`) | +| **CLI memory** β€” the CLI's own persistent, cross-session notes (Claude Code auto-memory, Codex memories, Gemini `GEMINI.md`, …) | The CLI | Dot-directories under the agent's `HOME` | **None.** Hive neither feeds, reads, filters nor clears it; it only decides which agents share the directory | + +The confusion at the meeting was between layers 2 and 3: "every agent clears +its context before it runs" is true of the conversation and false of CLI +memory. + +## 1. Knowledge: the kick primer (push) + +``` + knowledge.vaults / git_sources / connectors / bead_synthesizer vault + (FileStore per source; HTTP wiki layers for personal/project/org/community) + β”‚ + registerKnowledgeStores (cmd/hive/boot_knowledge_primer.go) + β–Ό + β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ Scheduler.primeKnowledge (pkg/scheduler/kick_priming.go) ───┐ + β”‚ 1. extractKeywords ← titles + labels of the agent's first β”‚ + β”‚ maxIssuesToPrime (=5) queued issues β”‚ + β”‚ 2. scope ← knowledge.agent_scopes[] (or base agent for a replica) β”‚ + β”‚ 3. Primer.PrimeScoped (pkg/knowledge/primer.go) β”‚ + β”‚ search every layer/store β†’ merge by layer precedence β†’ β”‚ + β”‚ FilterByLifecycle (approved only) β†’ scope.Filter β†’ β”‚ + β”‚ Fisher-Rao rerank β†’ type priority β†’ cap at max_facts (25) β†’ β”‚ + β”‚ expandWithGraph (never past the scope) β†’ Context7 hints β”‚ + β”‚ 4. PrimedKnowledge.Render (primer_render.go): "# Relevant Knowledge", β”‚ + β”‚ grouped by type, whole facts dropped past max_bytes (4096), β”‚ + β”‚ "_… truncated: N facts / M bytes omitted (size cap)._" marker β”‚ + β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β”‚ + templates.go: skills section + AGENTS.md section + knowledge section + substituted into the agent's kick template; lane kicks (lane_messages.go) + append the knowledge section for scanner / quality / architect / + β–Ό + tmux: [/clear ⏎] then the kick text ⏎ (crux/agent/manager_kick.go) +``` + +**Where the facts come from.** Everything the primer searches is registered +in `registerKnowledgeStores` (`src/cmd/hive/boot_knowledge_primer.go`): +configured `knowledge.vaults` (personal layer), `knowledge.git_sources` +(layer from config), connector output, the **bead synthesizer vault** +(closed beads from every agent, classified into fact types β€” the main +organic source on a running hive), plus any HTTP wiki layers in +`knowledge.layers`. The repo's own docs are *not* a source unless you index +them through a git source, connector or code map +([knowledge-curator.md](knowledge-curator.md)). + +**How it is scoped.** + +- *By the work, not the repo:* the query is keywords from the titles and + labels of the agent's first five queued issues (`extractKeywords`, + `isNoiseLabel`). There is no automatic per-repo or per-issue filter; a fact + tagged with another repo can surface if the keywords match. +- *By agent:* `knowledge.agent_scopes.` is an allow-list of `layers`, + `repos`, `types`, `tags` (`KnowledgeAgentScope` in + `src/pkg/config/knowledge_config.go`). It is applied inside + `Primer.PrimeScoped` *after* the lifecycle filter and can only narrow; graph + expansion never pulls in an out-of-scope fact. A replica inherits its base + agent's entry (`Config.KnowledgeAgentScope`). No entry means unrestricted. +- *By lifecycle:* kicks carry **approved facts only**. The kick primer is built + with no `IncludeStates` (`newKnowledgePrimer` in `src/cmd/hive/main.go`), so + `draft`, `deprecated` and `superseded` never reach a kick; `agent_scopes. + include_states` only widens API/TOC reads that already asked for those + states ([knowledge-lifecycle.md](knowledge-lifecycle.md)). + +**Size caps** (all drop whole units, never clip, and leave a visible marker): + +| Block | Cap | Config | +|---|---|---| +| `# Relevant Knowledge` | 25 facts, 4096 bytes (floor 512) | `knowledge.primer.max_facts`, `knowledge.primer.max_bytes` | +| Repo `AGENTS.md` | 8192 bytes (floor 2048), whole sections | `project.agents_md_max_bytes` (`agentsmd.DefaultMaxInjectionBytes`) | +| Declared skills | 8192 bytes, whole skills | `maxSkillsInjectionBytes` in `kick_priming.go` (fixed) | + +**Switches.** `knowledge.enabled` gates the primer only β€” the bead synthesizer +still writes facts when it is off, they just reach no kick. The dashboard +knowledge toggle registers/unregisters the primer live +(`knowledgePrimerControl`). `knowledge.enabled: false` or an empty keyword set +means the kick simply has no knowledge section. + +## 2. Knowledge: pull access + +Agents (and external tools) can fetch what the primer did not push: +`GET /api/knowledge/toc` and `/api/knowledge/entry/{id}` (pass `agent=` to +apply that agent's scope), `GET /api/knowledge/effective?agent=` to see what +a given agent is allowed to see and why entries are excluded, the public +`POST /mcp/knowledge` MCP (`knowledge_toc`, `knowledge_get`), `hivectl +knowledge search|list`, and `bd kb search` inside the pod. See +[knowledge-toc.md](knowledge-toc.md) and +[public-knowledge-mcp.md](public-knowledge-mcp.md). + +## 3. AGENTS.md / CLAUDE.md: injected by Hive, not read natively + +Each agent's CLI starts in `/data/agents/` (`crux.DefaultWorkDir`), +not inside a repository checkout, so the CLIs' native instruction-file +discovery (`AGENTS.md` for Codex/Copilot, `CLAUDE.md` for Claude Code, +`GEMINI.md`) finds **nothing from the monitored repo** at launch. Repo +instructions reach agents only through the scheduler: `primeAgentsMd` parses +`//AGENTS.md` for the agent's *primary* repo and +prepends the rendered block to the knowledge section (`templates.go`). Without +`project.checkouts_dir` (or a matching `policies.local_dir`) the block is +empty β€” see [agents-md.md](agents-md.md). Skills declared on the agent are +prepended the same way ([skills.md](skills.md)); the hive-wide house style +precedes both ([house-style.md](house-style.md)). + +Two instruction files *are* read natively: a user-level +`~/.claude/CLAUDE.md` / `~/.gemini/GEMINI.md` if one exists β€” and because +those directories are fleet-shared today (next section), such a file applies +to every Claude or Gemini agent on the hive. + +## 4. CLI memory and conversation: what Hive does and does not touch + +**Conversation.** `clear_on_kick` defaults to `true` (`ClearOnKick`, +`src/pkg/config/config.go`). `deliverKickLocked` in +`src/crux/agent/manager_kick.go` types `/clear` ⏎ before the kick text for +every TUI backend (agy headless starts a new conversation instead). The one +exception is a PR follow-up delivered with `SendResumeKick` +(`pr_session_resume.go`), which deliberately continues the authoring +conversation. `/clear` empties the chat; it does not touch anything on disk. + +**Home layout** (`interactive_home.go`, `manager_env.go`, `manager_homes.go`): + +- Per-UID agents get `HOME=/data/home/agents/` (`AgentHome`, + `interactiveHomePath`) with per-agent `XDG_DATA_HOME`/`XDG_STATE_HOME` under + `~/.local` (`setupAgentXDGDirs`). `/data/home` is the persistent volume. +- `interactiveHomeBridgeDirs` symlinks `.claude`, `.config`, `.codex`, `.bob`, + `.gemini` and `.cache` from every per-agent home to the **same** directories + under `/data/home`, so one login authenticates the fleet. Anything a CLI + keeps *inside* those directories is therefore fleet-shared and persistent. +- Real per-agent directories with one narrow bridge: `~/.copilot` (only + `config.json` shared β€” `setupCopilotHome`, #9444), `~/.local`, + `.claude.json`, `.npm`, `.gitconfig` (#9478). Codex gets + `CODEX_HOME=/data/home/.codex-` with only `auth.json` symlinked to the + shared login (`setupCodexHome`, `codexHomePath`). +- `HIVE_SHARED_AGENT_HOME=1` forces the legacy single shared home for all. +- Inference (API-routed) agents use throwaway homes under + `/tmp/.claude-inference-home-` that vanish with the pod. +- Contributor containers stage only `.credentials.json` and `settings.json` + into the container (`CLAUDE_STAGE_ALLOWLIST` in the `justfile`, #7836); + transcripts, history and memory stay on the host. This is the one place + memory is isolated by construction. + +**No memory handling.** Nothing in Hive sets a CLI's "disable auto-memory" +option (`inferenceSettingsSeed` seeds permissions/onboarding/remote-control +keys only), prunes a `memory/` directory, or clears memory on kick, `/clear` +or restart. What each CLI remembers, it remembers across kicks and pod +restarts, and β€” for the bridged directories β€” across agents. + +### What persists across what + +Columns: a cadence **kick**; an explicit **`/clear`**; a **pod restart**; a +**new agent** added to the same hive. βœ“ = survives / is visible, βœ— = gone / +not visible. + +| | Kick | `/clear` | Pod restart | New agent in same hive | +|---|---|---|---|---| +| **Knowledge** (`# Relevant Knowledge`, AGENTS.md, skills) | re-selected for *this* kick's issues | n/a β€” re-injected next kick | βœ“ vaults on `/data` | βœ“ same stores, filtered by its own `agent_scopes` | +| **Conversation** β€” all CLIs | βœ— (`/clear` first, unless `clear_on_kick: false` or a resume kick) | βœ— | βœ— (CLI relaunched) | βœ— | +| **Claude Code memory** (`~/.claude/projects//memory/`, `~/.claude/CLAUDE.md`, `history.jsonl`) | βœ“ | βœ“ | βœ“ (`/data/home/.claude` is shared & persistent) | βœ“ readable: the tree is fleet-shared; auto-memory is keyed by cwd `/data/agents/` so files are per-agent by path, but `CLAUDE.md`/history are one copy | +| **Codex memory** (`$CODEX_HOME`) | βœ“ | βœ“ | βœ“ (`/data/home/.codex-`) | βœ— per-agent dir; only `auth.json` shared | +| **Copilot CLI memory/sessions** (`~/.copilot`) | βœ“ | βœ“ | βœ“ | βœ— per-agent dir (#9444); only `config.json` shared | +| **Gemini memory** (`~/.gemini/GEMINI.md`, state) | βœ“ | βœ“ | βœ“ | βœ“ whole directory shared | +| **Bob memory/settings** (`~/.bob`) | βœ“ | βœ“ | βœ“ | βœ“ whole directory shared | + +The dashboard's cost accounting reads the shared Claude tree +(`data.claude_sessions_dir`, default `/data/home/.claude/projects`, and +attributes sessions by the `-data-agents-` project slug β€” +`src/pkg/tokens/claude_scanner.go`), which is why that tree is shared on +purpose today. + +## Known gaps / in progress + +Do not treat the following as current behaviour; follow the links. + +- [#17031](https://github.com/hivecommons/hive/issues/17031) β€” Claude/Gemini/Bob + persistent memory is shared across every agent on a hive. Fix in flight: + [PR #17039](https://github.com/hivecommons/hive/pull/17039) makes `.claude`, + `.gemini` and `.bob` real per-agent directories with only credential/settings + entries bridged (the `setupCopilotHome` pattern) and teaches the token + collector to scan `/data/home/agents/*/.claude/projects`. Until it merges, + the table above stands. +- [#17032](https://github.com/hivecommons/hive/issues/17032) β€” disable CLI + auto-memory by default and add a `clear_memory_on_kick` option alongside + `clear_on_kick`. +- [#17033](https://github.com/hivecommons/hive/issues/17033) β€” drain CLI + auto-memory into beads/knowledge at session end instead of a hidden + `MEMORY.md` (revives open question 5 of + [design/knowledge-system.md](design/knowledge-system.md)). +- Repo `AGENTS.md` injection still requires `project.checkouts_dir`; nearest- + file (`ParseNearest`) targeting is deferred ([agents-md.md](agents-md.md)). + +Closed history worth knowing: #7836 (contributor `~/.claude` staging), #9444 +(Copilot session-state isolation), #9478 (per-agent `.gitconfig`), #11201 +(`knowledge.agent_scopes`), #12124/#12129 (primer and AGENTS.md size caps), +#10524 (knowledge TOC, lifecycle, suggest). + +## Related + +[design/knowledge-system.md](design/knowledge-system.md) Β· +[knowledge-curator.md](knowledge-curator.md) Β· +[knowledge-lifecycle.md](knowledge-lifecycle.md) Β· +[knowledge-toc.md](knowledge-toc.md) Β· [agents-md.md](agents-md.md) Β· +[skills.md](skills.md) Β· [agent-configuration.md](agent-configuration.md) Β· +[contributor-relay.md](contributor-relay.md) diff --git a/src/docs/agents-md.md b/src/docs/agents-md.md index 9888e735d6..1875970174 100644 --- a/src/docs/agents-md.md +++ b/src/docs/agents-md.md @@ -236,6 +236,10 @@ every lane honours that statement as a written policy and skips the class. ## What to read next +- [agent-knowledge-and-memory.md](agent-knowledge-and-memory.md) β€” where this + block sits in the whole kick (skills, AGENTS.md, `# Relevant Knowledge`), why + the CLI does not read the repo's `AGENTS.md` natively, and what `/clear` + does and does not reset. - [house-style.md](house-style.md) β€” the short, hive-wide *how to change things* section that precedes this block in every kick; a repo overrides it with a root `HIVE_STYLE.md`, read from the same checkout root. diff --git a/src/docs/documentation-map.md b/src/docs/documentation-map.md index 6bdbafa8ea..d85f92f039 100644 --- a/src/docs/documentation-map.md +++ b/src/docs/documentation-map.md @@ -167,6 +167,7 @@ Documentation for the current Hive line (branch `v5`; the code and docs live und - [Custom dashboard stylesheets](custom-stylesheets.md) β€” operator-supplied CSS for the dashboard and public snapshot. - [Branding a hive](branding.md) β€” persistent per-deployment name, mark, and colours via `/branding/branding.json` and `custom.css` (`HIVE_BRANDING_JSON`/`HIVE_BRANDING_CSS`); distinct from the per-request `?style=` stylesheets above. - [Portable AgentDefinition format](../AGENT-DEFINITION.md) β€” standalone YAML schema for importing/exporting agent definitions. +- [How Hive injects knowledge and memory into agents](agent-knowledge-and-memory.md) β€” the three context layers (Hive knowledge primer, CLI conversation, CLI persistent memory), the kick-primer path and its scoping/caps, why repo `AGENTS.md` is injected rather than read natively, which per-agent home directories are fleet-shared, and what survives a kick, `/clear`, pod restart or a new agent. - [Knowledge connectors](knowledge-connectors.md) β€” **Owner configuration:** connect, validate, and sync external document/wiki sources; includes scope and API reference. - [Knowledge curator](knowledge-curator.md) β€” configure fact extraction, sources, vaults, document imports, code maps, and retention. - [Knowledge confidence scoring](knowledge-confidence.md) β€” interpret confidence badges and score explanations. diff --git a/src/docs/knowledge-curator.md b/src/docs/knowledge-curator.md index 54e2498c74..053f812fa2 100644 --- a/src/docs/knowledge-curator.md +++ b/src/docs/knowledge-curator.md @@ -627,7 +627,8 @@ Maps an agent name to the `layers`, `repos`, `types`, `tags` and are restricted to. A missing agent or an empty field is unrestricted (the default). Unknown layer or lifecycle state names fail config load. See [Per-agent scopes](knowledge-toc.md#per-agent-scopes) for the intersection -rules. +rules, and [agent-knowledge-and-memory.md](agent-knowledge-and-memory.md) for +how the scope is applied on the kick path end to end. ```yaml knowledge: diff --git a/src/docs/operator-reference/README.md b/src/docs/operator-reference/README.md index 0803386188..4528369000 100644 --- a/src/docs/operator-reference/README.md +++ b/src/docs/operator-reference/README.md @@ -301,7 +301,7 @@ Go callers keep reading `cfg.Agents`, `cfg.Governor.Gateways` and so on through | `knowledge.bead_synthesizer.publish` | Off (`enabled: false`). | Publishes synthesized facts into a repository through one PR on `hive/bead-synth-knowledge`: `enabled`, `repo` (empty = `project.primary_repo`), `path` (default `docs/knowledge/`; `.hive/knowledge/` is the alternative; relative, no `..`), `base_branch` (empty = repo default branch), `mode` (`pr` default or `commit`), `cadence` (`per_run` default or `daily`). ACMM L<=4 hives publish via PR only; `commit` is warned about and still opens a PR. See [knowledge-curator.md](../knowledge-curator.md#repository-publish-knowledgebead_synthesizerpublish). | | `variables.security.*` | Deny by default. | `allow_exec`, `allow_http`, and GitHub prompt-source allowlists are honored only from the trusted seed, not dashboard overlays. | | `agents..prompt_source` (size cap) | 96 KiB (`promptsrc.DefaultMaxPromptBytes`); `HIVE_PROMPT_SOURCE_MAX_BYTES` overrides it. | A fetched kick template over the cap is **rejected whole, not truncated** ([#12130](https://github.com/hivecommons/hive/issues/12130)): the kick uses the last successfully fetched copy if there is one, otherwise the ordinary `kick_template` β†’ pack β†’ convention β†’ embedded-default chain, and the scheduler logs `prompt from GitHub exceeds size cap; REJECTED whole (not truncated), falling back` with `repo`, `path`, `size_bytes` and `cap_bytes`. Saving an agent whose source is over the cap fails in the dashboard with the same sizes rather than baking a cut-off policy. A half-policy is a policy with its closing rules missing; no remote policy is the safer failure. Every shipped template fits with >2Γ— headroom (`TestShippedTemplatesFitUnderCap`). | -| `data.claude_sessions_dir` | `/data/home/.claude/projects` | Where the dashboard reads Claude Code session JSONL for per-agent token/cost accounting. Point it at the agents' real session directory if you relocate `HOME`. | +| `data.claude_sessions_dir` | `/data/home/.claude/projects` | Where the dashboard reads Claude Code session JSONL for per-agent token/cost accounting. Point it at the agents' real session directory if you relocate `HOME`. This shared tree is also where Claude Code keeps its persistent auto-memory; see [agent-knowledge-and-memory.md](../agent-knowledge-and-memory.md) for which per-agent home directories are fleet-shared and what `clear_on_kick` does and does not clear. | | `data.copilot_sessions_dir` | `/data/home/.copilot/session-state` | Same, for the Copilot CLI backend's session state. | ### Label-driven approval From 53c7877db27f0326c89b0aec4913d562e199de5f Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Sat, 7 Nov 2026 13:47:29 -0500 Subject: [PATCH 7/7] =?UTF-8?q?=F0=9F=90=9B=20hub:=20fall=20back=20to=20ot?= =?UTF-8?q?her=20pools=20when=20Auto-pick's=20preferred=20pool=20has=20no?= =?UTF-8?q?=20placeholders=20(#17046)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit handleApproveProvision only ever looked in the pool mapped from the request's auth_method, so a public request 409'd when hive-oke had no available placeholder even though lke648397 had 30. Auto-pick now tries the preferred pool first and, for the soft public/default preference, falls back to any other cluster registered in clusters.json that has an available placeholder (most available first, tie-break by cluster id), logging one INFO line. Private requests stay pinned to the GPU pool, the GPU pool is never used for public requests, and an admin's explicit placeholder choice remains strict. Fixes #17044 Signed-off-by: Andrew Anderson --- changelog.d/fixed-autopick-pool-fallback.md | 1 + src/docs/manual-provisioning.md | 16 ++ src/pkg/hub/autopick_pool_fallback_test.go | 246 ++++++++++++++++++++ src/pkg/hub/saas_provisioning.go | 83 ++++++- 4 files changed, 342 insertions(+), 4 deletions(-) create mode 100644 changelog.d/fixed-autopick-pool-fallback.md create mode 100644 src/pkg/hub/autopick_pool_fallback_test.go diff --git a/changelog.d/fixed-autopick-pool-fallback.md b/changelog.d/fixed-autopick-pool-fallback.md new file mode 100644 index 0000000000..2090913b17 --- /dev/null +++ b/changelog.d/fixed-autopick-pool-fallback.md @@ -0,0 +1 @@ +- hub: Auto-pick on "Approve provision" now falls back to any other registered non-GPU pool (most available first, tie-break by cluster ID) when the request's preferred pool has no available placeholder, instead of returning 409 while capacity exists elsewhere; private requests stay pinned to the GPU pool and an admin's explicit placeholder choice remains strict. diff --git a/src/docs/manual-provisioning.md b/src/docs/manual-provisioning.md index 2603e08ec0..a8b2c0c3cf 100644 --- a/src/docs/manual-provisioning.md +++ b/src/docs/manual-provisioning.md @@ -1689,6 +1689,22 @@ provision. Two pools, one per method type: - **Hub-reachable-cluster** placeholders serve **public** methods β€” provisioned via the API (Path A) and left running at **`replicas: 1`**. +**Auto-pick on approve.** When an admin approves a provision request without +choosing a specific placeholder, the hub picks one from the pool matching the +request's `auth_method`: `private` β†’ the heartbeat-only/GPU pool, anything else +β†’ the default hub-reachable pool (`hive-oke`). The **public preference is +soft**: if the default pool has no `available` placeholder, Auto-pick falls back +to any *other* cluster registered in `clusters.json` that has one β€” most +available first, ties broken by cluster ID β€” and logs one line +(`approve: pool hive-oke empty, falling back to lke648397`). Two pins stay +**hard** and never fall back: a `private` request only ever draws from the GPU +pool, and the GPU pool is never used to satisfy a public request. Clusters with +placeholders on disk but no `clusters.json` entry are skipped. The admin's +explicit pick in the approve modal is strict β€” if that placeholder is not +available the approve returns `409`, regardless of capacity elsewhere. +Only when every eligible pool is empty does Auto-pick return +`409 no available placeholder hive in pool "hive-oke" or any other registered pool`. + > **Gotcha β€” placeholder IDs are date + random suffix, NOT sequential integers.** > The live pool uses `hosted-available---`, where > `` is a short unique token (4 lowercase base36 chars) β€” e.g. diff --git a/src/pkg/hub/autopick_pool_fallback_test.go b/src/pkg/hub/autopick_pool_fallback_test.go new file mode 100644 index 0000000000..970d7d68c7 --- /dev/null +++ b/src/pkg/hub/autopick_pool_fallback_test.go @@ -0,0 +1,246 @@ +package hub + +import ( + "log/slog" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" +) + +// Cluster IDs used by the fallback tests. fallbackClusterID stands in for the +// second public pool (lke648397 in production); unregisteredClusterID is a +// pool that has placeholders on disk but no clusters.json entry. +const ( + fallbackClusterID = "lke648397" + secondFallbackCluster = "aks-eu" + unregisteredClusterID = "ghost-cluster" +) + +// placeholderSpec is one available placeholder to seed for a test case. +type placeholderSpec struct { + id string + cluster string +} + +func seedPlaceholders(t *testing.T, specs []placeholderSpec) { + t.Helper() + for _, p := range specs { + if err := saveSaaSHive(&SaaSHive{ID: p.id, Owner: hubAdminUsername, Status: statusAvailable, ClusterID: p.cluster}); err != nil { + t.Fatal(err) + } + } +} + +func newFallbackTestHub() *HubServer { + return &HubServer{ + logger: slog.Default(), + hubSecret: testHubSecret, + saveCh: make(chan struct{}, 1), + clusters: map[string]ClusterConfig{ + defaultClusterID: *dynamicCluster(), + fallbackClusterID: {ID: fallbackClusterID, Name: "lke"}, + secondFallbackCluster: {ID: secondFallbackCluster, Name: "aks"}, + gpuClusterID: {ID: gpuClusterID, Name: "gpu", PullOnly: true}, + }, + } +} + +// TestAutoPickPlaceholderFallback is the table over (preferred pool empty / +// non-empty) x (other pools empty / non-empty) plus the hard pins that must +// never fall back. +func TestAutoPickPlaceholderFallback(t *testing.T) { + tests := []struct { + name string + preferred string + placeholders []placeholderSpec + wantID string + wantPool string + }{ + { + name: "preferred non-empty, others non-empty -> preferred wins", + preferred: defaultClusterID, + placeholders: []placeholderSpec{ + {"oke-1", defaultClusterID}, + {"lke-1", fallbackClusterID}, + {"lke-2", fallbackClusterID}, + }, + wantID: "oke-1", + wantPool: defaultClusterID, + }, + { + name: "preferred empty, others empty -> nothing", + preferred: defaultClusterID, + placeholders: nil, + wantID: "", + wantPool: defaultClusterID, + }, + { + name: "preferred empty, one other pool non-empty -> fall back", + preferred: defaultClusterID, + placeholders: []placeholderSpec{ + {"lke-2", fallbackClusterID}, + {"lke-1", fallbackClusterID}, + }, + wantID: "lke-1", + wantPool: fallbackClusterID, + }, + { + name: "preferred empty, several pools -> most available first", + preferred: defaultClusterID, + placeholders: []placeholderSpec{ + {"aks-1", secondFallbackCluster}, + {"lke-1", fallbackClusterID}, + {"lke-2", fallbackClusterID}, + }, + wantID: "lke-1", + wantPool: fallbackClusterID, + }, + { + name: "preferred empty, equal counts -> tie-break by cluster id", + preferred: defaultClusterID, + placeholders: []placeholderSpec{ + {"lke-1", fallbackClusterID}, + {"aks-1", secondFallbackCluster}, + }, + wantID: "aks-1", + wantPool: secondFallbackCluster, + }, + { + name: "preferred empty, only GPU pool has placeholders -> no fallback onto GPU", + preferred: defaultClusterID, + placeholders: []placeholderSpec{ + {"gpu-1", gpuClusterID}, + }, + wantID: "", + wantPool: defaultClusterID, + }, + { + name: "preferred empty, only unregistered cluster has placeholders -> skipped", + preferred: defaultClusterID, + placeholders: []placeholderSpec{ + {"ghost-1", unregisteredClusterID}, + }, + wantID: "", + wantPool: defaultClusterID, + }, + { + name: "private request: GPU pool empty, public pools full -> stays pinned", + preferred: gpuClusterID, + placeholders: []placeholderSpec{ + {"oke-1", defaultClusterID}, + {"lke-1", fallbackClusterID}, + }, + wantID: "", + wantPool: gpuClusterID, + }, + { + name: "private request: GPU pool non-empty -> GPU placeholder", + preferred: gpuClusterID, + placeholders: []placeholderSpec{ + {"gpu-1", gpuClusterID}, + {"oke-1", defaultClusterID}, + }, + wantID: "gpu-1", + wantPool: gpuClusterID, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + cleanup := helperSetupTempDirs(t) + defer cleanup() + mkUser(t, hubAdminUsername) + seedPlaceholders(t, tc.placeholders) + + s := newFallbackTestHub() + gotID, gotPool := s.autoPickPlaceholder(tc.preferred) + if gotID != tc.wantID || gotPool != tc.wantPool { + t.Errorf("autoPickPlaceholder(%q) = (%q, %q), want (%q, %q)", tc.preferred, gotID, gotPool, tc.wantID, tc.wantPool) + } + }) + } +} + +// TestHandleApproveProvisionPoolFallback drives the handler end to end: a +// public request with an empty default pool is served from another pool and +// the claimed hive keeps that pool's cluster_id; an admin's explicit pick is +// still strict (409 when the chosen placeholder is not available) and a fully +// empty fleet still 409s with a message that names the fallback scope. +func TestHandleApproveProvisionPoolFallback(t *testing.T) { + pending := func(user string) *ProvisionRequest { + return &ProvisionRequest{ + Username: user, Org: "acme", Repos: "repo", PrimaryRepo: "repo", ACMMLevel: 2, + AuthMethod: "public", RequestedAt: time.Now().UTC().Format(time.RFC3339), Status: provisionStatusPending, + } + } + + tests := []struct { + name string + placeholders []placeholderSpec + body string + wantCode int + wantHive string + wantCluster string + wantBodyHas string + }{ + { + name: "auto-pick falls back to the other pool", + placeholders: []placeholderSpec{{"lke-1", fallbackClusterID}}, + body: "", + wantCode: http.StatusOK, + wantHive: "lke-1", + wantCluster: fallbackClusterID, + }, + { + name: "auto-pick with every pool empty -> 409 naming the fallback scope", + placeholders: nil, + body: "", + wantCode: http.StatusConflict, + wantBodyHas: "or any other registered pool", + }, + { + name: "explicit pick of an unavailable placeholder stays strict even though another pool has capacity", + placeholders: []placeholderSpec{{"lke-1", fallbackClusterID}}, + body: `{"hive_id":"oke-missing"}`, + wantCode: http.StatusConflict, + wantBodyHas: "selected placeholder is not available", + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + cleanup := helperSetupTempDirs(t) + defer cleanup() + mkUser(t, hubAdminUsername) + seedPlaceholders(t, tc.placeholders) + if err := saveProvisionRequest(pending("bob")); err != nil { + t.Fatal(err) + } + + s := newFallbackTestHub() + rec := httptest.NewRecorder() + req := setPathValue(reqWithUser(http.MethodPut, "/approve-prov", tc.body, hubAdminUsername), "username", "bob") + s.handleApproveProvision(rec, req) + + if rec.Code != tc.wantCode { + t.Fatalf("status = %d, want %d (body=%s)", rec.Code, tc.wantCode, rec.Body.String()) + } + if tc.wantBodyHas != "" && !strings.Contains(rec.Body.String(), tc.wantBodyHas) { + t.Errorf("body = %s, want it to contain %q", rec.Body.String(), tc.wantBodyHas) + } + if tc.wantHive == "" { + return + } + got := loadSaaSHive(tc.wantHive) + if got == nil { + t.Fatalf("hive %q missing after approve", tc.wantHive) + } + if got.Owner != "bob" || got.Status != statusAssigned { + t.Errorf("hive %q owner/status = %q/%q, want bob/%s", tc.wantHive, got.Owner, got.Status, statusAssigned) + } + if got.ClusterID != tc.wantCluster { + t.Errorf("hive %q ClusterID = %q, want %q (fallback must keep the placeholder's real pool)", tc.wantHive, got.ClusterID, tc.wantCluster) + } + }) + } +} diff --git a/src/pkg/hub/saas_provisioning.go b/src/pkg/hub/saas_provisioning.go index 28cec59029..ae46e7d3a7 100644 --- a/src/pkg/hub/saas_provisioning.go +++ b/src/pkg/hub/saas_provisioning.go @@ -613,8 +613,11 @@ func (s *HubServer) handleApproveProvision(w http.ResponseWriter, r *http.Reques // Approving now ASSIGNS an available placeholder instead of just bumping // quota for a manual provision. Pick the pool from the request's auth_method - // (private β†’ the heartbeat-only cluster / GPU pool, otherwise β†’ the hub-reachable cluster / public pool). If no - // placeholder is available in that pool, tell the admin to provision more. + // (private β†’ the heartbeat-only cluster / GPU pool, otherwise β†’ the hub-reachable cluster / public pool). + // The public preference is soft: when that pool is empty, Auto-pick falls + // back to any other registered non-GPU pool with a placeholder (see + // autoPickPlaceholder). Only when every eligible pool is empty does the + // admin get told to provision more. pool := poolClusterForAuthMethod(pr.AuthMethod) hiveID := strings.TrimSpace(approveBody.HiveID) if hiveID != "" { @@ -626,10 +629,19 @@ func (s *HubServer) handleApproveProvision(w http.ResponseWriter, r *http.Reques return } } else { - hiveID = findAvailablePlaceholder(pool) + var pickedPool string + hiveID, pickedPool = s.autoPickPlaceholder(pool) + if hiveID != "" && pickedPool != pool { + s.logger.Info(fmt.Sprintf("approve: pool %s empty, falling back to %s", pool, pickedPool), "target", targetUsername, "hive", hiveID) + pool = pickedPool + } } if hiveID == "" { - http.Error(w, fmt.Sprintf(`{"error":"no available placeholder hive in pool %q β€” provision more placeholders"}`, pool), http.StatusConflict) + scope := "" + if pool != gpuClusterID { + scope = " or any other registered pool" + } + http.Error(w, fmt.Sprintf(`{"error":"no available placeholder hive in pool %q%s β€” provision more placeholders"}`, pool, scope), http.StatusConflict) return } @@ -904,6 +916,69 @@ func findAvailablePlaceholder(clusterID string) string { return "" } +// autoPickPlaceholder implements Auto-pick for handleApproveProvision: it +// returns the ID of an available placeholder and the pool it came from. +// +// The preferred pool is tried first. If it is empty AND the preference is +// soft (the public/default pool), the pick falls back to any OTHER cluster +// registered in clusters.json that has an available placeholder β€” ordered by +// most available first, tie-broken by cluster ID, so the choice is +// deterministic. Two pins stay hard and never fall back: a private request +// stays on the GPU pool (its spokes are heartbeat-only and the pool is a +// tenancy boundary), and the GPU pool is never used to satisfy a public +// request. Returns ("", preferredPool) when no eligible pool has a +// placeholder. +func (s *HubServer) autoPickPlaceholder(preferredPool string) (hiveID, pool string) { + if id := findAvailablePlaceholder(preferredPool); id != "" { + return id, preferredPool + } + if preferredPool == gpuClusterID { + return "", preferredPool + } + + type poolCount struct { + id string + count int + first string + } + byPool := map[string]*poolCount{} + for _, h := range listSaaSHives() { + if h.Status != statusAvailable || !isHubAdmin(h.Owner) { + continue + } + cid := clusterIDForHive(&h) + if cid == preferredPool || cid == gpuClusterID { + continue + } + if _, registered := s.clusters[cid]; !registered { + continue + } + pc := byPool[cid] + if pc == nil { + pc = &poolCount{id: cid, first: h.ID} + byPool[cid] = pc + } + pc.count++ + if h.ID < pc.first { + pc.first = h.ID + } + } + if len(byPool) == 0 { + return "", preferredPool + } + candidates := make([]*poolCount, 0, len(byPool)) + for _, pc := range byPool { + candidates = append(candidates, pc) + } + sort.Slice(candidates, func(i, j int) bool { + if candidates[i].count != candidates[j].count { + return candidates[i].count > candidates[j].count + } + return candidates[i].id < candidates[j].id + }) + return candidates[0].first, candidates[0].id +} + // AvailablePlaceholder is one row of the approve-picker dropdown: an available // placeholder hive the admin can assign a provision request to. type AvailablePlaceholder struct {