diff --git a/.github/scripts/native-queue-policy.cjs b/.github/scripts/native-queue-policy.cjs index 5baeb710e0..48b4bc15a9 100644 --- a/.github/scripts/native-queue-policy.cjs +++ b/.github/scripts/native-queue-policy.cjs @@ -237,10 +237,12 @@ function isCosmeticLabelFlip({ eventName, payload }) { // A cosmetic flip skips the evidence wait but must not mint a fresh passing // required check: replay the latest completed `success` run of this check on // the head (rejecting one not from this workflow) and still read live PR -// eligibility. Returns null when there is nothing to replay — no completed -// run, or the latest one did not succeed (a failed, cancelled or timed-out -// wait) — so the caller falls through to the full evidence wait instead of -// pinning that earlier verdict onto every later stage-label flip. +// eligibility and the POLICY_CHECKS evidence. Returns null when there is +// nothing to replay — no completed run, or the latest one did not succeed (a +// failed, cancelled or timed-out wait), or a policy provider is missing, +// still running or reporting UNKNOWN — so the caller falls through to the +// full evidence wait instead of pinning that earlier verdict onto every later +// stage-label flip. async function replayLastVerdict({ github, context, currentCheckRunId }) { const repo = context.repo; const event = context.payload.pull_request; @@ -259,6 +261,19 @@ async function replayLastVerdict({ github, context, currentCheckRunId }) { // a run of this workflow file on this event may be replayed. await assertActionsProvenance(github, repo, pr.number, pr.head.sha, prior, POLICY_CHECK_SOURCE, new Map()); if (prior.conclusion !== 'success') return null; + // The prior success is only valid while the providers it validated are + // still green: one re-run red on this head must not be papered over. A + // provider that is missing or still running is not a verdict either way; + // fall through so the caller runs the full polling wait instead of failing. + // A DCO enumeration UNKNOWN also falls through: the full wait re-raises it + // once and renders the runbook job summary, so operators see the same + // surface whichever event fired. + try { + await assertEvidence(github, repo, pr.number, pr.head.sha, POLICY_CHECKS); + } catch (error) { + if (error instanceof PendingEvidence || error instanceof ProviderUnavailable) return null; + throw error; + } return `PR #${pr.number}: replayed ${POLICY_CHECK_NAME} run ${prior.id} (success) on ${pr.head.sha}`; } diff --git a/.github/scripts/native-queue-policy.test.cjs b/.github/scripts/native-queue-policy.test.cjs index 7a08dfec2f..c3d3e50f48 100644 --- a/.github/scripts/native-queue-policy.test.cjs +++ b/.github/scripts/native-queue-policy.test.cjs @@ -409,11 +409,16 @@ function replayFixture(priorRuns, labels = [], workflowOverride = {}) { check_suite: { id: 5000 + r.id }, details_url: `https://github.com/acme/widget/actions/runs/${3000 + r.id}/job/1`, ...r, })); - f.github.rest.actions.getWorkflowRun = async ({ run_id: id }) => ({ data: { + const baseGetWorkflowRun = f.github.rest.actions.getWorkflowRun; + const basePaginate = f.github.paginate; + f.github.rest.actions.getWorkflowRun = async ({ run_id: id }) => (id < 3000 ? baseGetWorkflowRun({ run_id: id }) : { data: { id, check_suite_id: id + 2000, path: '.github/workflows/native-queue-policy.yml', event: 'pull_request_target', head_sha: head, ...workflowOverride, } }); f.github.paginate = async (method, args) => { + // Policy provider evidence comes from the base fixture (all green). + if (method === 'jobs' && args.run_id < 3000) return basePaginate(method, args); + if (method === 'checks' && !args.check_name) return basePaginate(method, args); if (method === 'jobs') { const run = runs.find(r => 3000 + r.id === args.run_id); if (!run) return []; @@ -440,6 +445,39 @@ test('cosmetic flip replays the latest completed success on the same head', asyn assert.equal(f.state.listed[0].ref, sha(11)); }); +test('cosmetic flip does not replay a success when a policy provider re-ran red on the head', async () => { + for (const name of Object.keys(POLICY_CHECKS)) { + const f = replayFixture([{ id: 7, status: 'completed', conclusion: 'success' }]); + f.state.runs.get(sha(11)).find(r => r.name === name).conclusion = 'failure'; + await assert.rejects(replayLastVerdict(f), new RegExp(`${name}: failure`)); + } +}); + +test('cosmetic flip falls through to the full wait while a policy provider is re-running or missing', async () => { + // PendingEvidence is not a verdict: returning null sends the caller to the + // polling wait instead of failing the check with no retry. + for (const name of Object.keys(POLICY_CHECKS)) { + for (const change of [{ status: 'queued', conclusion: null }, { status: 'in_progress', conclusion: null }]) { + const f = replayFixture([{ id: 7, status: 'completed', conclusion: 'success' }]); + Object.assign(f.state.runs.get(sha(11)).find(r => r.name === name), change); + assert.equal(await replayLastVerdict(f), null); + } + const f = replayFixture([{ id: 7, status: 'completed', conclusion: 'success' }]); + const runs = f.state.runs.get(sha(11)); + runs.splice(runs.findIndex(r => r.name === name), 1); + assert.equal(await replayLastVerdict(f), null); + } +}); + +test('cosmetic flip falls through to the full wait when the DCO provider reports UNKNOWN', async () => { + // The full wait re-raises ProviderUnavailable once and renders the runbook + // job summary; failing here would skip that operator-facing surface. + const f = replayFixture([{ id: 7, status: 'completed', conclusion: 'success' }]); + Object.assign(f.state.runs.get(sha(11)).find(r => r.name === 'DCO'), + { conclusion: 'failure', output: { summary: unavailableSummary } }); + assert.equal(await replayLastVerdict(f), null); +}); + test('cosmetic flip has nothing to replay without a prior success and falls through', async () => { // A failed, cancelled (timed-out) or absent prior run must not be pinned // onto every later stage-label flip: null sends the caller to the full wait. diff --git a/changelog.d/added-crux-contained-run-service.md b/changelog.d/added-crux-contained-run-service.md new file mode 100644 index 0000000000..fb9167d36a --- /dev/null +++ b/changelog.d/added-crux-contained-run-service.md @@ -0,0 +1 @@ +- crux: contained run service (`crux runs`) on top of the sandbox launcher (Podman + kubejob) with saved run state under `runstate/` and `/v1/runs` serve routes (RFC hivecommons/crux#114, S2 + S7 minimum). diff --git a/changelog.d/changed-16567.md b/changelog.d/changed-16567.md new file mode 100644 index 0000000000..110a6a8f61 --- /dev/null +++ b/changelog.d/changed-16567.md @@ -0,0 +1 @@ +- build(deps): bump ws from 8.21.3 to 8.22.1 in /src/proxy (#16567) diff --git a/changelog.d/changed-16785-proxy-override-docs.md b/changelog.d/changed-16785-proxy-override-docs.md new file mode 100644 index 0000000000..3386ec5d5f --- /dev/null +++ b/changelog.d/changed-16785-proxy-override-docs.md @@ -0,0 +1 @@ +- `src/docs/env-vars.md` and `src/docs/security-model.md` now describe `HIVE_PROXY_ALLOW_PRIVATE_FORWARD` as it behaves since [#16785](https://github.com/hivecommons/hive/pull/16785): the destination guard covers plain-HTTP forwards, uninspected CONNECT tunnels and transparent-TLS passthrough, and the override lifts it for all three legs at once. The docs previously described the guard and its override as plain-HTTP-only. diff --git a/changelog.d/fixed-autopick-pool-fallback.md b/changelog.d/fixed-autopick-pool-fallback.md new file mode 100644 index 0000000000..2090913b17 --- /dev/null +++ b/changelog.d/fixed-autopick-pool-fallback.md @@ -0,0 +1 @@ +- hub: Auto-pick on "Approve provision" now falls back to any other registered non-GPU pool (most available first, tie-break by cluster ID) when the request's preferred pool has no available placeholder, instead of returning 409 while capacity exists elsewhere; private requests stay pinned to the GPU pool and an admin's explicit placeholder choice remains strict. diff --git a/changelog.d/fixed-kubejob-sa-token-exit-code.md b/changelog.d/fixed-kubejob-sa-token-exit-code.md new file mode 100644 index 0000000000..62b6efa971 --- /dev/null +++ b/changelog.d/fixed-kubejob-sa-token-exit-code.md @@ -0,0 +1 @@ +- Sandbox Job pods no longer receive the ServiceAccount token and now run with a restrictive security context, and an unreadable completed pod no longer reports a guessed exit code 0 ([#17021](https://github.com/hivecommons/hive/issues/17021), [#17022](https://github.com/hivecommons/hive/issues/17022)). Previously the kubejob manifest left `automountServiceAccountToken` at its default and set no `securityContext`, so a contained run could read the cluster token; it now sets `automountServiceAccountToken: false`, `allowPrivilegeEscalation: false`, drops all capabilities and uses the `RuntimeDefault` seccomp profile. When a Job completes but its pod can no longer be read, the result keeps the unknown exit code (`-1`) instead of claiming 0; the kick log shows `exit_code: unknown`, the transcript's trailing status line and the dashboard's job terminal show the code as `-1`, and neither the publish gate nor `publish.verify` withholds the PR for it. diff --git a/src/crux/agent/sandbox_executor.go b/src/crux/agent/sandbox_executor.go index 75349f9410..ec0d5d2ca9 100644 --- a/src/crux/agent/sandbox_executor.go +++ b/src/crux/agent/sandbox_executor.go @@ -17,6 +17,7 @@ import ( "github.com/hivecommons/hive/crux/config" "github.com/hivecommons/hive/crux/effects" "github.com/hivecommons/hive/crux/internal/textutil" + "github.com/hivecommons/hive/crux/kubejob" "github.com/hivecommons/hive/crux/logscrub" "github.com/hivecommons/hive/crux/outputschema" "github.com/hivecommons/hive/crux/pushbroker" @@ -129,8 +130,16 @@ type SandboxVerifyResult struct { TimedOut bool } -// Passed reports whether the command ran to completion with exit 0. -func (v SandboxVerifyResult) Passed() bool { return v.Err == "" && v.ExitCode == 0 && !v.TimedOut } +// Passed reports whether the command ran to completion with exit 0. A +// launcher that returned no error but could not read the exit code back +// (a Complete Job whose pod is already gone) counts as passed too: the +// Job's own success condition is the fact, the missing code is not a failure. +func (v SandboxVerifyResult) Passed() bool { + if v.Err != "" || v.TimedOut { + return false + } + return v.ExitCode == 0 || v.ExitCode == kubejob.ExitCodeUnknown +} type SandboxExecutor struct { Launcher sandbox.Launcher @@ -622,7 +631,9 @@ func sandboxPublishWithheld(res SandboxKickResult) string { } return reason } - if res.Sandbox.ExitCode != 0 { + // A completed Job whose pod could not be read has no exit code; that is + // not a failure, and the guessed 0 it used to carry was not a fact either. + if code := res.Sandbox.ExitCode; code != 0 && code != kubejob.ExitCodeUnknown { return fmt.Sprintf("sandbox exited %d", res.Sandbox.ExitCode) } return "" diff --git a/src/crux/agent/sandbox_kick_log.go b/src/crux/agent/sandbox_kick_log.go index 56aeede587..39833f57f3 100644 --- a/src/crux/agent/sandbox_kick_log.go +++ b/src/crux/agent/sandbox_kick_log.go @@ -4,8 +4,11 @@ import ( "fmt" "os" "path/filepath" + "strconv" "strings" "time" + + "github.com/hivecommons/hive/crux/kubejob" ) const ( @@ -25,7 +28,11 @@ const ( // text from the agent. func sandboxKickLogBody(runtime string, res SandboxKickResult, runErr error) string { var b strings.Builder - fmt.Fprintf(&b, "runtime: %s\nexit_code: %d\ncommits: %d\n", runtime, res.Sandbox.ExitCode, res.CommitCount) + exitCode := strconv.Itoa(res.Sandbox.ExitCode) + if res.Sandbox.ExitCode == kubejob.ExitCodeUnknown { + exitCode = "unknown" + } + fmt.Fprintf(&b, "runtime: %s\nexit_code: %s\ncommits: %d\n", runtime, exitCode, res.CommitCount) if res.Branch != "" { fmt.Fprintf(&b, "branch: %s\n", res.Branch) } diff --git a/src/crux/agent/sandbox_kick_log_test.go b/src/crux/agent/sandbox_kick_log_test.go index da308ac8a7..1ac931e14c 100644 --- a/src/crux/agent/sandbox_kick_log_test.go +++ b/src/crux/agent/sandbox_kick_log_test.go @@ -10,6 +10,7 @@ import ( "github.com/hivecommons/hive/crux" "github.com/hivecommons/hive/crux/config" + "github.com/hivecommons/hive/crux/kubejob" "github.com/hivecommons/hive/crux/outputschema" "github.com/hivecommons/hive/crux/sandbox" ) @@ -146,3 +147,39 @@ func TestBuildSandboxRunCarriesPRWithheld(t *testing.T) { t.Errorf("PRWithheld = %q (len %d), want the bounded reason", run.PRWithheld, len([]rune(run.PRWithheld))) } } + +func TestSandboxKickLogBodyUnknownExitCode(t *testing.T) { + res := SandboxKickResult{Sandbox: sandbox.Result{ExitCode: kubejob.ExitCodeUnknown}} + body := sandboxKickLogBody("job", res, nil) + if !strings.Contains(body, "exit_code: unknown\n") { + t.Errorf("unreadable-pod run must not render a guessed exit code:\n%s", body) + } +} + +func TestSandboxPublishWithheldIgnoresUnknownExitCode(t *testing.T) { + unknown := SandboxKickResult{Sandbox: sandbox.Result{ExitCode: kubejob.ExitCodeUnknown}} + if got := sandboxPublishWithheld(unknown); got != "" { + t.Errorf("completed-without-code must not withhold the PR, got %q", got) + } + failed := SandboxKickResult{Sandbox: sandbox.Result{ExitCode: 3}} + if got := sandboxPublishWithheld(failed); got != "sandbox exited 3" { + t.Errorf("got %q, want sandbox exited 3", got) + } +} + +func TestSandboxVerifyResultPassedUnknownExitCode(t *testing.T) { + // A Complete verify Job whose pod could not be read back carries no exit + // code and no error; that must not withhold the PR. + if !(SandboxVerifyResult{ExitCode: kubejob.ExitCodeUnknown}).Passed() { + t.Error("nil-error verify with unknown exit code must pass") + } + if (SandboxVerifyResult{ExitCode: kubejob.ExitCodeUnknown, Err: "boom"}).Passed() { + t.Error("unknown exit code with an error must not pass") + } + if (SandboxVerifyResult{ExitCode: kubejob.ExitCodeUnknown, TimedOut: true}).Passed() { + t.Error("unknown exit code after a timeout must not pass") + } + if (SandboxVerifyResult{ExitCode: 1}).Passed() { + t.Error("exit 1 must not pass") + } +} diff --git a/src/crux/api/runs.go b/src/crux/api/runs.go new file mode 100644 index 0000000000..29360311d3 --- /dev/null +++ b/src/crux/api/runs.go @@ -0,0 +1,122 @@ +package api + +import "time" + +// Contained runs and saved run results (hivecommons/crux#114, S2 and S7): +// +// POST /v1/runs run an untrusted command in a sandbox +// GET /v1/runs/{id} the saved record, whatever the process state +// POST /v1/runs/{id}/cancel cancel; after completion returns the result +// unchanged with already_finished: true +// +// Routes beyond these (restart, listing) are deferred; see +// src/docs/design/crux-work-services.md. +const PathRuns = "/" + Version + "/runs" + +// RunPath is the route of one run's saved record. +func RunPath(id string) string { return PathRuns + "/" + id } + +// RunCancelPath is the route that cancels one run. +func RunCancelPath(id string) string { return RunPath(id) + "/cancel" } + +// The runs routes answer CodeUnsupported (501, shared with the checkout +// routes) when the request asks for something the configured backend cannot +// enforce (a network mode, a profile, a workspace form); the message names it +// and the run never starts with the limit dropped. It differs from +// CodeNotSupported, which is a whole route the server cannot serve (here: a +// server started without a contained-run backend answers POST /v1/runs with +// CodeNotSupported while still serving the saved records). + +// Run statuses, the closed set every record reports. +const ( + RunAccepted = "accepted" + RunRunning = "running" + RunWaiting = "waiting" + RunCompleted = "completed" + RunFailed = "failed" + RunCancelled = "cancelled" + RunUnfinished = "unfinished" + RunUnsupported = "unsupported" +) + +// Run profiles. +const ( + // RunProfileExperiment (the default) runs with network none, no + // credentials and time and output limits. + RunProfileExperiment = "experiment" + // RunProfileAgent is S6's execution half and is not yet accepted + // (CodeUnsupported). + RunProfileAgent = "agent" +) + +// Network modes a run may request. +const ( + RunNetworkNone = "none" + RunNetworkRestricted = "restricted" +) + +// ContainedRunWorktree is the ad-hoc workspace form: the service clones repo and +// pins commit (a full 40-hex SHA) under the run's directory. +type ContainedRunWorktree struct { + Repo string `json:"repo"` + Commit string `json:"commit"` +} + +// ContainedRunRequest is POST /v1/runs. Exactly one of Checkout, Worktree or +// WorkspacePath names the workspace. Checkout (an S1 id) is accepted by the +// wire shape but answers CodeUnsupported until the checkout service lands. +type ContainedRunRequest struct { + Checkout string `json:"checkout,omitempty"` + Worktree *ContainedRunWorktree `json:"worktree,omitempty"` + WorkspacePath string `json:"workspace_path,omitempty"` + Command []string `json:"command"` + EnvAllowlist []string `json:"env_allowlist,omitempty"` + // Network is RunNetworkNone (default) or RunNetworkRestricted. + Network string `json:"network,omitempty"` + // TimeoutMs bounds the command; zero uses the server default. + TimeoutMs int64 `json:"timeout_ms,omitempty"` + // MaxOutputBytes caps each of stdout and stderr in the response; zero + // uses the server default. The full output is kept as an artifact. + MaxOutputBytes int64 `json:"max_output_bytes,omitempty"` + // Profile is RunProfileExperiment (default) or RunProfileAgent. + Profile string `json:"profile,omitempty"` + // IdempotencyKey, when set, makes a repeated request return the + // existing run instead of starting a second one (S7). + IdempotencyKey string `json:"idempotency_key,omitempty"` + // Detach returns as soon as the run is accepted (202) instead of + // waiting for the result; poll GET /v1/runs/{id}. + Detach bool `json:"detach,omitempty"` +} + +// Run is the saved record of one contained run: the response of every runs +// route. Fields that are not yet known are absent; ExitCode in particular is +// present only when the command's exit is known, never guessed. +type ContainedRun struct { + ID string `json:"id"` + Status string `json:"status"` + Owner string `json:"owner,omitempty"` + // Reason explains a failed, unfinished or unsupported status. + Reason string `json:"reason,omitempty"` + CreatedAt time.Time `json:"created_at"` + StartedAt *time.Time `json:"started_at,omitempty"` + FinishedAt *time.Time `json:"finished_at,omitempty"` + + Profile string `json:"profile,omitempty"` + Network string `json:"network,omitempty"` + Backend string `json:"backend,omitempty"` + Command []string `json:"command,omitempty"` + Workdir string `json:"workspace,omitempty"` + ExitCode *int `json:"exit_code,omitempty"` + TimedOut bool `json:"timed_out,omitempty"` + Stdout string `json:"stdout,omitempty"` + Stderr string `json:"stderr,omitempty"` + // StdoutTruncatedAt and StderrTruncatedAt are the byte offsets at which + // the response output was cut; absent when nothing was truncated. The + // full, scrubbed output is the stdout.txt / stderr.txt artifact in the + // run's directory. + StdoutTruncatedAt *int64 `json:"stdout_truncated_at,omitempty"` + StderrTruncatedAt *int64 `json:"stderr_truncated_at,omitempty"` + // AlreadyFinished is set on a cancel response when the run had already + // reached a terminal status; the record is returned unchanged. + AlreadyFinished bool `json:"already_finished,omitempty"` +} diff --git a/src/crux/api/runs_client.go b/src/crux/api/runs_client.go new file mode 100644 index 0000000000..446f05420a --- /dev/null +++ b/src/crux/api/runs_client.go @@ -0,0 +1,27 @@ +package api + +import ( + "context" + "net/http" +) + +// CreateRun is POST /v1/runs: the saved record, final unless req.Detach. +func (c *Client) CreateRun(ctx context.Context, req ContainedRunRequest) (ContainedRun, error) { + var out ContainedRun + err := c.Do(ctx, http.MethodPost, PathRuns, req, &out) + return out, err +} + +// GetRun is GET /v1/runs/{id}. +func (c *Client) GetRun(ctx context.Context, id string) (ContainedRun, error) { + var out ContainedRun + err := c.Do(ctx, http.MethodGet, RunPath(id), nil, &out) + return out, err +} + +// CancelRun is POST /v1/runs/{id}/cancel. +func (c *Client) CancelRun(ctx context.Context, id string) (ContainedRun, error) { + var out ContainedRun + err := c.Do(ctx, http.MethodPost, RunCancelPath(id), nil, &out) + return out, err +} diff --git a/src/crux/api/runs_test.go b/src/crux/api/runs_test.go new file mode 100644 index 0000000000..efe0433393 --- /dev/null +++ b/src/crux/api/runs_test.go @@ -0,0 +1,12 @@ +package api + +import "testing" + +func TestRunPaths(t *testing.T) { + if PathRuns != "/v1/runs" || RunPath("run_a") != "/v1/runs/run_a" || RunCancelPath("run_a") != "/v1/runs/run_a/cancel" { + t.Fatalf("paths: %s %s %s", PathRuns, RunPath("run_a"), RunCancelPath("run_a")) + } + if CodeUnsupported == CodeNotSupported { + t.Fatal("unsupported-limit refusals must be distinguishable from a not-configured server") + } +} diff --git a/src/crux/cli/cli.go b/src/crux/cli/cli.go index b3879de2b5..2ef01ed3d7 100644 --- a/src/crux/cli/cli.go +++ b/src/crux/cli/cli.go @@ -209,6 +209,11 @@ func (e *Env) dispatch(ctx context.Context, opts globalOptions, name string, arg } } return e.local(ctx, opts, name, args) + case "runs": + // The contained-run family lives on a `crux serve`; there is no + // in-process form, so the verb is only reachable with --server. + fmt.Fprintln(e.Stderr, "crux: runs needs --server ADDR (or CRUX_SERVER); contained runs execute on a crux serve") + return ExitUsage case "help", "-h", "--help": fs := flag.NewFlagSet("crux", flag.ContinueOnError) e.usage(fs) @@ -339,6 +344,8 @@ Commands (each maps onto crux.Runtime): runtime that spawned it, not run by hand cline-turn one bounded Cline CLI turn (stdin prompt); --model provider/model, --check for preflight + runs start|status|cancel contained command runs on a crux serve + (--server only; 'crux runs help' for flags) Work services (no runtime; hivecommons/crux#114): checkout add|rm|list pinned checkouts under /checkouts diff --git a/src/crux/cli/remote.go b/src/crux/cli/remote.go index 20882c6a90..accd3e92b1 100644 --- a/src/crux/cli/remote.go +++ b/src/crux/cli/remote.go @@ -53,6 +53,8 @@ func (e *Env) remote(ctx context.Context, opts globalOptions, name string, args return e.remoteStop(ctx, c, args) case "checkout": return e.remoteCheckout(ctx, c, args) + case "runs": + return e.remoteRuns(ctx, c, args) default: fmt.Fprintf(e.Stderr, "crux: %q is not available with --server; it runs where the runtime is\n", name) return ExitUsage diff --git a/src/crux/cli/runs.go b/src/crux/cli/runs.go new file mode 100644 index 0000000000..7b694ae606 --- /dev/null +++ b/src/crux/cli/runs.go @@ -0,0 +1,187 @@ +package cli + +import ( + "context" + "encoding/json" + "fmt" + "strings" + + "github.com/hivecommons/hive/crux/api" +) + +// runsUsage is the `crux runs` synopsis. The verb is `runs`, not `run`, +// because `crux run` already starts an agent from a definition. +const runsUsage = `usage: crux --server ADDR runs start [flags] -- [args…] + crux --server ADDR runs status + crux --server ADDR runs cancel + +start flags: + --repo URL --commit SHA clone URL at SHA into a fresh workspace + --workspace PATH run in an existing directory on the server + --env NAME pass this environment variable through (repeatable) + --network none|restricted network for the command (default none) + --timeout DURATION wall-clock limit (server default when unset) + --max-output BYTES bytes of stdout/stderr returned (server default) + --profile experiment|agent + --idempotency-key KEY return the existing run for a repeated request + --detach return the accepted record instead of waiting + --json print the record as JSON` + +// remoteRuns is `crux --server … runs`: the contained-run family (RFC +// crux#114 S2 + S7). It only exists against a server; the runtime that +// launches sandboxes and keeps run records lives there. +func (e *Env) remoteRuns(ctx context.Context, c *api.Client, args []string) int { + if len(args) == 0 { + fmt.Fprintln(e.Stderr, runsUsage) + return ExitUsage + } + switch args[0] { + case "start": + return e.remoteRunsStart(ctx, c, args[1:]) + case "status": + return e.remoteRunsGet(ctx, c, args[1:]) + case "cancel": + return e.remoteRunsCancel(ctx, c, args[1:]) + case "help", "-h", "--help": + fmt.Fprintln(e.Stderr, runsUsage) + return ExitOK + default: + fmt.Fprintf(e.Stderr, "crux: runs: unknown subcommand %q\n%s\n", args[0], runsUsage) + return ExitUsage + } +} + +// stringList is a repeatable string flag. +type stringList []string + +func (l *stringList) String() string { return strings.Join(*l, ",") } +func (l *stringList) Set(v string) error { *l = append(*l, v); return nil } + +func (e *Env) remoteRunsStart(ctx context.Context, c *api.Client, args []string) int { + fs := e.subFlags("runs start") + var req api.ContainedRunRequest + var wt api.ContainedRunWorktree + var env stringList + fs.StringVar(&wt.Repo, "repo", "", "repository URL to clone") + fs.StringVar(&wt.Commit, "commit", "", "full commit SHA to check out") + fs.StringVar(&req.WorkspacePath, "workspace", "", "existing directory on the server") + fs.Var(&env, "env", "environment variable to pass through (repeatable)") + fs.StringVar(&req.Network, "network", "", "none or restricted") + timeout := fs.Duration("timeout", 0, "wall-clock limit") + fs.Int64Var(&req.MaxOutputBytes, "max-output", 0, "bytes of output returned") + fs.StringVar(&req.Profile, "profile", "", "experiment or agent") + fs.StringVar(&req.IdempotencyKey, "idempotency-key", "", "idempotency key") + fs.BoolVar(&req.Detach, "detach", false, "return the accepted record") + asJSON := fs.Bool("json", false, "print the record as JSON") + if code, ok := parseSub(fs, args); !ok { + return code + } + if fs.NArg() == 0 { + fmt.Fprintln(e.Stderr, runsUsage) + return ExitUsage + } + req.Command = fs.Args() + req.EnvAllowlist = env + req.TimeoutMs = timeout.Milliseconds() + if wt.Repo != "" || wt.Commit != "" { + req.Worktree = &wt + } + run, err := c.CreateRun(ctx, req) + if err != nil { + return e.remoteErr("runs start", "", err) + } + return e.printRun(run, *asJSON) +} + +func (e *Env) remoteRunsGet(ctx context.Context, c *api.Client, args []string) int { + fs := e.subFlags("runs status") + asJSON := fs.Bool("json", false, "print the record as JSON") + if code, ok := parseSub(fs, args); !ok { + return code + } + if fs.NArg() != 1 { + fmt.Fprintln(e.Stderr, "usage: crux --server ADDR runs status [--json] ") + return ExitUsage + } + run, err := c.GetRun(ctx, fs.Arg(0)) + if err != nil { + return e.remoteErr("runs status", fs.Arg(0), err) + } + return e.printRun(run, *asJSON) +} + +func (e *Env) remoteRunsCancel(ctx context.Context, c *api.Client, args []string) int { + fs := e.subFlags("runs cancel") + asJSON := fs.Bool("json", false, "print the record as JSON") + if code, ok := parseSub(fs, args); !ok { + return code + } + if fs.NArg() != 1 { + fmt.Fprintln(e.Stderr, "usage: crux --server ADDR runs cancel [--json] ") + return ExitUsage + } + run, err := c.CancelRun(ctx, fs.Arg(0)) + if err != nil { + return e.remoteErr("runs cancel", fs.Arg(0), err) + } + if run.AlreadyFinished { + fmt.Fprintf(e.Stderr, "crux: run %s had already finished\n", run.ID) + } + return e.printRun(run, *asJSON) +} + +// printRun writes the record: JSON when asked, otherwise a one-line +// summary on stdout with the command's own output following so a caller +// can pipe `crux runs start` like the command itself. The exit code is +// the command's when it is known; an unknown exit is ExitError, never a +// guess. +func (e *Env) printRun(run api.ContainedRun, asJSON bool) int { + if asJSON { + enc := json.NewEncoder(e.Stdout) + enc.SetIndent("", " ") + if err := enc.Encode(run); err != nil { + fmt.Fprintln(e.Stderr, "crux:", err) + return ExitError + } + return runExit(run) + } + line := fmt.Sprintf("%s %s", run.ID, run.Status) + if run.ExitCode != nil { + line += fmt.Sprintf(" exit=%d", *run.ExitCode) + } + if run.TimedOut { + line += " timed_out" + } + if run.Reason != "" { + line += " (" + run.Reason + ")" + } + fmt.Fprintln(e.Stderr, line) + if run.Stdout != "" { + fmt.Fprint(e.Stdout, run.Stdout) + if run.StdoutTruncatedAt != nil { + fmt.Fprintf(e.Stderr, "\n[stdout truncated at byte %d; the full output is kept with the run]\n", *run.StdoutTruncatedAt) + } + } + if run.Stderr != "" { + fmt.Fprint(e.Stderr, run.Stderr) + if run.StderrTruncatedAt != nil { + fmt.Fprintf(e.Stderr, "\n[stderr truncated at byte %d; the full output is kept with the run]\n", *run.StderrTruncatedAt) + } + } + return runExit(run) +} + +// runExit maps a record onto the process exit code. +func runExit(run api.ContainedRun) int { + switch run.Status { + case api.RunCompleted: + if run.ExitCode != nil && *run.ExitCode != 0 { + return ExitError + } + return ExitOK + case api.RunAccepted, api.RunRunning, api.RunWaiting: + return ExitOK + default: + return ExitError + } +} diff --git a/src/crux/cli/runs_test.go b/src/crux/cli/runs_test.go new file mode 100644 index 0000000000..726de5c7cf --- /dev/null +++ b/src/crux/cli/runs_test.go @@ -0,0 +1,252 @@ +package cli + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "path/filepath" + "strings" + "sync" + "testing" + + "github.com/hivecommons/hive/crux/api" + "github.com/hivecommons/hive/crux/config" + "github.com/hivecommons/hive/crux/runs" + "github.com/hivecommons/hive/crux/runstate" +) + +// fakeRunsAPI is the contained-run family of a `crux serve` stand-in. +type fakeRunsAPI struct { + mu sync.Mutex + created []api.ContainedRunRequest + cancelled []string +} + +func (f *fakeRunsAPI) handler() http.Handler { + exit := func(v int) *int { return &v } + cut := int64(5) + records := map[string]api.ContainedRun{ + "run_ok": {ID: "run_ok", Status: api.RunCompleted, ExitCode: exit(0), Stdout: "hello\n", Stderr: "warned\n"}, + "run_fail": {ID: "run_fail", Status: api.RunCompleted, ExitCode: exit(2), Reason: "exit status 2"}, + "run_cut": {ID: "run_cut", Status: api.RunCompleted, ExitCode: exit(0), Stdout: "12345", StdoutTruncatedAt: &cut, Stderr: "abcde", StderrTruncatedAt: &cut}, + "run_lost": {ID: "run_lost", Status: api.RunUnfinished, Reason: "pod vanished"}, + "run_late": {ID: "run_late", Status: api.RunFailed, TimedOut: true, Reason: "timed out", AlreadyFinished: true}, + "run_sleep": {ID: "run_sleep", Status: api.RunRunning}, + } + mux := http.NewServeMux() + mux.HandleFunc("POST "+api.PathRuns, func(w http.ResponseWriter, r *http.Request) { + var req api.ContainedRunRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + w.WriteHeader(http.StatusBadRequest) + _ = json.NewEncoder(w).Encode(api.Error{Error: err.Error(), Code: api.CodeBadRequest}) + return + } + f.mu.Lock() + f.created = append(f.created, req) + f.mu.Unlock() + if req.Profile == api.RunProfileAgent { + w.WriteHeader(http.StatusNotImplemented) + _ = json.NewEncoder(w).Encode(api.Error{Error: "profile agent is not available yet", Code: api.CodeUnsupported}) + return + } + if req.Detach { + w.WriteHeader(http.StatusAccepted) + _ = json.NewEncoder(w).Encode(api.ContainedRun{ID: "run_new", Status: api.RunAccepted}) + return + } + _ = json.NewEncoder(w).Encode(records["run_ok"]) + }) + mux.HandleFunc("GET "+api.PathRuns+"/{id}", func(w http.ResponseWriter, r *http.Request) { + rec, ok := records[r.PathValue("id")] + if !ok { + w.WriteHeader(http.StatusNotFound) + _ = json.NewEncoder(w).Encode(api.Error{Error: "run not found", Code: api.CodeNotFound}) + return + } + _ = json.NewEncoder(w).Encode(rec) + }) + mux.HandleFunc("POST "+api.PathRuns+"/{id}/cancel", func(w http.ResponseWriter, r *http.Request) { + id := r.PathValue("id") + f.mu.Lock() + f.cancelled = append(f.cancelled, id) + f.mu.Unlock() + rec, ok := records[id] + if !ok { + w.WriteHeader(http.StatusNotFound) + _ = json.NewEncoder(w).Encode(api.Error{Error: "run not found", Code: api.CodeNotFound}) + return + } + if rec.Status == api.RunRunning { + rec.Status = api.RunCancelled + } + _ = json.NewEncoder(w).Encode(rec) + }) + return mux +} + +func TestRunsIsRemoteOnly(t *testing.T) { + t.Setenv(config.ServerEnvVar, "") + h := newHarness(t) + if code := h.run("runs", "status", "run_ok"); code != ExitUsage || !strings.Contains(h.stderr.String(), "needs --server") { + t.Fatalf("local runs: %d %s", code, h.stderr) + } + if len(h.rt.Calls()) != 0 { + t.Fatalf("runs touched the local runtime: %v", h.rt.Calls()) + } + h = newHarness(t) + h.run("help") + if !strings.Contains(h.stderr.String(), "runs start|status|cancel") { + t.Fatalf("usage does not list runs:\n%s", h.stderr) + } +} + +func TestRunsStartStatusCancel(t *testing.T) { + t.Setenv(config.ServerEnvVar, "") + t.Setenv(config.ServeTokenEnvVar, "") + f := &fakeRunsAPI{} + ts := httptest.NewServer(f.handler()) + t.Cleanup(ts.Close) + server := ts.URL + + h := newHarness(t) + code := h.run("--server", server, "runs", "start", "--repo", "https://github.com/hivecommons/hive", "--commit", strings.Repeat("a", 40), + "--env", "PATH", "--env", "LANG", "--network", "none", "--timeout", "2s", "--max-output", "64", "--idempotency-key", "k1", "--", "make", "-j2", "test") + if code != ExitOK { + t.Fatalf("start: %d %s", code, h.stderr) + } + if h.stdout.String() != "hello\n" || !strings.Contains(h.stderr.String(), "run_ok completed exit=0") || !strings.Contains(h.stderr.String(), "warned") { + t.Fatalf("start output:\nstdout=%q\nstderr=%q", h.stdout, h.stderr) + } + req := f.created[0] + if req.Worktree == nil || req.Worktree.Repo != "https://github.com/hivecommons/hive" || req.Worktree.Commit != strings.Repeat("a", 40) || + strings.Join(req.Command, " ") != "make -j2 test" || strings.Join(req.EnvAllowlist, ",") != "PATH,LANG" || + req.Network != "none" || req.TimeoutMs != 2000 || req.MaxOutputBytes != 64 || req.IdempotencyKey != "k1" || req.Detach { + t.Fatalf("request = %+v (worktree %+v)", req, req.Worktree) + } + + h = newHarness(t) + if code := h.run("--server", server, "runs", "start", "--workspace", "/srv/ws", "--detach", "--json", "--", "true"); code != ExitOK { + t.Fatalf("detach: %d %s", code, h.stderr) + } + var detached api.ContainedRun + if err := json.Unmarshal([]byte(h.stdout.String()), &detached); err != nil || detached.ID != "run_new" || detached.Status != api.RunAccepted { + t.Fatalf("detach json = %q (%v)", h.stdout, err) + } + if req := f.created[1]; req.WorkspacePath != "/srv/ws" || req.Worktree != nil || !req.Detach { + t.Fatalf("detach request = %+v", req) + } + + h = newHarness(t) + if code := h.run("--server", server, "runs", "start", "--workspace", "/srv/ws", "--profile", "agent", "--", "true"); code != ExitError || !strings.Contains(h.stderr.String(), "not available yet") { + t.Fatalf("unsupported: %d %s", code, h.stderr) + } + + for _, tc := range []struct { + id string + code int + stdout string + stderr string + }{ + {"run_ok", ExitOK, "hello\n", "run_ok completed exit=0"}, + {"run_fail", ExitError, "", "run_fail completed exit=2 (exit status 2)"}, + {"run_cut", ExitOK, "12345", "stdout truncated at byte 5"}, + {"run_lost", ExitError, "", "run_lost unfinished (pod vanished)"}, + {"run_sleep", ExitOK, "", "run_sleep running"}, + {"run_missing", ExitError, "", "run not found"}, + } { + h = newHarness(t) + if code := h.run("--server", server, "runs", "status", tc.id); code != tc.code || h.stdout.String() != tc.stdout || !strings.Contains(h.stderr.String(), tc.stderr) { + t.Fatalf("status %s: %d stdout=%q stderr=%q", tc.id, code, h.stdout, h.stderr) + } + } + h = newHarness(t) + if code := h.run("--server", server, "runs", "status", "--json", "run_cut"); code != ExitOK || !strings.Contains(h.stdout.String(), `"stderr_truncated_at": 5`) { + t.Fatalf("status json: %d %s", code, h.stdout) + } + + h = newHarness(t) + if code := h.run("--server", server, "runs", "cancel", "run_sleep"); code != ExitError || !strings.Contains(h.stderr.String(), "run_sleep cancelled") { + t.Fatalf("cancel: %d %s", code, h.stderr) + } + h = newHarness(t) + if code := h.run("--server", server, "runs", "cancel", "--json", "run_late"); code != ExitError || !strings.Contains(h.stderr.String(), "had already finished") || !strings.Contains(h.stdout.String(), `"timed_out": true`) { + t.Fatalf("late cancel: %d stdout=%s stderr=%s", code, h.stdout, h.stderr) + } + if strings.Join(f.cancelled, ",") != "run_sleep,run_late" { + t.Fatalf("cancelled = %v", f.cancelled) + } +} + +func TestRunsUsage(t *testing.T) { + t.Setenv(config.ServerEnvVar, "") + t.Setenv(config.ServeTokenEnvVar, "") + ts := httptest.NewServer((&fakeRunsAPI{}).handler()) + t.Cleanup(ts.Close) + for _, tc := range []struct { + args []string + code int + stderr string + }{ + {[]string{"runs"}, ExitUsage, "usage: crux --server ADDR runs start"}, + {[]string{"runs", "help"}, ExitOK, "usage: crux --server ADDR runs start"}, + {[]string{"runs", "bogus"}, ExitUsage, `unknown subcommand "bogus"`}, + {[]string{"runs", "start"}, ExitUsage, "usage: crux --server ADDR runs start"}, + {[]string{"runs", "start", "-h"}, ExitOK, "-repo"}, + {[]string{"runs", "start", "--bogus", "x"}, ExitUsage, "bogus"}, + {[]string{"runs", "status"}, ExitUsage, "usage: crux --server ADDR runs status"}, + {[]string{"runs", "status", "-bogus"}, ExitUsage, "bogus"}, + {[]string{"runs", "cancel", "a", "b"}, ExitUsage, "usage: crux --server ADDR runs cancel"}, + {[]string{"runs", "cancel", "-bogus"}, ExitUsage, "bogus"}, + } { + h := newHarness(t) + if code := h.run(append([]string{"--server", ts.URL}, tc.args...)...); code != tc.code || !strings.Contains(h.stderr.String(), tc.stderr) { + t.Fatalf("%v: %d %s", tc.args, code, h.stderr) + } + } +} + +func TestOpenRunsReconcilesAndLoadsTheBackend(t *testing.T) { + t.Setenv(runsConfigEnvVar, "") + h := newHarness(t) + workDir := t.TempDir() + store, err := runstate.Open(workDir) + if err != nil { + t.Fatal(err) + } + stale, _, err := store.Create(runstate.Record{}) + if err != nil { + t.Fatal(err) + } + svc, err := h.env.openRuns(workDir, "") + if err != nil { + t.Fatal(err) + } + if svc.Configured() { + t.Fatal("configured without a config file") + } + if run, err := svc.Get(stale.ID); err != nil || run.Status != api.RunUnfinished { + t.Fatalf("stale record = %+v, %v", run, err) + } + if !strings.Contains(h.stderr.String(), stale.ID+" was in flight") { + t.Fatalf("reconcile not reported: %s", h.stderr) + } + + cfgPath := writeFile(t, filepath.Join(t.TempDir(), "runs.json"), `{"image":"img","workspace_roots":["/srv"]}`) + svc, err = h.env.openRuns(workDir, cfgPath) + if err != nil || !svc.Configured() || svc.Backend.Name != runs.BackendPodman { + t.Fatalf("configured = %v, %v", svc, err) + } + t.Setenv(runsConfigEnvVar, cfgPath) + if svc, err = h.env.openRuns(workDir, ""); err != nil || !svc.Configured() { + t.Fatalf("env config = %v, %v", svc, err) + } + if _, err := h.env.openRuns(workDir, writeFile(t, filepath.Join(t.TempDir(), "bad.json"), `{"backend":"job","image":"img"}`)); err == nil { + t.Fatal("bad config accepted") + } + if _, err := h.env.openRuns(writeFile(t, filepath.Join(t.TempDir(), "file"), ""), cfgPath); err == nil { + t.Fatal("work dir under a file accepted") + } + if code := h.run("serve", "--runs-config", filepath.Join(t.TempDir(), "missing.json")); code != ExitUsage || !strings.Contains(h.stderr.String(), "crux: runs:") { + t.Fatalf("serve with a missing runs config: %d %s", code, h.stderr) + } +} diff --git a/src/crux/cli/serve.go b/src/crux/cli/serve.go index 104848eccf..f188cecc03 100644 --- a/src/crux/cli/serve.go +++ b/src/crux/cli/serve.go @@ -12,6 +12,8 @@ import ( "github.com/hivecommons/hive/crux/checkout" "github.com/hivecommons/hive/crux/config" "github.com/hivecommons/hive/crux/lanes" + "github.com/hivecommons/hive/crux/runs" + "github.com/hivecommons/hive/crux/runstate" "github.com/hivecommons/hive/crux/serve" "github.com/nats-io/nats.go" ) @@ -40,12 +42,14 @@ func (e *Env) cmdServe(ctx context.Context, opts globalOptions, args []string) i fs.StringVar(&checkoutDir, "checkout-dir", "", "where pinned checkouts live (default $"+config.CheckoutDirEnvVar+", then /"+checkout.DirName+")") fs.StringVar(&operatorName, "operator-name", "", "name of the principal behind the control token on work-services routes (default $"+config.ServeOperatorNameEnvVar+", then "+serve.DefaultOperatorName+")") fs.StringVar(&laneConfig, "lanes-config", "", "opt-in durable lane configuration file (default $CRUX_LANES_CONFIG)") + var runsConfig string + fs.StringVar(&runsConfig, "runs-config", "", "opt-in contained-run backend configuration file (default $"+runsConfigEnvVar+")") fs.StringVar(&hiveAddr, "hive-callback", "", "hive capability endpoint to dial (default $"+callback.SocketEnvVar+"; none when unset)") if code, ok := parseSub(fs, args); !ok { return code } if fs.NArg() != 0 { - fmt.Fprintln(e.Stderr, "usage: crux serve [-socket PATH] [-listen HOST:PORT] [-token T] [-hive-callback ADDR] [-lanes-config PATH] [-operator-name NAME] [-checkout-allow PREFIXES] [-checkout-dir DIR]") + fmt.Fprintln(e.Stderr, "usage: crux serve [-socket PATH] [-listen HOST:PORT] [-token T] [-hive-callback ADDR] [-lanes-config PATH] [-operator-name NAME] [-checkout-allow PREFIXES] [-checkout-dir DIR] [-runs-config PATH]") return ExitUsage } if cfg.Token == "" { @@ -117,6 +121,12 @@ func (e *Env) cmdServe(ctx context.Context, opts globalOptions, args []string) i defer broker.Close() srv.Lanes, srv.LaneAuth = broker, auth } + runSvc, err := e.openRuns(opts.workDir, runsConfig) + if err != nil { + fmt.Fprintln(e.Stderr, "crux: runs:", err) + return ExitUsage + } + srv.Runs = runSvc if hive != nil { hive.install(ctx, rt, srv, e.Stdout) hive.watchPolicy(ctx, rt, e.Stdout, e.Stderr) @@ -148,3 +158,42 @@ func (e *Env) laneBrokerFailure(err error) int { fmt.Fprintf(e.Stderr, "crux: lanes: cannot initialize durable broker (%s)\n", category) return ExitError } + +// runsConfigEnvVar names the contained-run configuration file when +// --runs-config is not given. +const runsConfigEnvVar = "CRUX_RUNS_CONFIG" + +// openRuns builds the contained-run service. Saved records under +// /runs are always served, so a server with no backend configured +// still answers `crux runs status`; the backend comes from the config file +// when one is named. Records left in flight by an earlier process are +// marked unfinished here, before the first request can observe them. +func (e *Env) openRuns(workDir, path string) (*runs.Service, error) { + if path == "" { + path = os.Getenv(runsConfigEnvVar) + } + var cfg runs.Config + var backend *runs.Backend + if path != "" { + var err error + if cfg, err = runs.LoadConfig(path); err != nil { + return nil, err + } + if backend, err = runs.BackendFromConfig(cfg); err != nil { + return nil, err + } + } + store, err := runstate.Open(workDir) + if err != nil { + return nil, err + } + svc := runs.New(store, backend, cfg) + reconciled, err := svc.Reconcile() + if err != nil { + return nil, err + } + for _, rec := range reconciled { + fmt.Fprintf(e.Stderr, "crux: runs: %s was in flight at the last shutdown; marked unfinished\n", rec.ID) + } + return svc, nil +} diff --git a/src/crux/cli/verbs_test.go b/src/crux/cli/verbs_test.go index 712e98ced8..aad7ebbbc3 100644 --- a/src/crux/cli/verbs_test.go +++ b/src/crux/cli/verbs_test.go @@ -14,8 +14,9 @@ import ( ) // nonRuntimeVerbs are dispatch cases that do not operate on a runtime: serve -// exposes one, help prints usage, cline-turn executes one pane-side worker. -var nonRuntimeVerbs = map[string]bool{"serve": true, "help": true, "-h": true, "--help": true, "cline-turn": true, "checkout": true} +// exposes one, help prints usage, cline-turn executes one pane-side worker, +// checkout and runs drive work services on a server. +var nonRuntimeVerbs = map[string]bool{"serve": true, "help": true, "-h": true, "--help": true, "cline-turn": true, "checkout": true, "runs": true} func TestVerbsHaveRuntimeMethods(t *testing.T) { rt := reflect.TypeOf((*crux.Runtime)(nil)).Elem() diff --git a/src/crux/kubejob/kubejob.go b/src/crux/kubejob/kubejob.go index d930cadfdd..42d4b6a9fd 100644 --- a/src/crux/kubejob/kubejob.go +++ b/src/crux/kubejob/kubejob.go @@ -225,45 +225,50 @@ func (l *Launcher) Run(ctx context.Context, spec sandbox.LaunchSpec) (sandbox.Re done, waitErr := l.wait(ctx, jobPath) // Collect whatever the pod produced BEFORE deciding anything: a timed-out // or failed kick with no transcript is the worst outcome for an operator. - // After a timeout the caller's ctx is already dead, so the collection and - // the cleanup below run on a fresh, request-bounded context. - collectCtx := ctx - if ctx.Err() != nil { - var cancel context.CancelFunc - collectCtx, cancel = context.WithTimeout(context.Background(), requestTimeout) - defer cancel() - } - pod, podErr := l.findPod(collectCtx, ns, name) + // The caller's ctx may be dead already (kick timeout) or die between any + // two of these requests (cancel during log collection), so every step + // re-checks it and falls back to a fresh, request-bounded context. + // Otherwise a readable, Complete pod would report exitCodeUnknown merely + // because ctx was cancelled after the log was read. + var cancels []context.CancelFunc + defer func() { + for _, c := range cancels { + c() + } + }() + live := func() context.Context { + if ctx.Err() == nil { + return ctx + } + fresh, cancel := context.WithTimeout(context.Background(), requestTimeout) + cancels = append(cancels, cancel) + return fresh + } + pod, podErr := l.findPod(live(), ns, name) if podErr == nil && pod != "" { - res.Stdout = l.podLog(collectCtx, ns, pod) - res.ExitCode = l.podExitCode(collectCtx, ns, pod) + res.Stdout = l.podLog(live(), ns, pod) + res.ExitCode = l.podExitCode(live(), ns, pod) } else { res.ExitCode = exitCodeUnknown } if waitErr != nil { // Context ended (kick timeout or cancel). The Job would otherwise // keep running the workload on the accelerator node; stop it. - _ = l.delete(collectCtx, jobPath) + _ = l.delete(live(), jobPath) return res, waitErr } if !done.succeeded { res.Stderr = done.reason return res, fmt.Errorf("kubejob: job %s failed: %s", name, done.reason) } - if res.ExitCode == exitCodeUnknown { - // The Job reports Complete, so the container exited 0 even though the - // pod could not be read back (already GC'd, or RBAC on pods missing). - res.ExitCode = 0 - } - // Collection may have expired the context even on success; use a fresh one - // for cleanup to ensure the finished Job is immediately removed. - deleteCtx := ctx - if ctx.Err() != nil { - var cancel context.CancelFunc - deleteCtx, cancel = context.WithTimeout(context.Background(), requestTimeout) - defer cancel() - } - _ = l.delete(deleteCtx, jobPath) + // When the Job reports Complete but the pod could not be read back + // (already GC'd, or RBAC on pods missing), res.ExitCode stays + // exitCodeUnknown rather than a guessed 0; callers treat a nil error with + // an unknown code as completed-without-code. + // + // Collection may have expired the context even on success; the fresh + // context ensures the finished Job is still removed immediately. + _ = l.delete(live(), jobPath) return res, nil } @@ -454,6 +459,13 @@ func (l *Launcher) manifest(ns, name, subPath string, spec sandbox.LaunchSpec) m } container := map[string]any{ + // The workspace PVC and the agent's home need to stay writable, so the + // root filesystem is deliberately not made read-only. + "securityContext": map[string]any{ + "allowPrivilegeEscalation": false, + "capabilities": map[string]any{"drop": []string{"ALL"}}, + "seccompProfile": map[string]string{"type": "RuntimeDefault"}, + }, "name": containerName, "image": spec.Image, "workingDir": workdir, @@ -471,6 +483,12 @@ func (l *Launcher) manifest(ns, name, subPath string, spec sandbox.LaunchSpec) m "restartPolicy": restartNever, "containers": []map[string]any{container}, "volumes": volumes, + // Nothing in the kick pod calls the Kubernetes API, so the SA token is + // never mounted into a contained run. + "automountServiceAccountToken": false, + "securityContext": map[string]any{ + "seccompProfile": map[string]string{"type": "RuntimeDefault"}, + }, } if len(l.Options.NodeSelector) > 0 { podSpec["nodeSelector"] = l.Options.NodeSelector diff --git a/src/crux/kubejob/kubejob_test.go b/src/crux/kubejob/kubejob_test.go index bd885d22d0..51f0488e07 100644 --- a/src/crux/kubejob/kubejob_test.go +++ b/src/crux/kubejob/kubejob_test.go @@ -219,7 +219,7 @@ func TestRunContextTimeoutDeletesTheJob(t *testing.T) { } } -func TestRunCompleteWithNoPodStillSucceeds(t *testing.T) { +func TestRunCompleteWithNoPodKeepsExitCodeUnknown(t *testing.T) { api := newFakeAPI() api.statuses = []string{statusComplete} api.noPod = true @@ -230,8 +230,8 @@ func TestRunCompleteWithNoPodStillSucceeds(t *testing.T) { if err != nil { t.Fatalf("Run: %v", err) } - if res.ExitCode != 0 || res.Stdout != "" { - t.Errorf("result = %+v, want exit 0 (Complete) and no log", res) + if res.ExitCode != exitCodeUnknown || res.Stdout != "" { + t.Errorf("result = %+v, want unknown exit code (Complete, pod unreadable) and no log", res) } } @@ -359,6 +359,10 @@ func TestManifestShape(t *testing.T) { `"tolerations":[{"effect":"NoSchedule","key":"spyre","operator":"Exists"}]`, `"limits":{"ibm.com/spyre_pf":"2"}`, `"serviceAccountName":"hive"`, + `"automountServiceAccountToken":false`, + `"allowPrivilegeEscalation":false`, + `"capabilities":{"drop":["ALL"]}`, + `"seccompProfile":{"type":"RuntimeDefault"}`, `"envFrom":[{"secretRef":{"name":"artifact-registry"}}]`, `"claimName":"hive-data"`, `"subPath":"agents/sandbox/deps-123"`, diff --git a/src/crux/runs/backend.go b/src/crux/runs/backend.go new file mode 100644 index 0000000000..8b2aea2a57 --- /dev/null +++ b/src/crux/runs/backend.go @@ -0,0 +1,118 @@ +package runs + +import ( + "context" + "fmt" + "os/exec" + "strings" + + "github.com/hivecommons/hive/crux/config" + "github.com/hivecommons/hive/crux/kubejob" + "github.com/hivecommons/hive/crux/sandbox" +) + +// Capabilities is what a backend can enforce. A request for anything it +// cannot is refused before the run starts. +type Capabilities struct { + NetworkNone bool + NetworkRestricted bool +} + +// Backend pairs a sandbox.Launcher with its capabilities and the teardown +// that guarantees a cancelled or timed-out run leaves nothing behind. +type Backend struct { + Name string + Launcher sandbox.Launcher + Capabilities Capabilities + // Teardown, when set, is run after a cancel or timeout with the launch + // name. The podman launcher's client process dies with the context but + // the container it started does not, so podman removes it here; the + // Job launcher deletes its Job itself (propagationPolicy Background) + // and leaves Teardown nil. + Teardown func(ctx context.Context, name string) error +} + +// Supports reports whether the backend can enforce network mode mode; the +// returned string names what is missing when it cannot. +func (b Backend) Supports(network string) (string, bool) { + switch network { + case sandbox.NetworkModeNone: + if !b.Capabilities.NetworkNone { + return fmt.Sprintf("backend %s cannot enforce network %q", b.Name, network), false + } + case sandbox.NetworkModeRestricted: + if !b.Capabilities.NetworkRestricted { + return fmt.Sprintf("backend %s cannot enforce network %q", b.Name, network), false + } + default: + return fmt.Sprintf("unknown network mode %q", network), false + } + return "", true +} + +// BackendFromConfig selects and builds the backend cfg names. +func BackendFromConfig(cfg Config) (*Backend, error) { + switch cfg.Backend { + case BackendPodman: + bin := cfg.PodmanBinary + if bin == "" { + bin = sandbox.PodmanBinary + } + return &Backend{ + Name: BackendPodman, + Launcher: sandbox.PodmanLauncher{Binary: bin}, + Capabilities: Capabilities{NetworkNone: true, NetworkRestricted: cfg.NetworkRestricted}, + Teardown: podmanTeardown(bin), + }, nil + case BackendJob: + if cfg.Job == nil { + return nil, fmt.Errorf("%w: backend %q needs a job block", ErrConfig, BackendJob) + } + opts := jobOptions(*cfg.Job) + if err := opts.Validate(); err != nil { + return nil, fmt.Errorf("%w: %v", ErrConfig, err) + } + return &Backend{ + Name: BackendJob, + Launcher: &kubejob.Launcher{Options: opts}, + // The Job manifest carries no network mode; only an operator's + // NetworkPolicy can enforce one, and only "none" is claimable. + Capabilities: Capabilities{NetworkNone: cfg.JobNetworkPolicy}, + }, nil + } + return nil, fmt.Errorf("%w: unknown backend %q", ErrConfig, cfg.Backend) +} + +// podmanTeardown removes the named container whether or not it is still +// running; --ignore makes an already-removed container a success. +func podmanTeardown(bin string) func(context.Context, string) error { + return func(ctx context.Context, name string) error { + out, err := exec.CommandContext(ctx, bin, "rm", "--force", "--ignore", name).CombinedOutput() + if err != nil { + return fmt.Errorf("podman rm %s: %w: %s", name, err, strings.TrimSpace(string(out))) + } + return nil + } +} + +// jobOptions mirrors the agent sandbox's config-to-launcher mapping +// (agent.JobOptionsFromConfig) without importing the agent manager. +func jobOptions(job config.SandboxJobConfig) kubejob.Options { + opts := kubejob.Options{ + WorkspaceClaim: job.WorkspaceClaim, + WorkspaceClaimMount: job.WorkspaceClaimMount, + NodeSelector: job.NodeSelector, + ServiceAccount: job.ServiceAccount, + SchedulerName: job.SchedulerName, + EnvFromSecrets: job.EnvFromSecrets, + TTLSeconds: job.TTLSeconds, + Resources: kubejob.Resources{Limits: job.Resources.Limits, Requests: job.Resources.Requests}, + } + for _, t := range job.Tolerations { + opts.Tolerations = append(opts.Tolerations, kubejob.Toleration{Key: t.Key, Operator: t.Operator, Value: t.Value, Effect: t.Effect}) + } + for _, v := range job.Volumes { + opts.ExtraVolumes = append(opts.ExtraVolumes, kubejob.VolumeMount{Name: v.Name, Claim: v.Claim, MountPath: v.MountPath, ReadOnly: v.ReadOnly}) + } + return opts +} diff --git a/src/crux/runs/config.go b/src/crux/runs/config.go new file mode 100644 index 0000000000..92040ba151 --- /dev/null +++ b/src/crux/runs/config.go @@ -0,0 +1,159 @@ +// Package runs is the contained-run service of the Crux work services (S2 +// in hivecommons/crux#114): run a command the caller does not trust in a +// sandbox with no network and no credentials, bounded in time and output, +// and report the real exit code — or, when the exit is not knowable, say +// so. Records live in runstate (S7) so a caller that lost its connection +// can ask again by ID. +// +// sandbox.Launcher is the execution seam; Backend pairs a launcher with +// what it can enforce. A request for a limit the selected backend cannot +// enforce is refused with an *UnsupportedError, never run weakened. +package runs + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "strings" + "time" + + "github.com/hivecommons/hive/crux/config" +) + +// Backend names, Config.Backend. +const ( + BackendPodman = "podman" + BackendJob = "job" +) + +// Limits and defaults. +const ( + DefaultTimeout = 10 * time.Minute + DefaultMaxTimeout = 2 * time.Hour + DefaultMaxOutputBytes = 1 << 20 + // DefaultMaxOutputCap bounds what a caller may ask for; the full output + // is kept as an artifact regardless, so the cap protects the response + // and the record, not the evidence. + DefaultMaxOutputCap = 64 << 20 + maxConfigBytes = 1 << 20 + // TeardownTimeout bounds the container removal after a cancel or + // timeout, run on its own context because the run's is already dead. + TeardownTimeout = 30 * time.Second +) + +// ErrConfig wraps every configuration error. +var ErrConfig = errors.New("runs: invalid configuration") + +// Config is the operator's choice of backend and bounds, loaded by +// LoadConfig from the JSON file `crux serve --runs-config` names. +// Selection is configuration, not per request. +type Config struct { + // Backend is BackendPodman (default) or BackendJob. + Backend string `json:"backend,omitempty"` + // Image is the container image every run uses. Required. + Image string `json:"image"` + // PodmanBinary overrides the podman executable (tests). + PodmanBinary string `json:"podman_binary,omitempty"` + // Job holds the Kubernetes Job settings when Backend is BackendJob; + // the same shape as the agent sandbox's job block. + Job *config.SandboxJobConfig `json:"job,omitempty"` + // JobNetworkPolicy is the operator's assertion that a NetworkPolicy in + // the Job namespace denies egress for pods labelled + // app.kubernetes.io/managed-by=hive-kubejob. The Job launcher cannot + // set a network mode itself, so without this assertion a request for + // network "none" on the job backend is refused. + JobNetworkPolicy bool `json:"job_network_policy,omitempty"` + // NetworkRestricted is the operator's assertion that a container + // network named "restricted" exists on the podman host; without it a + // request for network "restricted" is refused. + NetworkRestricted bool `json:"network_restricted,omitempty"` + // RepoAllowlist is the set of repository URL prefixes the ad-hoc + // worktree form may clone. Empty refuses every worktree request. + RepoAllowlist []string `json:"repo_allowlist,omitempty"` + // WorkspaceRoots is the set of host directories a plain workspace_path + // may lie under. Empty refuses every workspace_path request. + WorkspaceRoots []string `json:"workspace_roots,omitempty"` + // DefaultTimeoutMs and MaxTimeoutMs bound timeout_ms; zero takes the + // package defaults. + DefaultTimeoutMs int64 `json:"default_timeout_ms,omitempty"` + MaxTimeoutMs int64 `json:"max_timeout_ms,omitempty"` + // DefaultMaxOutputBytes and MaxOutputBytesCap bound max_output_bytes; + // zero takes the package defaults. + DefaultMaxOutputBytes int64 `json:"default_max_output_bytes,omitempty"` + MaxOutputBytesCap int64 `json:"max_output_bytes_cap,omitempty"` +} + +// LoadConfig reads and validates the JSON configuration at path. +func LoadConfig(path string) (Config, error) { + f, err := os.Open(path) + if err != nil { + return Config{}, fmt.Errorf("%w: %v", ErrConfig, err) + } + defer f.Close() + return loadConfig(f) +} + +func loadConfig(r io.Reader) (Config, error) { + data, err := io.ReadAll(io.LimitReader(r, maxConfigBytes+1)) + if err != nil || len(data) > maxConfigBytes { + return Config{}, fmt.Errorf("%w: configuration size", ErrConfig) + } + d := json.NewDecoder(bytes.NewReader(data)) + d.DisallowUnknownFields() + var cfg Config + if err := d.Decode(&cfg); err != nil { + return Config{}, fmt.Errorf("%w: %v", ErrConfig, err) + } + cfg.Defaults() + return cfg, cfg.Validate() +} + +// Defaults fills the zero fields. +func (c *Config) Defaults() { + if c.Backend == "" { + c.Backend = BackendPodman + } + if c.DefaultTimeoutMs <= 0 { + c.DefaultTimeoutMs = DefaultTimeout.Milliseconds() + } + if c.MaxTimeoutMs <= 0 { + c.MaxTimeoutMs = DefaultMaxTimeout.Milliseconds() + } + if c.DefaultMaxOutputBytes <= 0 { + c.DefaultMaxOutputBytes = DefaultMaxOutputBytes + } + if c.MaxOutputBytesCap <= 0 { + c.MaxOutputBytesCap = DefaultMaxOutputCap + } +} + +// Validate reports the errors that would otherwise surface as a failed run. +func (c Config) Validate() error { + if strings.TrimSpace(c.Image) == "" { + return fmt.Errorf("%w: image is required", ErrConfig) + } + switch c.Backend { + case BackendPodman: + case BackendJob: + if c.Job == nil { + return fmt.Errorf("%w: backend %q needs a job block", ErrConfig, BackendJob) + } + default: + return fmt.Errorf("%w: backend must be %q or %q", ErrConfig, BackendPodman, BackendJob) + } + if c.DefaultTimeoutMs > c.MaxTimeoutMs { + return fmt.Errorf("%w: default_timeout_ms exceeds max_timeout_ms", ErrConfig) + } + if c.DefaultMaxOutputBytes > c.MaxOutputBytesCap { + return fmt.Errorf("%w: default_max_output_bytes exceeds max_output_bytes_cap", ErrConfig) + } + for _, root := range c.WorkspaceRoots { + if !strings.HasPrefix(root, "/") { + return fmt.Errorf("%w: workspace_roots entries must be absolute: %q", ErrConfig, root) + } + } + return nil +} diff --git a/src/crux/runs/config_test.go b/src/crux/runs/config_test.go new file mode 100644 index 0000000000..0298fecbc7 --- /dev/null +++ b/src/crux/runs/config_test.go @@ -0,0 +1,244 @@ +package runs + +import ( + "context" + "errors" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/hivecommons/hive/crux/api" + "github.com/hivecommons/hive/crux/config" + "github.com/hivecommons/hive/crux/kubejob" + "github.com/hivecommons/hive/crux/sandbox" +) + +func TestLoadConfig(t *testing.T) { + cases := []struct { + name string + body string + err string + }{ + {"podman defaults", `{"image":"img"}`, ""}, + {"job", `{"backend":"job","image":"img","job":{"workspace_claim":"pvc"}}`, ""}, + {"unknown field", `{"image":"img","bogus":1}`, "bogus"}, + {"no image", `{"backend":"podman"}`, "image is required"}, + {"bad backend", `{"backend":"docker","image":"img"}`, "backend must be"}, + {"job without block", `{"backend":"job","image":"img"}`, "needs a job block"}, + {"timeout order", `{"image":"img","default_timeout_ms":5,"max_timeout_ms":1}`, "default_timeout_ms exceeds"}, + {"output order", `{"image":"img","default_max_output_bytes":5,"max_output_bytes_cap":1}`, "default_max_output_bytes exceeds"}, + {"relative root", `{"image":"img","workspace_roots":["ws"]}`, "must be absolute"}, + {"too big", `{"image":"` + strings.Repeat("x", maxConfigBytes) + `"}`, "configuration size"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + cfg, err := loadConfig(strings.NewReader(tc.body)) + if tc.err == "" { + if err != nil { + t.Fatal(err) + } + if cfg.DefaultTimeoutMs != DefaultTimeout.Milliseconds() || cfg.MaxOutputBytesCap != DefaultMaxOutputCap { + t.Fatalf("defaults not applied: %+v", cfg) + } + return + } + if err == nil || !errors.Is(err, ErrConfig) || !strings.Contains(err.Error(), tc.err) { + t.Fatalf("err = %v, want %q", err, tc.err) + } + }) + } + path := filepath.Join(t.TempDir(), "runs.json") + if err := os.WriteFile(path, []byte(`{"image":"img","repo_allowlist":["https://github.com/"]}`), 0o600); err != nil { + t.Fatal(err) + } + cfg, err := LoadConfig(path) + if err != nil || cfg.Backend != BackendPodman || len(cfg.RepoAllowlist) != 1 { + t.Fatalf("LoadConfig = %+v, %v", cfg, err) + } + if _, err := LoadConfig(filepath.Join(t.TempDir(), "missing.json")); !errors.Is(err, ErrConfig) { + t.Fatalf("missing file = %v", err) + } +} + +func TestBackendFromConfig(t *testing.T) { + podman, err := BackendFromConfig(Config{Backend: BackendPodman, Image: "img"}) + if err != nil { + t.Fatal(err) + } + if podman.Name != BackendPodman || podman.Teardown == nil { + t.Fatalf("podman = %+v", podman) + } + if l, ok := podman.Launcher.(sandbox.PodmanLauncher); !ok || l.Binary != sandbox.PodmanBinary { + t.Fatalf("launcher = %#v", podman.Launcher) + } + if _, ok := podman.Supports(api.RunNetworkNone); !ok { + t.Fatal("podman cannot do network none") + } + if what, ok := podman.Supports(api.RunNetworkRestricted); ok || !strings.Contains(what, `"restricted"`) { + t.Fatalf("restricted without the flag: %q %v", what, ok) + } + if what, ok := podman.Supports("bridge"); ok || !strings.Contains(what, "unknown network mode") { + t.Fatalf("unknown mode: %q %v", what, ok) + } + restricted, _ := BackendFromConfig(Config{Backend: BackendPodman, Image: "img", NetworkRestricted: true, PodmanBinary: "/opt/podman"}) + if _, ok := restricted.Supports(api.RunNetworkRestricted); !ok { + t.Fatal("restricted flag ignored") + } + if l := restricted.Launcher.(sandbox.PodmanLauncher); l.Binary != "/opt/podman" { + t.Fatalf("binary override = %q", l.Binary) + } + + job := &config.SandboxJobConfig{WorkspaceClaim: "pvc", TTLSeconds: 60} + job.Tolerations = []config.SandboxJobToleration{{Key: "k", Operator: "Exists", Effect: "NoSchedule"}} + job.Volumes = []config.SandboxJobVolume{{Name: "cache", Claim: "c", MountPath: "/cache", ReadOnly: true}} + kube, err := BackendFromConfig(Config{Backend: BackendJob, Image: "img", Job: job}) + if err != nil { + t.Fatal(err) + } + if kube.Teardown != nil || kube.Name != BackendJob { + t.Fatalf("job backend = %+v", kube) + } + if what, ok := kube.Supports(api.RunNetworkNone); ok || !strings.Contains(what, `cannot enforce network "none"`) { + t.Fatalf("job none without a policy: %q %v", what, ok) + } + opts := kube.Launcher.(*kubejob.Launcher).Options + if opts.WorkspaceClaim != "pvc" || opts.TTLSeconds != 60 || len(opts.Tolerations) != 1 || opts.Tolerations[0].Key != "k" || + len(opts.ExtraVolumes) != 1 || opts.ExtraVolumes[0].MountPath != "/cache" || !opts.ExtraVolumes[0].ReadOnly { + t.Fatalf("job options = %+v", opts) + } + policy, _ := BackendFromConfig(Config{Backend: BackendJob, Image: "img", Job: job, JobNetworkPolicy: true}) + if _, ok := policy.Supports(api.RunNetworkNone); !ok { + t.Fatal("job_network_policy ignored") + } + if _, ok := policy.Supports(api.RunNetworkRestricted); ok { + t.Fatal("job claims restricted") + } + + for name, cfg := range map[string]Config{ + "job without block": {Backend: BackendJob, Image: "img"}, + "job invalid": {Backend: BackendJob, Image: "img", Job: &config.SandboxJobConfig{}}, + "unknown": {Backend: "docker", Image: "img"}, + } { + if _, err := BackendFromConfig(cfg); !errors.Is(err, ErrConfig) { + t.Errorf("%s: err = %v", name, err) + } + } +} + +func TestPodmanTeardownReportsFailures(t *testing.T) { + for _, bin := range []string{"true", "false"} { + if _, err := exec.LookPath(bin); err != nil { + t.Skipf("%s not on PATH", bin) + } + } + if err := podmanTeardown("true")(context.Background(), "crux-run-x"); err != nil { + t.Fatalf("true: %v", err) + } + err := podmanTeardown("false")(context.Background(), "crux-run-x") + if err == nil || !strings.Contains(err.Error(), "podman rm crux-run-x") { + t.Fatalf("false: %v", err) + } +} + +func TestGitProvisionerWithARecordedRunner(t *testing.T) { + dir := t.TempDir() + var calls [][]string + runner := func(_ context.Context, cwd, name string, args ...string) ([]byte, error) { + calls = append(calls, append([]string{filepath.Base(cwd), name}, args...)) + if args[0] == "rev-parse" { + return []byte(testSHA + "\n"), nil + } + return nil, nil + } + wt := api.ContainedRunWorktree{Repo: "https://github.com/hivecommons/hive", Commit: testSHA} + path, err := GitProvisioner{Run: runner}.Provision(context.Background(), dir, wt) + if err != nil || path != filepath.Join(dir, WorkspaceDirName) { + t.Fatalf("Provision = %q, %v", path, err) + } + want := [][]string{ + {filepath.Base(dir), "git", "clone", "--no-checkout", "--", wt.Repo, path}, + {WorkspaceDirName, "git", "checkout", "--detach", testSHA}, + {WorkspaceDirName, "git", "rev-parse", "HEAD"}, + } + if len(calls) != len(want) { + t.Fatalf("calls = %v", calls) + } + for i := range want { + if strings.Join(calls[i], " ") != strings.Join(want[i], " ") { + t.Fatalf("call %d = %v, want %v", i, calls[i], want[i]) + } + } + + failing := func(step string, out string) Runner { + return func(_ context.Context, _, _ string, args ...string) ([]byte, error) { + if args[0] == step { + return []byte(out), errors.New("exit 128") + } + if args[0] == "rev-parse" { + return []byte("deadbeef\n"), nil + } + return nil, nil + } + } + for _, tc := range []struct{ step, want string }{ + {"clone", "git clone: exit 128: not found"}, + {"checkout", "git checkout " + testSHA + ": exit 128: not found"}, + {"rev-parse", "git rev-parse HEAD: exit 128"}, + {"none", "checkout pinned " + testSHA + " but HEAD is deadbeef"}, + } { + if _, err := (GitProvisioner{Run: failing(tc.step, "not found")}).Provision(context.Background(), t.TempDir(), wt); err == nil || err.Error() != tc.want { + t.Errorf("%s: err = %v, want %q", tc.step, err, tc.want) + } + } + if _, err := (GitProvisioner{Run: runner}).Provision(context.Background(), filepath.Join(writeTempFile(t), "sub"), wt); err == nil { + t.Fatal("mkdir under a file succeeded") + } +} + +func TestGitProvisionerClonesALocalRepository(t *testing.T) { + if _, err := exec.LookPath("git"); err != nil { + t.Skip("git not on PATH") + } + src := t.TempDir() + git := func(args ...string) string { + t.Helper() + cmd := exec.Command("git", args...) + cmd.Dir = src + cmd.Env = append(os.Environ(), "GIT_AUTHOR_NAME=t", "GIT_AUTHOR_EMAIL=t@x", "GIT_COMMITTER_NAME=t", "GIT_COMMITTER_EMAIL=t@x") + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("git %v: %v: %s", args, err, out) + } + return strings.TrimSpace(string(out)) + } + git("init", "-q", "-b", "main") + if err := os.WriteFile(filepath.Join(src, "f"), []byte("1"), 0o644); err != nil { + t.Fatal(err) + } + git("add", "f") + git("commit", "-q", "-m", "one") + sha := git("rev-parse", "HEAD") + + dir := t.TempDir() + path, err := GitProvisioner{}.Provision(context.Background(), dir, api.ContainedRunWorktree{Repo: src, Commit: sha}) + if err != nil { + t.Fatal(err) + } + if b, err := os.ReadFile(filepath.Join(path, "f")); err != nil || string(b) != "1" { + t.Fatalf("checked-out file = %q, %v", b, err) + } + if _, err := (GitProvisioner{}).Provision(context.Background(), t.TempDir(), api.ContainedRunWorktree{Repo: src, Commit: testSHA}); err == nil || !strings.Contains(err.Error(), "git checkout") { + t.Fatalf("unknown commit = %v", err) + } +} + +func writeTempFile(t *testing.T) string { + t.Helper() + path := filepath.Join(t.TempDir(), "file") + if err := os.WriteFile(path, nil, 0o600); err != nil { + t.Fatal(err) + } + return path +} diff --git a/src/crux/runs/provision.go b/src/crux/runs/provision.go new file mode 100644 index 0000000000..9c8780a8a7 --- /dev/null +++ b/src/crux/runs/provision.go @@ -0,0 +1,78 @@ +package runs + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + + "github.com/hivecommons/hive/crux/api" + "github.com/hivecommons/hive/crux/sandbox" +) + +// gitEnvAllowlist is what the clone may see of the service's environment. +var gitEnvAllowlist = []string{"PATH", "HOME", "LANG", "LC_ALL", "TMPDIR", "SSL_CERT_FILE", "SSL_CERT_DIR"} + +// WorkspaceDirName is the workspace subdirectory of a run's directory when +// the ad-hoc worktree form provisions one. +const WorkspaceDirName = "workspace" + +// Provisioner materialises the ad-hoc worktree form under dir and returns +// the workspace path. It is the stand-in for the checkout service (S1): +// when that lands, the service delegates to it and worktrees get an owner, +// a size cap and a TTL. +type Provisioner interface { + Provision(ctx context.Context, dir string, wt api.ContainedRunWorktree) (string, error) +} + +// Runner executes one command in dir and returns its combined output; the +// default runs it, tests substitute a script. +type Runner func(ctx context.Context, dir, name string, args ...string) ([]byte, error) + +// GitProvisioner clones the repository with git and pins the commit. +type GitProvisioner struct { + Run Runner +} + +// Provision clones wt.Repo into dir/workspace with every ref, detaches at +// wt.Commit and verifies HEAD is that commit. No credential reaches git: +// the environment is the service's and the repository must be public. +func (p GitProvisioner) Provision(ctx context.Context, dir string, wt api.ContainedRunWorktree) (string, error) { + path := filepath.Join(dir, WorkspaceDirName) + if err := os.MkdirAll(dir, 0o770); err != nil { + return "", err + } + run := p.Run + if run == nil { + run = execRunner + } + if out, err := run(ctx, dir, "git", "clone", "--no-checkout", "--", wt.Repo, path); err != nil { + return "", fmt.Errorf("git clone: %w: %s", err, strings.TrimSpace(string(out))) + } + if out, err := run(ctx, path, "git", "checkout", "--detach", wt.Commit); err != nil { + _ = os.RemoveAll(path) + return "", fmt.Errorf("git checkout %s: %w: %s", wt.Commit, err, strings.TrimSpace(string(out))) + } + head, err := run(ctx, path, "git", "rev-parse", "HEAD") + if err != nil { + _ = os.RemoveAll(path) + return "", fmt.Errorf("git rev-parse HEAD: %w", err) + } + if got := strings.TrimSpace(string(head)); got != wt.Commit { + _ = os.RemoveAll(path) + return "", fmt.Errorf("checkout pinned %s but HEAD is %s", wt.Commit, got) + } + return path, nil +} + +func execRunner(ctx context.Context, dir, name string, args ...string) ([]byte, error) { + cmd := exec.CommandContext(ctx, name, args...) + cmd.Dir = dir + // Never prompt and never carry a credential: the ad-hoc form is for + // public repositories on the allowlist, so git gets the sandbox's + // credential-free environment plus a no-prompt switch. + cmd.Env = append(sandbox.SanitizedEnv(os.Environ(), gitEnvAllowlist, nil), "GIT_TERMINAL_PROMPT=0") + return cmd.CombinedOutput() +} diff --git a/src/crux/runs/service.go b/src/crux/runs/service.go new file mode 100644 index 0000000000..29db74e01b --- /dev/null +++ b/src/crux/runs/service.go @@ -0,0 +1,494 @@ +package runs + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "regexp" + "strings" + "sync" + "time" + + "github.com/hivecommons/hive/crux/api" + "github.com/hivecommons/hive/crux/logscrub" + "github.com/hivecommons/hive/crux/runstate" + "github.com/hivecommons/hive/crux/sandbox" +) + +// Artifact names inside a run's directory: the full scrubbed output. +const ( + StdoutArtifact = "stdout.txt" + StderrArtifact = "stderr.txt" + // launchNamePrefix names the container/Job after the run so a teardown + // can find it. + launchNamePrefix = "crux-run-" + // ReconcileReason is recorded on runs that were in flight when the + // serving process stopped. + ReconcileReason = "crux serve restarted while the run was in flight; the exit is not knowable" + // exitUnknown is the launcher convention for "exit not read". + exitUnknown = -1 +) + +// Typed errors. The serve routes map each onto an api.Code*. +var ( + // ErrNotConfigured: the server has no backend; records are still readable. + ErrNotConfigured = errors.New("runs: no contained-run backend is configured on this server") + // ErrNotFound: unknown run ID. + ErrNotFound = runstate.ErrNotFound +) + +// RequestError is a malformed request (400). +type RequestError struct{ Msg string } + +func (e *RequestError) Error() string { return e.Msg } + +// ForbiddenError is a workspace the configuration does not allow (403). +type ForbiddenError struct{ Msg string } + +func (e *ForbiddenError) Error() string { return e.Msg } + +// UnsupportedError names a requested feature or limit the selected backend +// cannot enforce (501). The run never starts. +type UnsupportedError struct{ What string } + +func (e *UnsupportedError) Error() string { return e.What } + +var fullSHA = regexp.MustCompile(`^[0-9a-f]{40}$`) + +// detail is the service's payload inside the runstate record. +type detail struct { + Profile string `json:"profile"` + Network string `json:"network"` + Backend string `json:"backend"` + Command []string `json:"command"` + Workspace string `json:"workspace,omitempty"` + ExitCode *int `json:"exit_code,omitempty"` + TimedOut bool `json:"timed_out,omitempty"` + Stdout string `json:"stdout,omitempty"` + Stderr string `json:"stderr,omitempty"` + StdoutTruncatedAt *int64 `json:"stdout_truncated_at,omitempty"` + StderrTruncatedAt *int64 `json:"stderr_truncated_at,omitempty"` +} + +// plan is a validated request. +type plan struct { + worktree *api.ContainedRunWorktree + workspace string + command []string + allowlist []string + network string + timeout time.Duration + maxOutput int64 + profile string +} + +type active struct { + cancel context.CancelFunc + done chan struct{} + cancelled bool +} + +// Service accepts, executes, cancels and reports contained runs. +type Service struct { + Store *runstate.Store + Backend *Backend + Config Config + // Provisioner materialises the ad-hoc worktree form; nil uses git. + Provisioner Provisioner + // Environ supplies the base environment the allowlist filters; nil uses + // os.Environ. + Environ func() []string + // Scrub redacts output before it is persisted; nil uses logscrub. + Scrub func(string) string + + mu sync.Mutex + active map[string]*active +} + +// New returns a Service over store. backend nil means not configured: the +// saved records stay readable and every create answers ErrNotConfigured. +func New(store *runstate.Store, backend *Backend, cfg Config) *Service { + cfg.Defaults() + return &Service{Store: store, Backend: backend, Config: cfg, active: map[string]*active{}} +} + +// Configured reports whether runs can be executed here. +func (s *Service) Configured() bool { return s.Backend != nil } + +// Reconcile marks every run left accepted or running by a previous process +// unfinished. Call it once at serve start, before accepting requests. +func (s *Service) Reconcile() ([]runstate.Record, error) { + return s.Store.Reconcile(ReconcileReason) +} + +// Create validates req, saves the accepted record and starts the run under +// launchCtx (which must outlive the request). created is false when +// req.IdempotencyKey matched an existing run, whose record is returned. +func (s *Service) Create(launchCtx context.Context, owner string, req api.ContainedRunRequest) (api.ContainedRun, bool, error) { + if !s.Configured() { + return api.ContainedRun{}, false, ErrNotConfigured + } + p, err := s.validate(req) + if err != nil { + return api.ContainedRun{}, false, err + } + d := detail{Profile: p.profile, Network: p.network, Backend: s.Backend.Name, Command: p.command, Workspace: p.workspace} + raw, err := json.Marshal(d) + if err != nil { + return api.ContainedRun{}, false, err + } + rec, created, err := s.Store.Create(runstate.Record{ + Kind: runstate.KindRun, + Owner: owner, + IdempotencyKey: runstate.DeriveIdempotencyKey(owner, req.IdempotencyKey), + Detail: raw, + }) + if err != nil { + return api.ContainedRun{}, false, err + } + if !created { + return toWire(rec), false, nil + } + ctx, cancel := context.WithCancel(launchCtx) + a := &active{cancel: cancel, done: make(chan struct{})} + s.mu.Lock() + s.active[rec.ID] = a + s.mu.Unlock() + go s.execute(ctx, rec.ID, p, a) + return toWire(rec), true, nil +} + +// Get returns the saved record whatever the process state. +func (s *Service) Get(id string) (api.ContainedRun, error) { + rec, err := s.Store.Get(id) + if err != nil { + return api.ContainedRun{}, err + } + return toWire(rec), nil +} + +// Wait blocks until the run reaches a terminal status or ctx ends, then +// returns the saved record. +func (s *Service) Wait(ctx context.Context, id string) (api.ContainedRun, error) { + s.mu.Lock() + a := s.active[id] + s.mu.Unlock() + if a != nil { + select { + case <-a.done: + case <-ctx.Done(): + return api.ContainedRun{}, ctx.Err() + } + } + return s.Get(id) +} + +// Cancel stops a run in flight and waits for its record to settle. A run +// that already finished is returned unchanged with AlreadyFinished set. +func (s *Service) Cancel(ctx context.Context, id string) (api.ContainedRun, error) { + rec, err := s.Store.Get(id) + if err != nil { + return api.ContainedRun{}, err + } + if runstate.Terminal(rec.Status) { + out := toWire(rec) + out.AlreadyFinished = true + return out, nil + } + s.mu.Lock() + a := s.active[id] + if a != nil { + a.cancelled = true + a.cancel() + } + s.mu.Unlock() + if a == nil { + // Accepted or running with no goroutine: not possible after + // Reconcile ran at start; report the record as it is. + return toWire(rec), nil + } + return s.Wait(ctx, id) +} + +func (s *Service) validate(req api.ContainedRunRequest) (plan, error) { + var p plan + if len(req.Command) == 0 || strings.TrimSpace(req.Command[0]) == "" { + return p, &RequestError{"command is required"} + } + p.command = req.Command + switch p.profile = req.Profile; p.profile { + case "": + p.profile = api.RunProfileExperiment + case api.RunProfileExperiment: + case api.RunProfileAgent: + return p, &UnsupportedError{fmt.Sprintf("profile %q is not available yet (schema-checked agent steps, S6)", req.Profile)} + default: + return p, &RequestError{fmt.Sprintf("profile must be %q or %q", api.RunProfileExperiment, api.RunProfileAgent)} + } + switch p.network = req.Network; p.network { + case "": + p.network = api.RunNetworkNone + case api.RunNetworkNone, api.RunNetworkRestricted: + default: + return p, &RequestError{fmt.Sprintf("network must be %q or %q", api.RunNetworkNone, api.RunNetworkRestricted)} + } + if what, ok := s.Backend.Supports(p.network); !ok { + return p, &UnsupportedError{what} + } + for _, name := range req.EnvAllowlist { + if sandbox.IsCredentialName(name) { + return p, &RequestError{fmt.Sprintf("env_allowlist names credential %q; the experiment profile carries no credentials", name)} + } + } + p.allowlist = req.EnvAllowlist + p.timeout = time.Duration(s.Config.DefaultTimeoutMs) * time.Millisecond + if req.TimeoutMs > 0 { + if req.TimeoutMs > s.Config.MaxTimeoutMs { + return p, &RequestError{fmt.Sprintf("timeout_ms %d exceeds this server's cap of %d", req.TimeoutMs, s.Config.MaxTimeoutMs)} + } + p.timeout = time.Duration(req.TimeoutMs) * time.Millisecond + } + p.maxOutput = s.Config.DefaultMaxOutputBytes + if req.MaxOutputBytes > 0 { + if req.MaxOutputBytes > s.Config.MaxOutputBytesCap { + return p, &RequestError{fmt.Sprintf("max_output_bytes %d exceeds this server's cap of %d", req.MaxOutputBytes, s.Config.MaxOutputBytesCap)} + } + p.maxOutput = req.MaxOutputBytes + } + return p, s.validateWorkspace(req, &p) +} + +func (s *Service) validateWorkspace(req api.ContainedRunRequest, p *plan) error { + sources := 0 + for _, set := range []bool{req.Checkout != "", req.Worktree != nil, req.WorkspacePath != ""} { + if set { + sources++ + } + } + switch { + case sources == 0: + return &RequestError{"one of worktree or workspace_path is required"} + case sources > 1: + return &RequestError{"checkout, worktree and workspace_path are mutually exclusive"} + case req.Checkout != "": + return &UnsupportedError{"checkout ids need the checkout service (S1), which is not available yet; use worktree {repo, commit} or workspace_path"} + case req.Worktree != nil: + wt := *req.Worktree + if strings.TrimSpace(wt.Repo) == "" { + return &RequestError{"worktree.repo is required"} + } + if !fullSHA.MatchString(wt.Commit) { + return &RequestError{"worktree.commit must be a full 40-hex commit SHA; a tag or branch name cannot pin the run"} + } + if !repoAllowed(wt.Repo, s.Config.RepoAllowlist) { + return &ForbiddenError{fmt.Sprintf("repository %q is not on this server's repo_allowlist", wt.Repo)} + } + p.worktree = &wt + return nil + } + path := filepath.Clean(req.WorkspacePath) + if !filepath.IsAbs(path) { + return &RequestError{"workspace_path must be absolute"} + } + if !pathUnder(path, s.Config.WorkspaceRoots) { + return &ForbiddenError{fmt.Sprintf("workspace_path %q is not under this server's workspace_roots", path)} + } + if st, err := os.Stat(path); err != nil || !st.IsDir() { + return &RequestError{fmt.Sprintf("workspace_path %q is not a directory", path)} + } + p.workspace = path + return nil +} + +// repoAllowed reports whether repo starts with one of the allowlisted +// URL prefixes. +func repoAllowed(repo string, prefixes []string) bool { + for _, p := range prefixes { + if p = strings.TrimSpace(p); p != "" && strings.HasPrefix(repo, p) { + return true + } + } + return false +} + +// pathUnder reports whether path is one of roots or lies beneath one. +func pathUnder(path string, roots []string) bool { + for _, root := range roots { + root = filepath.Clean(strings.TrimSpace(root)) + if root == "" || root == "." { + continue + } + if path == root || strings.HasPrefix(path, strings.TrimSuffix(root, "/")+"/") { + return true + } + } + return false +} + +// execute runs one accepted run to a terminal status. Every exit path +// writes the record; nothing is left accepted or running by this process. +func (s *Service) execute(ctx context.Context, id string, p plan, a *active) { + defer func() { + s.mu.Lock() + delete(s.active, id) + s.mu.Unlock() + close(a.done) + }() + finish := func(status, reason string, fill func(*detail)) { + _, _ = s.Store.Update(id, func(rec *runstate.Record) error { + var d detail + _ = json.Unmarshal(rec.Detail, &d) + if fill != nil { + fill(&d) + } + raw, err := json.Marshal(d) + if err != nil { + return err + } + rec.Status, rec.Reason, rec.Detail = status, reason, raw + return nil + }) + } + workspace := p.workspace + if p.worktree != nil { + prov := s.Provisioner + if prov == nil { + prov = GitProvisioner{} + } + path, err := prov.Provision(ctx, s.Store.Path(id), *p.worktree) + if err != nil { + if ctx.Err() != nil { + finish(runstate.StatusCancelled, "cancelled while provisioning the worktree", nil) + } else { + finish(runstate.StatusFailed, "provisioning the worktree: "+s.scrub(err.Error()), nil) + } + return + } + workspace = path + } + if _, err := s.Store.Update(id, func(rec *runstate.Record) error { + var d detail + _ = json.Unmarshal(rec.Detail, &d) + d.Workspace = workspace + raw, err := json.Marshal(d) + rec.Status, rec.Detail = runstate.StatusRunning, raw + return err + }); err != nil { + finish(runstate.StatusFailed, "recording the start: "+err.Error(), nil) + return + } + + runCtx, cancelTimeout := context.WithTimeout(ctx, p.timeout) + defer cancelTimeout() + name := launchNamePrefix + id + spec := sandbox.LaunchSpec{ + Image: s.Config.Image, + Name: name, + Workspace: workspace, + Command: p.command, + Env: s.allowlistedEnv(p.allowlist), + EnvAllowlist: p.allowlist, + NetworkMode: p.network, + } + res, err := s.Backend.Launcher.Run(runCtx, spec) + // A launcher that returned without error finished the command; a + // cancel or deadline that landed after that does not unmake the result. + cancelled := err != nil && ctx.Err() != nil + timedOut := err != nil && !cancelled && errors.Is(runCtx.Err(), context.DeadlineExceeded) + status, reason, exit := classify(res, err, cancelled, timedOut) + if (cancelled || timedOut) && s.Backend.Teardown != nil { + tctx, cancel := context.WithTimeout(context.Background(), TeardownTimeout) + if terr := s.Backend.Teardown(tctx, name); terr != nil { + reason += "; teardown: " + s.scrub(terr.Error()) + } + cancel() + } + stdout, stdoutCut := s.persistOutput(id, StdoutArtifact, res.Stdout, p.maxOutput) + stderr, stderrCut := s.persistOutput(id, StderrArtifact, res.Stderr, p.maxOutput) + finish(status, reason, func(d *detail) { + d.ExitCode, d.TimedOut = exit, timedOut + d.Stdout, d.StdoutTruncatedAt = stdout, stdoutCut + d.Stderr, d.StderrTruncatedAt = stderr, stderrCut + }) +} + +// classify turns the launcher's answer into a status. The exit code is +// reported only when the launcher read one: a zero Result beside an error +// means the launcher refused before anything ran, and the launchers' +// exitUnknown means the process ended without a readable exit. Neither is +// ever turned into a guessed code. +func classify(res sandbox.Result, err error, cancelled, timedOut bool) (status, reason string, exit *int) { + launched := err == nil || res != (sandbox.Result{}) + if launched && res.ExitCode != exitUnknown { + code := res.ExitCode + exit = &code + } + switch { + case cancelled: + return runstate.StatusCancelled, "cancelled by the caller", exit + case timedOut: + return runstate.StatusFailed, "timed out", exit + case err == nil: + return runstate.StatusCompleted, "", exit + case exit != nil: + // The command ran and its exit was read; the error is the backend's + // own description (a Job that reports Failed on a non-zero exit). + return runstate.StatusCompleted, logscrub.ScrubString(err.Error()), exit + case !launched: + return runstate.StatusFailed, logscrub.ScrubString(err.Error()), nil + } + return runstate.StatusUnfinished, logscrub.ScrubString(err.Error()), nil +} + +// persistOutput scrubs raw, writes the whole of it as an artifact and +// returns the response copy, cut at max with the offset when it was. +func (s *Service) persistOutput(id, name, raw string, max int64) (string, *int64) { + text := s.scrub(raw) + _ = os.WriteFile(filepath.Join(s.Store.Path(id), name), []byte(text), 0o660) + if int64(len(text)) <= max { + return text, nil + } + cut := max + return text[:max], &cut +} + +func (s *Service) scrub(v string) string { + if s.Scrub != nil { + return s.Scrub(v) + } + return logscrub.ScrubString(v) +} + +// allowlistedEnv is the explicit environment the container receives: the +// allowlisted names present in the service's environment, credentials +// excluded by sandbox.SanitizedEnv. The podman launcher forwards only +// explicit entries into the container, so the allowlist alone is not +// enough. +func (s *Service) allowlistedEnv(allowlist []string) map[string]string { + environ := os.Environ + if s.Environ != nil { + environ = s.Environ + } + out := map[string]string{} + for _, entry := range sandbox.SanitizedEnv(environ(), allowlist, nil) { + if k, v, ok := strings.Cut(entry, "="); ok { + out[k] = v + } + } + return out +} + +func toWire(rec runstate.Record) api.ContainedRun { + var d detail + _ = json.Unmarshal(rec.Detail, &d) + return api.ContainedRun{ + ID: rec.ID, Status: rec.Status, Owner: rec.Owner, Reason: rec.Reason, + CreatedAt: rec.CreatedAt, StartedAt: rec.StartedAt, FinishedAt: rec.FinishedAt, + Profile: d.Profile, Network: d.Network, Backend: d.Backend, Command: d.Command, Workdir: d.Workspace, + ExitCode: d.ExitCode, TimedOut: d.TimedOut, Stdout: d.Stdout, Stderr: d.Stderr, + StdoutTruncatedAt: d.StdoutTruncatedAt, StderrTruncatedAt: d.StderrTruncatedAt, + } +} diff --git a/src/crux/runs/service_test.go b/src/crux/runs/service_test.go new file mode 100644 index 0000000000..9f2d6ef609 --- /dev/null +++ b/src/crux/runs/service_test.go @@ -0,0 +1,458 @@ +package runs + +import ( + "context" + "errors" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/hivecommons/hive/crux/api" + "github.com/hivecommons/hive/crux/runstate" + "github.com/hivecommons/hive/crux/sandbox" +) + +// fakeLauncher drives every classify path without a container runtime. +type fakeLauncher struct { + run func(ctx context.Context, spec sandbox.LaunchSpec) (sandbox.Result, error) + specs chan sandbox.LaunchSpec +} + +func newFakeLauncher(run func(ctx context.Context, spec sandbox.LaunchSpec) (sandbox.Result, error)) *fakeLauncher { + return &fakeLauncher{run: run, specs: make(chan sandbox.LaunchSpec, 8)} +} + +func (f *fakeLauncher) Run(ctx context.Context, spec sandbox.LaunchSpec) (sandbox.Result, error) { + f.specs <- spec + return f.run(ctx, spec) +} + +// blocking returns a launcher that waits for ctx and reports its error. +func blocking() *fakeLauncher { + return newFakeLauncher(func(ctx context.Context, _ sandbox.LaunchSpec) (sandbox.Result, error) { + <-ctx.Done() + return sandbox.Result{ExitCode: -1}, ctx.Err() + }) +} + +type fixture struct { + t *testing.T + svc *Service + launcher *fakeLauncher + torn chan string + root string +} + +const testSHA = "0123456789abcdef0123456789abcdef01234567" + +func newFixture(t *testing.T, launcher *fakeLauncher) *fixture { + t.Helper() + workDir := t.TempDir() + store, err := runstate.Open(workDir) + if err != nil { + t.Fatal(err) + } + root := filepath.Join(t.TempDir(), "ws") + if err := os.MkdirAll(filepath.Join(root, "proj"), 0o755); err != nil { + t.Fatal(err) + } + f := &fixture{t: t, launcher: launcher, torn: make(chan string, 8), root: root} + backend := &Backend{ + Name: "fake", + Launcher: launcher, + Capabilities: Capabilities{NetworkNone: true}, + Teardown: func(_ context.Context, name string) error { f.torn <- name; return nil }, + } + cfg := Config{ + Image: "img:1", + RepoAllowlist: []string{"https://github.com/hivecommons/"}, + WorkspaceRoots: []string{root}, + MaxTimeoutMs: time.Minute.Milliseconds(), + } + f.svc = New(store, backend, cfg) + f.svc.Environ = func() []string { return []string{"KEEP=yes", "GITHUB_TOKEN=secret", "DROP=no"} } + return f +} + +func (f *fixture) workspace() string { return filepath.Join(f.root, "proj") } + +func (f *fixture) start(req api.ContainedRunRequest) api.ContainedRun { + f.t.Helper() + run, created, err := f.svc.Create(context.Background(), "operator", req) + if err != nil || !created { + f.t.Fatalf("Create: %v created=%v", err, created) + } + if run.Status != api.RunAccepted || run.ID == "" { + f.t.Fatalf("accepted record = %+v", run) + } + return run +} + +func (f *fixture) wait(id string) api.ContainedRun { + f.t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + run, err := f.svc.Wait(ctx, id) + if err != nil { + f.t.Fatal(err) + } + return run +} + +func (f *fixture) spec() sandbox.LaunchSpec { + f.t.Helper() + select { + case s := <-f.launcher.specs: + return s + case <-time.After(10 * time.Second): + f.t.Fatal("launcher was never called") + } + return sandbox.LaunchSpec{} +} + +func TestCompletedRunCarriesOutputAndSanitizedEnv(t *testing.T) { + f := newFixture(t, newFakeLauncher(func(_ context.Context, _ sandbox.LaunchSpec) (sandbox.Result, error) { + return sandbox.Result{Stdout: "hello\n", Stderr: "warn\n"}, nil + })) + run := f.start(api.ContainedRunRequest{ + WorkspacePath: f.workspace(), Command: []string{"echo", "hello"}, + EnvAllowlist: []string{"KEEP", "MISSING"}, + }) + spec := f.spec() + if spec.Image != "img:1" || spec.Name != "crux-run-"+run.ID || spec.Workspace != f.workspace() || spec.NetworkMode != "none" { + t.Fatalf("spec = %+v", spec) + } + if spec.Env["KEEP"] != "yes" || len(spec.Env) != 1 { + t.Fatalf("env = %v", spec.Env) + } + done := f.wait(run.ID) + if done.Status != api.RunCompleted || done.ExitCode == nil || *done.ExitCode != 0 || done.Stdout != "hello\n" || done.Stderr != "warn\n" { + t.Fatalf("completed = %+v", done) + } + if done.Profile != api.RunProfileExperiment || done.Network != api.RunNetworkNone || done.Backend != "fake" || done.Workdir != f.workspace() { + t.Fatalf("plan not recorded: %+v", done) + } + if done.StartedAt == nil || done.FinishedAt == nil || done.Owner != "operator" { + t.Fatalf("timestamps/owner = %+v", done) + } + if out, _ := os.ReadFile(filepath.Join(f.svc.Store.Path(run.ID), StdoutArtifact)); string(out) != "hello\n" { + t.Fatalf("stdout artifact = %q", out) + } + if got, err := f.svc.Get(run.ID); err != nil || got.Status != api.RunCompleted { + t.Fatalf("Get = %+v, %v", got, err) + } + if len(f.torn) != 0 { + t.Fatal("teardown ran for a completed run") + } +} + +func TestNonZeroExitIsCompletedWithTheCode(t *testing.T) { + f := newFixture(t, newFakeLauncher(func(_ context.Context, _ sandbox.LaunchSpec) (sandbox.Result, error) { + return sandbox.Result{Stderr: "boom", ExitCode: 3}, errors.New("podman: exit status 3") + })) + run := f.wait(f.start(api.ContainedRunRequest{WorkspacePath: f.workspace(), Command: []string{"false"}}).ID) + if run.Status != api.RunCompleted || run.ExitCode == nil || *run.ExitCode != 3 || run.Reason != "podman: exit status 3" { + t.Fatalf("non-zero = %+v", run) + } +} + +func TestLauncherRefusalIsFailedWithoutAnExitCode(t *testing.T) { + f := newFixture(t, newFakeLauncher(func(_ context.Context, _ sandbox.LaunchSpec) (sandbox.Result, error) { + return sandbox.Result{}, errors.New("image pull failed") + })) + run := f.wait(f.start(api.ContainedRunRequest{WorkspacePath: f.workspace(), Command: []string{"x"}}).ID) + if run.Status != api.RunFailed || run.ExitCode != nil || run.Reason != "image pull failed" { + t.Fatalf("refusal = %+v", run) + } +} + +func TestUnknownExitIsUnfinishedNeverGuessed(t *testing.T) { + f := newFixture(t, newFakeLauncher(func(_ context.Context, _ sandbox.LaunchSpec) (sandbox.Result, error) { + return sandbox.Result{Stdout: "partial", ExitCode: -1}, errors.New("pod vanished") + })) + run := f.wait(f.start(api.ContainedRunRequest{WorkspacePath: f.workspace(), Command: []string{"x"}}).ID) + if run.Status != api.RunUnfinished || run.ExitCode != nil || run.Stdout != "partial" || run.Reason != "pod vanished" { + t.Fatalf("unknown exit = %+v", run) + } +} + +func TestTimeoutFailsAndTearsDown(t *testing.T) { + f := newFixture(t, blocking()) + run := f.start(api.ContainedRunRequest{WorkspacePath: f.workspace(), Command: []string{"sleep"}, TimeoutMs: 20}) + done := f.wait(run.ID) + if done.Status != api.RunFailed || !done.TimedOut || done.ExitCode != nil || done.Reason != "timed out" { + t.Fatalf("timeout = %+v", done) + } + if name := <-f.torn; name != "crux-run-"+run.ID { + t.Fatalf("teardown name = %q", name) + } +} + +func TestCancelTearsDownAndLateCancelReportsAlreadyFinished(t *testing.T) { + f := newFixture(t, blocking()) + run := f.start(api.ContainedRunRequest{WorkspacePath: f.workspace(), Command: []string{"sleep"}}) + f.spec() + f.svc.Backend.Teardown = func(context.Context, string) error { return errors.New("rm: token=abc123 gone") } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + cancelled, err := f.svc.Cancel(ctx, run.ID) + if err != nil { + t.Fatal(err) + } + if cancelled.Status != api.RunCancelled || cancelled.AlreadyFinished || cancelled.ExitCode != nil { + t.Fatalf("cancel = %+v", cancelled) + } + if !strings.HasPrefix(cancelled.Reason, "cancelled by the caller; teardown: ") { + t.Fatalf("teardown error not recorded: %q", cancelled.Reason) + } + late, err := f.svc.Cancel(ctx, run.ID) + if err != nil || !late.AlreadyFinished || late.Status != api.RunCancelled { + t.Fatalf("late cancel = %+v, %v", late, err) + } + if _, err := f.svc.Cancel(ctx, "run_missing"); !errors.Is(err, ErrNotFound) { + t.Fatalf("cancel unknown = %v", err) + } + if _, err := f.svc.Get("run_missing"); !errors.Is(err, ErrNotFound) { + t.Fatalf("get unknown = %v", err) + } +} + +func TestCancelWithoutAGoroutineReportsTheRecord(t *testing.T) { + f := newFixture(t, blocking()) + rec, _, err := f.svc.Store.Create(runstate.Record{}) + if err != nil { + t.Fatal(err) + } + run, err := f.svc.Cancel(context.Background(), rec.ID) + if err != nil || run.Status != api.RunAccepted || run.AlreadyFinished { + t.Fatalf("orphan cancel = %+v, %v", run, err) + } + changed, err := f.svc.Reconcile() + if err != nil || len(changed) != 1 || changed[0].Status != runstate.StatusUnfinished || changed[0].Reason != ReconcileReason { + t.Fatalf("Reconcile = %+v, %v", changed, err) + } +} + +func TestWaitHonoursItsContext(t *testing.T) { + f := newFixture(t, blocking()) + run := f.start(api.ContainedRunRequest{WorkspacePath: f.workspace(), Command: []string{"sleep"}}) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := f.svc.Wait(ctx, run.ID); !errors.Is(err, context.Canceled) { + t.Fatalf("Wait = %v", err) + } + f.svc.Cancel(context.Background(), run.ID) +} + +func TestOutputIsScrubbedAndTruncatedWithTheOffsetRecorded(t *testing.T) { + long := strings.Repeat("a", 50) + "secret" + strings.Repeat("b", 50) + f := newFixture(t, newFakeLauncher(func(_ context.Context, _ sandbox.LaunchSpec) (sandbox.Result, error) { + return sandbox.Result{Stdout: long, Stderr: "short"}, nil + })) + f.svc.Scrub = func(v string) string { return strings.ReplaceAll(v, "secret", "[x]") } + run := f.wait(f.start(api.ContainedRunRequest{WorkspacePath: f.workspace(), Command: []string{"x"}, MaxOutputBytes: 10}).ID) + if run.Stdout != strings.Repeat("a", 10) || run.StdoutTruncatedAt == nil || *run.StdoutTruncatedAt != 10 { + t.Fatalf("truncated stdout = %q at %v", run.Stdout, run.StdoutTruncatedAt) + } + if run.Stderr != "short" || run.StderrTruncatedAt != nil { + t.Fatalf("stderr = %q at %v", run.Stderr, run.StderrTruncatedAt) + } + full, err := os.ReadFile(filepath.Join(f.svc.Store.Path(run.ID), StdoutArtifact)) + if err != nil { + t.Fatal(err) + } + if want := strings.ReplaceAll(long, "secret", "[x]"); string(full) != want { + t.Fatalf("artifact = %q", full) + } +} + +func TestDefaultScrubRedactsCredentials(t *testing.T) { + f := newFixture(t, newFakeLauncher(func(_ context.Context, _ sandbox.LaunchSpec) (sandbox.Result, error) { + return sandbox.Result{Stdout: "token ghp_abcdefghijklmnopqrstuvwxyz0123456789 ok"}, nil + })) + run := f.wait(f.start(api.ContainedRunRequest{WorkspacePath: f.workspace(), Command: []string{"x"}}).ID) + if strings.Contains(run.Stdout, "ghp_abcdefghijklmnopqrstuvwxyz0123456789") { + t.Fatalf("token survived scrubbing: %q", run.Stdout) + } +} + +func TestIdempotencyKeyReturnsTheExistingRun(t *testing.T) { + f := newFixture(t, newFakeLauncher(func(_ context.Context, _ sandbox.LaunchSpec) (sandbox.Result, error) { + return sandbox.Result{Stdout: "once"}, nil + })) + req := api.ContainedRunRequest{WorkspacePath: f.workspace(), Command: []string{"x"}, IdempotencyKey: "k1"} + first := f.wait(f.start(req).ID) + again, created, err := f.svc.Create(context.Background(), "operator", req) + if err != nil || created || again.ID != first.ID || again.Status != api.RunCompleted || again.Stdout != "once" { + t.Fatalf("replay = %+v created=%v err=%v", again, created, err) + } + other, created, err := f.svc.Create(context.Background(), "someone-else", req) + if err != nil || !created || other.ID == first.ID { + t.Fatalf("other owner shared the record: %+v created=%v err=%v", other, created, err) + } + f.wait(other.ID) + if len(f.launcher.specs) != 2 { + t.Fatalf("launches = %d, want 2", len(f.launcher.specs)) + } +} + +func TestWorktreeIsProvisionedBeforeLaunch(t *testing.T) { + f := newFixture(t, newFakeLauncher(func(_ context.Context, _ sandbox.LaunchSpec) (sandbox.Result, error) { + return sandbox.Result{}, nil + })) + var got api.ContainedRunWorktree + f.svc.Provisioner = provisionFunc(func(_ context.Context, dir string, wt api.ContainedRunWorktree) (string, error) { + got = wt + return filepath.Join(dir, "workspace"), nil + }) + wt := &api.ContainedRunWorktree{Repo: "https://github.com/hivecommons/hive", Commit: testSHA} + run := f.start(api.ContainedRunRequest{Worktree: wt, Command: []string{"make"}}) + spec := f.spec() + done := f.wait(run.ID) + if got != *wt || spec.Workspace != filepath.Join(f.svc.Store.Path(run.ID), "workspace") || done.Workdir != spec.Workspace { + t.Fatalf("worktree = %+v spec.Workspace=%q done=%+v", got, spec.Workspace, done) + } + if done.Status != api.RunCompleted { + t.Fatalf("status = %s", done.Status) + } +} + +func TestProvisionFailureIsFailedOrCancelled(t *testing.T) { + f := newFixture(t, blocking()) + f.svc.Provisioner = provisionFunc(func(context.Context, string, api.ContainedRunWorktree) (string, error) { + return "", errors.New("clone refused") + }) + wt := &api.ContainedRunWorktree{Repo: "https://github.com/hivecommons/hive", Commit: testSHA} + run := f.wait(f.start(api.ContainedRunRequest{Worktree: wt, Command: []string{"make"}}).ID) + if run.Status != api.RunFailed || run.Reason != "provisioning the worktree: clone refused" { + t.Fatalf("failed provision = %+v", run) + } + + f.svc.Provisioner = provisionFunc(func(ctx context.Context, _ string, _ api.ContainedRunWorktree) (string, error) { + <-ctx.Done() + return "", ctx.Err() + }) + run = f.start(api.ContainedRunRequest{Worktree: wt, Command: []string{"make"}}) + cancelled, err := f.svc.Cancel(context.Background(), run.ID) + if err != nil || cancelled.Status != api.RunCancelled || cancelled.Reason != "cancelled while provisioning the worktree" { + t.Fatalf("cancelled provision = %+v, %v", cancelled, err) + } + if len(f.launcher.specs) != 0 { + t.Fatal("launcher ran without a workspace") + } +} + +type provisionFunc func(ctx context.Context, dir string, wt api.ContainedRunWorktree) (string, error) + +func (p provisionFunc) Provision(ctx context.Context, dir string, wt api.ContainedRunWorktree) (string, error) { + return p(ctx, dir, wt) +} + +func TestValidationErrors(t *testing.T) { + f := newFixture(t, blocking()) + ws := f.workspace() + wt := &api.ContainedRunWorktree{Repo: "https://github.com/hivecommons/hive", Commit: testSHA} + cases := []struct { + name string + req api.ContainedRunRequest + want any + msg string + }{ + {"no command", api.ContainedRunRequest{WorkspacePath: ws}, &RequestError{}, "command is required"}, + {"blank command", api.ContainedRunRequest{WorkspacePath: ws, Command: []string{" "}}, &RequestError{}, "command is required"}, + {"agent profile", api.ContainedRunRequest{WorkspacePath: ws, Command: []string{"x"}, Profile: "agent"}, &UnsupportedError{}, "S6"}, + {"bad profile", api.ContainedRunRequest{WorkspacePath: ws, Command: []string{"x"}, Profile: "root"}, &RequestError{}, "profile must be"}, + {"bad network", api.ContainedRunRequest{WorkspacePath: ws, Command: []string{"x"}, Network: "host"}, &RequestError{}, "network must be"}, + {"restricted unsupported", api.ContainedRunRequest{WorkspacePath: ws, Command: []string{"x"}, Network: "restricted"}, &UnsupportedError{}, `cannot enforce network "restricted"`}, + {"credential env", api.ContainedRunRequest{WorkspacePath: ws, Command: []string{"x"}, EnvAllowlist: []string{"GITHUB_TOKEN"}}, &RequestError{}, "credential"}, + {"timeout cap", api.ContainedRunRequest{WorkspacePath: ws, Command: []string{"x"}, TimeoutMs: time.Hour.Milliseconds()}, &RequestError{}, "timeout_ms"}, + {"output cap", api.ContainedRunRequest{WorkspacePath: ws, Command: []string{"x"}, MaxOutputBytes: DefaultMaxOutputCap + 1}, &RequestError{}, "max_output_bytes"}, + {"no source", api.ContainedRunRequest{Command: []string{"x"}}, &RequestError{}, "one of worktree or workspace_path"}, + {"two sources", api.ContainedRunRequest{Command: []string{"x"}, WorkspacePath: ws, Worktree: wt}, &RequestError{}, "mutually exclusive"}, + {"checkout id", api.ContainedRunRequest{Command: []string{"x"}, Checkout: "chk_1"}, &UnsupportedError{}, "S1"}, + {"worktree repo", api.ContainedRunRequest{Command: []string{"x"}, Worktree: &api.ContainedRunWorktree{Commit: testSHA}}, &RequestError{}, "worktree.repo"}, + {"worktree short sha", api.ContainedRunRequest{Command: []string{"x"}, Worktree: &api.ContainedRunWorktree{Repo: wt.Repo, Commit: "main"}}, &RequestError{}, "40-hex"}, + {"worktree forbidden", api.ContainedRunRequest{Command: []string{"x"}, Worktree: &api.ContainedRunWorktree{Repo: "https://example.com/r", Commit: testSHA}}, &ForbiddenError{}, "repo_allowlist"}, + {"relative workspace", api.ContainedRunRequest{Command: []string{"x"}, WorkspacePath: "proj"}, &RequestError{}, "absolute"}, + {"workspace outside roots", api.ContainedRunRequest{Command: []string{"x"}, WorkspacePath: filepath.Dir(f.root)}, &ForbiddenError{}, "workspace_roots"}, + {"workspace sibling prefix", api.ContainedRunRequest{Command: []string{"x"}, WorkspacePath: f.root + "2"}, &ForbiddenError{}, "workspace_roots"}, + {"workspace missing", api.ContainedRunRequest{Command: []string{"x"}, WorkspacePath: filepath.Join(ws, "nope")}, &RequestError{}, "not a directory"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + _, created, err := f.svc.Create(context.Background(), "operator", tc.req) + if created || err == nil { + t.Fatalf("Create accepted: created=%v err=%v", created, err) + } + var ok bool + switch tc.want.(type) { + case *RequestError: + var e *RequestError + ok = errors.As(err, &e) + case *ForbiddenError: + var e *ForbiddenError + ok = errors.As(err, &e) + case *UnsupportedError: + var e *UnsupportedError + ok = errors.As(err, &e) + } + if !ok || !strings.Contains(err.Error(), tc.msg) { + t.Fatalf("err = %T %v, want %T containing %q", err, err, tc.want, tc.msg) + } + }) + } + if len(f.launcher.specs) != 0 { + t.Fatal("a refused request reached the launcher") + } +} + +func TestNotConfiguredServesRecordsButRefusesCreates(t *testing.T) { + store, err := runstate.Open(t.TempDir()) + if err != nil { + t.Fatal(err) + } + svc := New(store, nil, Config{}) + if svc.Configured() { + t.Fatal("Configured without a backend") + } + if _, _, err := svc.Create(context.Background(), "operator", api.ContainedRunRequest{Command: []string{"x"}}); !errors.Is(err, ErrNotConfigured) { + t.Fatalf("Create = %v", err) + } + rec, _, _ := store.Create(runstate.Record{}) + if run, err := svc.Get(rec.ID); err != nil || run.ID != rec.ID { + t.Fatalf("Get = %+v, %v", run, err) + } +} + +func TestClassifyTable(t *testing.T) { + unknown := sandbox.Result{ExitCode: -1} + boom := errors.New("boom") + cases := []struct { + name string + res sandbox.Result + err error + cancelled bool + timedOut bool + status string + wantExit *int + wantReason string + }{ + {"ok", sandbox.Result{}, nil, false, false, runstate.StatusCompleted, ptr(0), ""}, + {"non-zero", sandbox.Result{ExitCode: 2}, boom, false, false, runstate.StatusCompleted, ptr(2), "boom"}, + {"refused", sandbox.Result{}, boom, false, false, runstate.StatusFailed, nil, "boom"}, + {"unknown", unknown, boom, false, false, runstate.StatusUnfinished, nil, "boom"}, + {"cancelled", unknown, context.Canceled, true, false, runstate.StatusCancelled, nil, "cancelled by the caller"}, + {"cancelled with exit", sandbox.Result{ExitCode: 137}, boom, true, false, runstate.StatusCancelled, ptr(137), "cancelled by the caller"}, + {"timed out", unknown, context.DeadlineExceeded, false, true, runstate.StatusFailed, nil, "timed out"}, + } + for _, tc := range cases { + status, reason, exit := classify(tc.res, tc.err, tc.cancelled, tc.timedOut) + if status != tc.status || reason != tc.wantReason || (exit == nil) != (tc.wantExit == nil) || (exit != nil && *exit != *tc.wantExit) { + t.Errorf("%s: got %s %q %v", tc.name, status, reason, exit) + } + } +} + +func ptr(v int) *int { return &v } diff --git a/src/crux/runstate/runstate.go b/src/crux/runstate/runstate.go new file mode 100644 index 0000000000..97771f2e95 --- /dev/null +++ b/src/crux/runstate/runstate.go @@ -0,0 +1,362 @@ +// Package runstate is the saved-run-results store of the Crux work services +// (S7 in hivecommons/crux#114): a stable, caller-visible ID assigned at +// accept time, an explicit status set, one JSON record per item under +// WorkDir()/runs//record.json and an append-only runs/index.jsonl +// history, mirroring the sandbox-runs conventions in sandbox_run.go. +// +// The store never guesses. A record that was accepted or running when the +// serving process stopped is reconciled to StatusUnfinished on the next +// start (Reconcile); the caller decides whether to re-run. An idempotency +// key supplied by the caller makes a repeated create return the existing +// record instead of a second one. +package runstate + +import ( + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + "sync" + "time" +) + +// Statuses. The set is closed: a driver maps these onto its own meaning; +// crux only ever reports what happened to the step. +const ( + StatusAccepted = "accepted" + StatusRunning = "running" + StatusWaiting = "waiting" + StatusCompleted = "completed" + StatusFailed = "failed" + StatusCancelled = "cancelled" + StatusUnfinished = "unfinished" + StatusUnsupported = "unsupported" +) + +const ( + // DirName is the per-concern directory under the work dir. + DirName = "runs" + // RecordName is the per-item record file inside the item's directory. + RecordName = "record.json" + // IndexName is the append-only status history, one IndexEntry per line. + IndexName = "index.jsonl" + // KindRun is a contained run (S2); S6 steps add their own kind later. + KindRun = "run" + + idRandomBytes = 6 + maxIDLen = 64 + dirMode = 0o770 + fileMode = 0o660 +) + +// ErrNotFound is returned by Get and Update for an unknown ID. +var ErrNotFound = errors.New("runstate: record not found") + +// Record is the saved state of one item. Detail carries the owning service's +// payload (for S2 the command result) and is opaque to this package. +type Record struct { + ID string `json:"id"` + Kind string `json:"kind"` + Status string `json:"status"` + Owner string `json:"owner,omitempty"` + IdempotencyKey string `json:"idempotency_key,omitempty"` + CreatedAt time.Time `json:"created_at"` + StartedAt *time.Time `json:"started_at,omitempty"` + FinishedAt *time.Time `json:"finished_at,omitempty"` + Reason string `json:"reason,omitempty"` + Detail json.RawMessage `json:"detail,omitempty"` +} + +// IndexEntry is one line of the append-only history: every status the item +// passed through, in order. +type IndexEntry struct { + At time.Time `json:"at"` + ID string `json:"id"` + Kind string `json:"kind"` + Status string `json:"status"` + Reason string `json:"reason,omitempty"` +} + +// Terminal reports whether status is final: nothing moves out of it. +func Terminal(status string) bool { + switch status { + case StatusCompleted, StatusFailed, StatusCancelled, StatusUnfinished, StatusUnsupported: + return true + } + return false +} + +// Valid reports whether status is one of the closed set. +func Valid(status string) bool { + switch status { + case StatusAccepted, StatusRunning, StatusWaiting: + return true + } + return Terminal(status) +} + +// Dir is the runs directory under workDir. +func Dir(workDir string) string { return filepath.Join(workDir, DirName) } + +// DeriveIdempotencyKey scopes a caller-supplied key to its owner so two +// callers using the same key cannot read each other's records. The result +// is what Record.IdempotencyKey stores; an empty key derives to "". +func DeriveIdempotencyKey(owner, key string) string { + if strings.TrimSpace(key) == "" { + return "" + } + h := sha256.New() + for _, p := range []string{owner, key} { + h.Write([]byte(p)) + h.Write([]byte{0}) + } + return hex.EncodeToString(h.Sum(nil)) +} + +// Store is the on-disk record store for one work dir. It is safe for +// concurrent use within one process; a work dir is served by one process. +type Store struct { + dir string + + mu sync.Mutex + keys map[string]string // derived idempotency key -> id + // Now and NewID are overridable for tests. + Now func() time.Time + NewID func() string +} + +// Open loads the store at Dir(workDir), creating the directory and +// rebuilding the idempotency index from the records already on disk. +func Open(workDir string) (*Store, error) { + s := &Store{dir: Dir(workDir), keys: map[string]string{}, Now: time.Now, NewID: newID} + if err := os.MkdirAll(s.dir, dirMode); err != nil { + return nil, fmt.Errorf("runstate: %w", err) + } + recs, err := s.List() + if err != nil { + return nil, err + } + for _, r := range recs { + if r.IdempotencyKey != "" { + s.keys[r.IdempotencyKey] = r.ID + } + } + return s, nil +} + +// Path is the directory of one item: its record and the owning service's +// artifacts live there. +func (s *Store) Path(id string) string { return filepath.Join(s.dir, id) } + +// Create saves rec as a new item in StatusAccepted and returns it with its +// ID and timestamps filled in. When rec.IdempotencyKey names an existing +// item, that item's current record is returned with created=false and +// nothing is written. +func (s *Store) Create(rec Record) (Record, bool, error) { + s.mu.Lock() + defer s.mu.Unlock() + if rec.IdempotencyKey != "" { + if id, ok := s.keys[rec.IdempotencyKey]; ok { + existing, err := s.read(id) + return existing, false, err + } + } + if rec.Kind == "" { + rec.Kind = KindRun + } + rec.ID = s.NewID() + rec.Status = StatusAccepted + rec.CreatedAt = s.Now().UTC() + rec.StartedAt, rec.FinishedAt = nil, nil + if err := os.MkdirAll(s.Path(rec.ID), dirMode); err != nil { + return Record{}, false, fmt.Errorf("runstate: %w", err) + } + if err := s.write(rec); err != nil { + return Record{}, false, err + } + if rec.IdempotencyKey != "" { + s.keys[rec.IdempotencyKey] = rec.ID + } + return rec, true, nil +} + +// Get returns the saved record whatever the process state. +func (s *Store) Get(id string) (Record, error) { + s.mu.Lock() + defer s.mu.Unlock() + return s.read(id) +} + +// Update applies fn to the saved record and writes the result. fn may change +// Status (validated against the closed set), Reason and Detail; a terminal +// record is never moved to another status. Timestamps are maintained here: +// StartedAt on the move to running, FinishedAt on the move to a terminal +// status. +func (s *Store) Update(id string, fn func(*Record) error) (Record, error) { + s.mu.Lock() + defer s.mu.Unlock() + rec, err := s.read(id) + if err != nil { + return Record{}, err + } + before := rec.Status + if err := fn(&rec); err != nil { + return Record{}, err + } + rec.ID = id + if !Valid(rec.Status) { + return Record{}, fmt.Errorf("runstate: invalid status %q", rec.Status) + } + if Terminal(before) && rec.Status != before { + return Record{}, fmt.Errorf("runstate: %s is already %s", id, before) + } + if rec.Status != before { + now := s.Now().UTC() + if rec.Status == StatusRunning && rec.StartedAt == nil { + rec.StartedAt = &now + } + if Terminal(rec.Status) && rec.FinishedAt == nil { + rec.FinishedAt = &now + } + } + if err := s.write(rec); err != nil { + return Record{}, err + } + return rec, nil +} + +// List returns every saved record, oldest first. +func (s *Store) List() ([]Record, error) { + entries, err := os.ReadDir(s.dir) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, fmt.Errorf("runstate: %w", err) + } + var out []Record + for _, e := range entries { + if !e.IsDir() { + continue + } + rec, err := s.read(e.Name()) + if err != nil { + if errors.Is(err, ErrNotFound) { + continue // a directory with artifacts but no record yet + } + return nil, err + } + out = append(out, rec) + } + sort.Slice(out, func(i, j int) bool { + if out[i].CreatedAt.Equal(out[j].CreatedAt) { + return out[i].ID < out[j].ID + } + return out[i].CreatedAt.Before(out[j].CreatedAt) + }) + return out, nil +} + +// Reconcile moves every accepted or running record to StatusUnfinished with +// reason and returns the records it changed. It runs at `crux serve` start: +// the process that owned those items is gone, so their exit is not knowable +// and they are never restarted silently. +func (s *Store) Reconcile(reason string) ([]Record, error) { + recs, err := s.List() + if err != nil { + return nil, err + } + var changed []Record + for _, r := range recs { + if r.Status != StatusAccepted && r.Status != StatusRunning { + continue + } + rec, err := s.Update(r.ID, func(rec *Record) error { + rec.Status = StatusUnfinished + rec.Reason = reason + return nil + }) + if err != nil { + return changed, err + } + changed = append(changed, rec) + } + return changed, nil +} + +func (s *Store) read(id string) (Record, error) { + if !validID(id) { + return Record{}, ErrNotFound + } + data, err := os.ReadFile(filepath.Join(s.Path(id), RecordName)) + if err != nil { + if os.IsNotExist(err) { + return Record{}, ErrNotFound + } + return Record{}, fmt.Errorf("runstate: %w", err) + } + var rec Record + if err := json.Unmarshal(data, &rec); err != nil { + return Record{}, fmt.Errorf("runstate: %s: %w", id, err) + } + return rec, nil +} + +// write replaces the record atomically and appends the status to the index. +func (s *Store) write(rec Record) error { + data, err := json.Marshal(rec) + if err != nil { + return fmt.Errorf("runstate: %w", err) + } + path := filepath.Join(s.Path(rec.ID), RecordName) + tmp := path + ".tmp" + if err := os.WriteFile(tmp, data, fileMode); err != nil { + return fmt.Errorf("runstate: %w", err) + } + if err := os.Rename(tmp, path); err != nil { + return fmt.Errorf("runstate: %w", err) + } + line, err := json.Marshal(IndexEntry{At: s.Now().UTC(), ID: rec.ID, Kind: rec.Kind, Status: rec.Status, Reason: rec.Reason}) + if err != nil { + return fmt.Errorf("runstate: %w", err) + } + f, err := os.OpenFile(filepath.Join(s.dir, IndexName), os.O_CREATE|os.O_APPEND|os.O_WRONLY, fileMode) + if err != nil { + return fmt.Errorf("runstate: %w", err) + } + _, werr := f.Write(append(line, '\n')) + if cerr := f.Close(); werr == nil { + werr = cerr + } + if werr != nil { + return fmt.Errorf("runstate: %w", werr) + } + return nil +} + +// validID keeps a caller-supplied ID from escaping the runs directory. +func validID(id string) bool { + if id == "" || len(id) > maxIDLen { + return false + } + for _, r := range id { + if !(r >= 'a' && r <= 'z' || r >= '0' && r <= '9' || r == '_' || r == '-') { + return false + } + } + return true +} + +func newID() string { + var b [idRandomBytes]byte + if _, err := rand.Read(b[:]); err != nil { + panic(fmt.Sprintf("runstate: crypto/rand: %v", err)) + } + return "run_" + hex.EncodeToString(b[:]) +} diff --git a/src/crux/runstate/runstate_test.go b/src/crux/runstate/runstate_test.go new file mode 100644 index 0000000000..7ecdfe7e55 --- /dev/null +++ b/src/crux/runstate/runstate_test.go @@ -0,0 +1,381 @@ +package runstate + +import ( + "bufio" + "encoding/json" + "errors" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +func newStore(t *testing.T) (*Store, string) { + t.Helper() + dir := t.TempDir() + s, err := Open(dir) + if err != nil { + t.Fatal(err) + } + ids := 0 + s.NewID = func() string { ids++; return "run_" + strings.Repeat("0", 11) + string(rune('a'+ids-1)) } + now := time.Date(2026, 11, 1, 12, 0, 0, 0, time.UTC) + s.Now = func() time.Time { now = now.Add(time.Second); return now } + return s, dir +} + +func TestCreateGetAndIndex(t *testing.T) { + s, dir := newStore(t) + rec, created, err := s.Create(Record{Owner: "operator", Detail: json.RawMessage(`{"x":1}`)}) + if err != nil || !created { + t.Fatalf("Create: %v created=%v", err, created) + } + if rec.ID == "" || rec.Status != StatusAccepted || rec.Kind != KindRun || rec.CreatedAt.IsZero() { + t.Fatalf("record not filled in: %+v", rec) + } + got, err := s.Get(rec.ID) + if err != nil { + t.Fatal(err) + } + if got.ID != rec.ID || got.Owner != "operator" || string(got.Detail) != `{"x":1}` { + t.Fatalf("Get = %+v", got) + } + if _, err := os.Stat(filepath.Join(Dir(dir), rec.ID, RecordName)); err != nil { + t.Fatal(err) + } + if _, err := s.Get("nope"); !errors.Is(err, ErrNotFound) { + t.Fatalf("Get missing = %v", err) + } + if _, err := s.Get("../escape"); !errors.Is(err, ErrNotFound) { + t.Fatalf("Get invalid id = %v", err) + } + entries := readIndex(t, dir) + if len(entries) != 1 || entries[0].ID != rec.ID || entries[0].Status != StatusAccepted { + t.Fatalf("index = %+v", entries) + } +} + +func TestUpdateTimestampsAndTerminalGuard(t *testing.T) { + s, dir := newStore(t) + rec, _, _ := s.Create(Record{}) + run, err := s.Update(rec.ID, func(r *Record) error { r.Status = StatusRunning; return nil }) + if err != nil { + t.Fatal(err) + } + if run.StartedAt == nil || run.FinishedAt != nil { + t.Fatalf("running timestamps = %+v", run) + } + done, err := s.Update(rec.ID, func(r *Record) error { + r.Status = StatusCompleted + r.Reason = "ok" + r.ID = "forged" + return nil + }) + if err != nil { + t.Fatal(err) + } + if done.ID != rec.ID || done.FinishedAt == nil || !done.FinishedAt.After(*done.StartedAt) || done.Reason != "ok" { + t.Fatalf("completed = %+v", done) + } + if _, err := s.Update(rec.ID, func(r *Record) error { r.Status = StatusRunning; return nil }); err == nil || !strings.Contains(err.Error(), "already completed") { + t.Fatalf("leaving terminal: %v", err) + } + // Reason may still be amended on a terminal record. + if _, err := s.Update(rec.ID, func(r *Record) error { r.Reason = "ok; late note"; return nil }); err != nil { + t.Fatal(err) + } + if _, err := s.Update(rec.ID, func(r *Record) error { r.Status = "bogus"; return nil }); err == nil || !strings.Contains(err.Error(), "invalid status") { + t.Fatalf("invalid status: %v", err) + } + boom := errors.New("boom") + if _, err := s.Update(rec.ID, func(r *Record) error { return boom }); !errors.Is(err, boom) { + t.Fatalf("fn error: %v", err) + } + if _, err := s.Update("missing", func(r *Record) error { return nil }); !errors.Is(err, ErrNotFound) { + t.Fatalf("missing: %v", err) + } + statuses := []string{} + for _, e := range readIndex(t, dir) { + statuses = append(statuses, e.Status) + } + if want := "accepted running completed completed"; strings.Join(statuses, " ") != want { + t.Fatalf("index statuses = %v", statuses) + } +} + +func TestIdempotencyAcrossReopen(t *testing.T) { + s, dir := newStore(t) + key := DeriveIdempotencyKey("operator", "k1") + if key == "" || key == DeriveIdempotencyKey("other", "k1") || DeriveIdempotencyKey("operator", " ") != "" { + t.Fatalf("DeriveIdempotencyKey scoping broken") + } + first, created, err := s.Create(Record{IdempotencyKey: key}) + if err != nil || !created { + t.Fatal(err, created) + } + if _, err := s.Update(first.ID, func(r *Record) error { r.Status = StatusFailed; return nil }); err != nil { + t.Fatal(err) + } + again, created, err := s.Create(Record{IdempotencyKey: key}) + if err != nil || created || again.ID != first.ID || again.Status != StatusFailed { + t.Fatalf("replay = %+v created=%v err=%v", again, created, err) + } + other, created, _ := s.Create(Record{IdempotencyKey: DeriveIdempotencyKey("operator", "k2")}) + if !created || other.ID == first.ID { + t.Fatalf("different key reused record") + } + reopened, err := Open(dir) + if err != nil { + t.Fatal(err) + } + replay, created, err := reopened.Create(Record{IdempotencyKey: key}) + if err != nil || created || replay.ID != first.ID { + t.Fatalf("after reopen = %+v created=%v err=%v", replay, created, err) + } +} + +func TestListAndReconcile(t *testing.T) { + s, dir := newStore(t) + a, _, _ := s.Create(Record{}) + b, _, _ := s.Create(Record{}) + c, _, _ := s.Create(Record{}) + s.Update(b.ID, func(r *Record) error { r.Status = StatusRunning; return nil }) + s.Update(c.ID, func(r *Record) error { r.Status = StatusCompleted; return nil }) + // A directory with artifacts but no record is skipped, not an error. + if err := os.MkdirAll(filepath.Join(Dir(dir), "run_orphan"), 0o770); err != nil { + t.Fatal(err) + } + list, err := s.List() + if err != nil { + t.Fatal(err) + } + if len(list) != 3 || list[0].ID != a.ID || list[1].ID != b.ID || list[2].ID != c.ID { + t.Fatalf("List = %+v", list) + } + changed, err := s.Reconcile("restarted") + if err != nil { + t.Fatal(err) + } + if len(changed) != 2 || changed[0].ID != a.ID || changed[1].ID != b.ID { + t.Fatalf("Reconcile changed %+v", changed) + } + for _, id := range []string{a.ID, b.ID} { + rec, _ := s.Get(id) + if rec.Status != StatusUnfinished || rec.Reason != "restarted" || rec.FinishedAt == nil { + t.Fatalf("%s = %+v", id, rec) + } + } + if rec, _ := s.Get(c.ID); rec.Status != StatusCompleted { + t.Fatalf("completed record touched: %+v", rec) + } + if again, _ := s.Reconcile("x"); len(again) != 0 { + t.Fatalf("second reconcile changed %+v", again) + } + empty := &Store{dir: filepath.Join(t.TempDir(), "none"), Now: time.Now, NewID: newID} + if list, err := empty.List(); err != nil || list != nil { + t.Fatalf("missing dir List = %v, %v", list, err) + } +} + +func TestCorruptRecordIsAnError(t *testing.T) { + s, dir := newStore(t) + rec, _, _ := s.Create(Record{}) + if err := os.WriteFile(filepath.Join(Dir(dir), rec.ID, RecordName), []byte("{"), 0o660); err != nil { + t.Fatal(err) + } + if _, err := s.Get(rec.ID); err == nil || errors.Is(err, ErrNotFound) { + t.Fatalf("corrupt Get = %v", err) + } + if _, err := s.List(); err == nil { + t.Fatal("corrupt List succeeded") + } + if _, err := Open(dir); err == nil { + t.Fatal("Open over a corrupt record succeeded") + } +} + +func TestOpenFailsOnUnwritableWorkDir(t *testing.T) { + file := filepath.Join(t.TempDir(), "file") + if err := os.WriteFile(file, nil, 0o600); err != nil { + t.Fatal(err) + } + if _, err := Open(file); err == nil { + t.Fatal("Open under a file succeeded") + } +} + +func TestStatusSets(t *testing.T) { + for _, st := range []string{StatusCompleted, StatusFailed, StatusCancelled, StatusUnfinished, StatusUnsupported} { + if !Terminal(st) || !Valid(st) { + t.Errorf("%s should be terminal and valid", st) + } + } + for _, st := range []string{StatusAccepted, StatusRunning, StatusWaiting} { + if Terminal(st) || !Valid(st) { + t.Errorf("%s should be valid and not terminal", st) + } + } + if Valid("") || Valid("done") { + t.Error("unknown statuses accepted") + } + if !validID(newID()) || validID("") || validID(strings.Repeat("a", maxIDLen+1)) || validID("A") { + t.Error("validID") + } +} + +func readIndex(t *testing.T, workDir string) []IndexEntry { + t.Helper() + f, err := os.Open(filepath.Join(Dir(workDir), IndexName)) + if err != nil { + t.Fatal(err) + } + defer f.Close() + var out []IndexEntry + sc := bufio.NewScanner(f) + for sc.Scan() { + var e IndexEntry + if err := json.Unmarshal(sc.Bytes(), &e); err != nil { + t.Fatal(err) + } + out = append(out, e) + } + return out +} + +func TestUpdateRejectsUnknownStatus(t *testing.T) { + s, _ := newStore(t) + rec, _, _ := s.Create(Record{}) + if _, err := s.Update(rec.ID, func(r *Record) error { r.Status = "paused"; return nil }); err == nil || !strings.Contains(err.Error(), "invalid status") { + t.Fatalf("Update with unknown status = %v", err) + } + if got, _ := s.Get(rec.ID); got.Status != StatusAccepted { + t.Fatalf("rejected update was written: %+v", got) + } +} + +func TestInvalidIDsAreNotFound(t *testing.T) { + s, _ := newStore(t) + for _, id := range []string{"Run_ABC", "run/x", "run x", strings.Repeat("a", maxIDLen+1)} { + if _, err := s.Get(id); !errors.Is(err, ErrNotFound) { + t.Errorf("Get(%q) = %v, want ErrNotFound", id, err) + } + } +} + +// Filesystem obstacles in the item's directory surface as errors rather +// than a half-written record: the item dir is a file, the temp file is a +// directory, the record is a non-empty directory, the index is a directory. +func TestCreateReportsWriteObstacles(t *testing.T) { + cases := map[string]func(t *testing.T, dir, id string){ + "item dir is a file": func(t *testing.T, dir, id string) { + if err := os.WriteFile(filepath.Join(Dir(dir), id), nil, 0o600); err != nil { + t.Fatal(err) + } + }, + "temp file is a directory": func(t *testing.T, dir, id string) { + if err := os.MkdirAll(filepath.Join(Dir(dir), id, RecordName+".tmp"), 0o770); err != nil { + t.Fatal(err) + } + }, + "record is a non-empty directory": func(t *testing.T, dir, id string) { + if err := os.MkdirAll(filepath.Join(Dir(dir), id, RecordName, "child"), 0o770); err != nil { + t.Fatal(err) + } + }, + "index is a directory": func(t *testing.T, dir, id string) { + if err := os.MkdirAll(filepath.Join(Dir(dir), IndexName), 0o770); err != nil { + t.Fatal(err) + } + }, + } + for name, obstruct := range cases { + t.Run(name, func(t *testing.T) { + s, dir := newStore(t) + obstruct(t, dir, s.NewID()) + s.NewID = func() string { return "run_" + strings.Repeat("0", 11) + "a" } + if _, _, err := s.Create(Record{}); err == nil { + t.Fatal("Create succeeded through the obstacle") + } + }) + } +} + +func TestUnreadableRecordIsAnErrorNotMissing(t *testing.T) { + s, dir := newStore(t) + rec, _, _ := s.Create(Record{}) + // A record that is a directory cannot be read; it is not ErrNotFound. + if err := os.Remove(filepath.Join(Dir(dir), rec.ID, RecordName)); err != nil { + t.Fatal(err) + } + if err := os.Mkdir(filepath.Join(Dir(dir), rec.ID, RecordName), 0o770); err != nil { + t.Fatal(err) + } + if _, err := s.Get(rec.ID); err == nil || errors.Is(err, ErrNotFound) { + t.Fatalf("Get = %v, want a read error", err) + } + if _, err := s.List(); err == nil || errors.Is(err, ErrNotFound) { + t.Fatalf("List = %v, want a read error", err) + } + if _, err := s.Reconcile("restart"); err == nil { + t.Fatal("Reconcile succeeded over an unreadable record") + } +} + +func TestReconcileReportsWriteFailure(t *testing.T) { + s, dir := newStore(t) + rec, _, _ := s.Create(Record{}) + if err := os.Mkdir(filepath.Join(Dir(dir), rec.ID, RecordName+".tmp"), 0o770); err != nil { + t.Fatal(err) + } + changed, err := s.Reconcile("restart") + if err == nil || len(changed) != 0 { + t.Fatalf("Reconcile = %+v, %v; want a write error", changed, err) + } + if got, _ := s.Get(rec.ID); got.Status != StatusAccepted { + t.Fatalf("record moved despite the failed write: %+v", got) + } +} + +func TestListFailsWhenDirIsAFile(t *testing.T) { + file := filepath.Join(t.TempDir(), "runs") + if err := os.WriteFile(file, nil, 0o600); err != nil { + t.Fatal(err) + } + s := &Store{dir: file, keys: map[string]string{}, Now: time.Now, NewID: newID} + if _, err := s.List(); err == nil || errors.Is(err, ErrNotFound) { + t.Fatalf("List over a file = %v", err) + } +} + +func TestListBreaksCreatedAtTiesByID(t *testing.T) { + s, _ := newStore(t) + fixed := time.Date(2026, 11, 1, 12, 0, 0, 0, time.UTC) + s.Now = func() time.Time { return fixed } + ids := []string{"run_00000000000c", "run_00000000000a", "run_00000000000b"} + next := 0 + s.NewID = func() string { id := ids[next]; next++; return id } + for range ids { + if _, _, err := s.Create(Record{}); err != nil { + t.Fatal(err) + } + } + list, err := s.List() + if err != nil { + t.Fatal(err) + } + var got []string + for _, r := range list { + got = append(got, r.ID) + } + if strings.Join(got, ",") != "run_00000000000a,run_00000000000b,run_00000000000c" { + t.Fatalf("List order = %v", got) + } +} + +func TestNewIDShape(t *testing.T) { + a, b := newID(), newID() + if a == b || !validID(a) || !strings.HasPrefix(a, "run_") { + t.Fatalf("newID = %q, %q", a, b) + } +} diff --git a/src/crux/serve/runs.go b/src/crux/serve/runs.go new file mode 100644 index 0000000000..13933037e6 --- /dev/null +++ b/src/crux/serve/runs.go @@ -0,0 +1,121 @@ +package serve + +import ( + "context" + "errors" + "net/http" + + "github.com/hivecommons/hive/crux/api" + "github.com/hivecommons/hive/crux/runs" +) + +// runsOwner is the principal a contained run is recorded under. +// +// TODO(S0): switch to CallerFrom(r.Context()) once the caller-principal +// change lands; until then every authenticated caller is the operator, +// which is exactly what the bearer-token check already establishes. +func runsOwner(r *http.Request) string { + return "operator" +} + +// registerRuns mounts the contained-run family (RFC crux#114 S2 + S7). +// The routes are always present so a server without a backend answers +// with a clear not_supported instead of a bare 404. +func (s *Server) registerRuns(mux *http.ServeMux) { + mux.HandleFunc("POST "+api.PathRuns, s.handleRunsCreate) + mux.HandleFunc("GET "+api.PathRuns+"/{id}", s.handleRunsGet) + mux.HandleFunc("POST "+api.PathRuns+"/{id}/cancel", s.handleRunsCancel) +} + +// runsReady reports whether the run service exists; when it does not, a +// 501 has been written. A service without a backend still serves saved +// records, so only Create checks Configured separately. +func (s *Server) runsReady(w http.ResponseWriter) bool { + if s.Runs == nil { + s.writeError(w, http.StatusNotImplemented, api.CodeNotSupported, "contained runs are not enabled on this server") + return false + } + return true +} + +func (s *Server) handleRunsCreate(w http.ResponseWriter, r *http.Request) { + if !s.runsReady(w) { + return + } + var req api.ContainedRunRequest + if !s.readJSON(w, r, &req) { + return + } + run, created, err := s.Runs.Create(s.LaunchContext, runsOwner(r), req) + if err != nil { + s.writeRunsError(w, err) + return + } + if !created { + // Idempotency hit: the earlier record, whatever state it is in. + s.writeJSON(w, http.StatusOK, run) + return + } + if req.Detach { + s.writeJSON(w, http.StatusAccepted, run) + return + } + // The run itself is bound to LaunchContext; only the wait is bound to + // the request, so a caller that goes away leaves the run in flight. + run, err = s.Runs.Wait(r.Context(), run.ID) + if err != nil { + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return + } + s.writeRunsError(w, err) + return + } + s.writeJSON(w, http.StatusOK, run) +} + +func (s *Server) handleRunsGet(w http.ResponseWriter, r *http.Request) { + if !s.runsReady(w) { + return + } + run, err := s.Runs.Get(r.PathValue("id")) + if err != nil { + s.writeRunsError(w, err) + return + } + s.writeJSON(w, http.StatusOK, run) +} + +func (s *Server) handleRunsCancel(w http.ResponseWriter, r *http.Request) { + if !s.runsReady(w) { + return + } + run, err := s.Runs.Cancel(r.Context(), r.PathValue("id")) + if err != nil { + s.writeRunsError(w, err) + return + } + s.writeJSON(w, http.StatusOK, run) +} + +// writeRunsError maps the run service's typed errors onto wire codes. The +// refusals keep their own codes so a caller can tell "you asked for +// something wrong" from "this backend cannot enforce that". +func (s *Server) writeRunsError(w http.ResponseWriter, err error) { + var reqErr *runs.RequestError + var forbidden *runs.ForbiddenError + var unsupported *runs.UnsupportedError + switch { + case errors.Is(err, runs.ErrNotConfigured): + s.writeError(w, http.StatusNotImplemented, api.CodeNotSupported, err.Error()) + case errors.Is(err, runs.ErrNotFound): + s.writeError(w, http.StatusNotFound, api.CodeNotFound, "run not found") + case errors.As(err, &reqErr): + s.writeError(w, http.StatusBadRequest, api.CodeBadRequest, err.Error()) + case errors.As(err, &forbidden): + s.writeError(w, http.StatusForbidden, api.CodeForbidden, err.Error()) + case errors.As(err, &unsupported): + s.writeError(w, http.StatusNotImplemented, api.CodeUnsupported, err.Error()) + default: + s.writeError(w, http.StatusInternalServerError, api.CodeRuntime, err.Error()) + } +} diff --git a/src/crux/serve/runs_test.go b/src/crux/serve/runs_test.go new file mode 100644 index 0000000000..bedfb47ed2 --- /dev/null +++ b/src/crux/serve/runs_test.go @@ -0,0 +1,131 @@ +package serve + +import ( + "context" + "net/http" + "os" + "path/filepath" + "testing" + "time" + + "github.com/hivecommons/hive/crux/api" + "github.com/hivecommons/hive/crux/runs" + "github.com/hivecommons/hive/crux/runstate" + "github.com/hivecommons/hive/crux/sandbox" +) + +type runsLauncher func(ctx context.Context, spec sandbox.LaunchSpec) (sandbox.Result, error) + +func (l runsLauncher) Run(ctx context.Context, spec sandbox.LaunchSpec) (sandbox.Result, error) { + return l(ctx, spec) +} + +// newRunsServer wires a run service with a fake launcher onto a family +// server; the workspace root holds one directory, "proj". +func newRunsServer(t *testing.T, launcher runsLauncher) (*famServer, string) { + t.Helper() + f := newFamServer(t) + store, err := runstate.Open(t.TempDir()) + if err != nil { + t.Fatal(err) + } + root := t.TempDir() + ws := filepath.Join(root, "proj") + if err := os.MkdirAll(ws, 0o755); err != nil { + t.Fatal(err) + } + backend := &runs.Backend{Name: "fake", Launcher: launcher, Capabilities: runs.Capabilities{NetworkNone: true}} + f.srv.Runs = runs.New(store, backend, runs.Config{Image: "img", WorkspaceRoots: []string{root}}) + return f, ws +} + +func TestRunsRoutesAnswerNotSupportedWithoutAService(t *testing.T) { + f := newFamServer(t) + f.expectCode(http.MethodPost, api.PathRuns, api.ContainedRunRequest{Command: []string{"x"}}, http.StatusNotImplemented, api.CodeNotSupported) + f.expectCode(http.MethodGet, api.RunPath("run_x"), nil, http.StatusNotImplemented, api.CodeNotSupported) + f.expectCode(http.MethodPost, api.RunCancelPath("run_x"), nil, http.StatusNotImplemented, api.CodeNotSupported) +} + +func TestRunsWithoutABackendStillServesRecords(t *testing.T) { + f := newFamServer(t) + store, err := runstate.Open(t.TempDir()) + if err != nil { + t.Fatal(err) + } + f.srv.Runs = runs.New(store, nil, runs.Config{}) + rec, _, err := store.Create(runstate.Record{Owner: "operator"}) + if err != nil { + t.Fatal(err) + } + f.expectCode(http.MethodPost, api.PathRuns, api.ContainedRunRequest{Command: []string{"x"}}, http.StatusNotImplemented, api.CodeNotSupported) + var run api.ContainedRun + f.expect(http.MethodGet, api.RunPath(rec.ID), nil, http.StatusOK, &run) + if run.ID != rec.ID || run.Status != api.RunAccepted { + t.Fatalf("GET = %+v", run) + } +} + +func TestRunsCreateWaitsAndReturnsTheRecord(t *testing.T) { + f, ws := newRunsServer(t, func(_ context.Context, spec sandbox.LaunchSpec) (sandbox.Result, error) { + return sandbox.Result{Stdout: "ran " + spec.Command[0]}, nil + }) + req := api.ContainedRunRequest{WorkspacePath: ws, Command: []string{"make", "test"}, IdempotencyKey: "k"} + var run api.ContainedRun + f.expect(http.MethodPost, api.PathRuns, req, http.StatusOK, &run) + if run.Status != api.RunCompleted || run.Stdout != "ran make" || run.ExitCode == nil || *run.ExitCode != 0 || run.Owner != "operator" { + t.Fatalf("POST = %+v", run) + } + var again api.ContainedRun + f.expect(http.MethodPost, api.PathRuns, req, http.StatusOK, &again) + if again.ID != run.ID { + t.Fatalf("idempotent replay returned %s, want %s", again.ID, run.ID) + } + var got api.ContainedRun + f.expect(http.MethodGet, api.RunPath(run.ID), nil, http.StatusOK, &got) + if got.ID != run.ID || got.Status != api.RunCompleted { + t.Fatalf("GET = %+v", got) + } + var late api.ContainedRun + f.expect(http.MethodPost, api.RunCancelPath(run.ID), nil, http.StatusOK, &late) + if !late.AlreadyFinished || late.Status != api.RunCompleted { + t.Fatalf("late cancel = %+v", late) + } +} + +func TestRunsDetachAndCancel(t *testing.T) { + started := make(chan struct{}, 1) + f, ws := newRunsServer(t, func(ctx context.Context, _ sandbox.LaunchSpec) (sandbox.Result, error) { + started <- struct{}{} + <-ctx.Done() + return sandbox.Result{ExitCode: -1}, ctx.Err() + }) + var run api.ContainedRun + f.expect(http.MethodPost, api.PathRuns, api.ContainedRunRequest{WorkspacePath: ws, Command: []string{"sleep"}, Detach: true}, http.StatusAccepted, &run) + if run.Status != api.RunAccepted || run.ID == "" { + t.Fatalf("detached = %+v", run) + } + select { + case <-started: + case <-time.After(10 * time.Second): + t.Fatal("launcher never started") + } + var cancelled api.ContainedRun + f.expect(http.MethodPost, api.RunCancelPath(run.ID), nil, http.StatusOK, &cancelled) + if cancelled.Status != api.RunCancelled || cancelled.AlreadyFinished || cancelled.ExitCode != nil { + t.Fatalf("cancel = %+v", cancelled) + } +} + +func TestRunsErrorCodes(t *testing.T) { + f, ws := newRunsServer(t, func(context.Context, sandbox.LaunchSpec) (sandbox.Result, error) { + return sandbox.Result{}, nil + }) + f.expectCode(http.MethodPost, api.PathRuns, "{not json", http.StatusBadRequest, api.CodeBadRequest) + f.expectCode(http.MethodPost, api.PathRuns, api.ContainedRunRequest{WorkspacePath: ws}, http.StatusBadRequest, api.CodeBadRequest) + f.expectCode(http.MethodPost, api.PathRuns, api.ContainedRunRequest{WorkspacePath: ws, Command: []string{"x"}, Profile: api.RunProfileAgent}, http.StatusNotImplemented, api.CodeUnsupported) + f.expectCode(http.MethodPost, api.PathRuns, api.ContainedRunRequest{WorkspacePath: ws, Command: []string{"x"}, Network: api.RunNetworkRestricted}, http.StatusNotImplemented, api.CodeUnsupported) + f.expectCode(http.MethodPost, api.PathRuns, api.ContainedRunRequest{Checkout: "chk_1", Command: []string{"x"}}, http.StatusNotImplemented, api.CodeUnsupported) + f.expectCode(http.MethodPost, api.PathRuns, api.ContainedRunRequest{WorkspacePath: "/elsewhere", Command: []string{"x"}}, http.StatusForbidden, api.CodeForbidden) + f.expectCode(http.MethodGet, api.RunPath("run_missing"), nil, http.StatusNotFound, api.CodeNotFound) + f.expectCode(http.MethodPost, api.RunCancelPath("run_missing"), nil, http.StatusNotFound, api.CodeNotFound) +} diff --git a/src/crux/serve/serve.go b/src/crux/serve/serve.go index c020ef905d..813e2b778d 100644 --- a/src/crux/serve/serve.go +++ b/src/crux/serve/serve.go @@ -38,6 +38,7 @@ import ( "github.com/hivecommons/hive/crux/checkout" "github.com/hivecommons/hive/crux/config" "github.com/hivecommons/hive/crux/lanes" + "github.com/hivecommons/hive/crux/runs" ) const ( @@ -109,6 +110,10 @@ type Server struct { // (api.PathCheckouts); nil answers those routes 501. Checkouts *checkout.Service + // Runs, when set, serves the contained-run family (RFC crux#114 S2 + + // S7); nil answers those routes with not_supported. + Runs *runs.Service + now func() time.Time started time.Time @@ -166,6 +171,7 @@ func (s *Server) Handler() http.Handler { mux.HandleFunc("GET "+api.PathEvents, s.handleEvents) s.registerFamilies(mux) s.registerCheckouts(mux) + s.registerRuns(mux) return s.laneHandler(s.authenticate(mux)) } diff --git a/src/docs/agent-knowledge-and-memory.md b/src/docs/agent-knowledge-and-memory.md new file mode 100644 index 0000000000..5bc6bd08a3 --- /dev/null +++ b/src/docs/agent-knowledge-and-memory.md @@ -0,0 +1,215 @@ +# How Hive injects knowledge and memory into agents + +> Written to answer the 2026-11-05 community-meeting question "how does +> Hive's memory/knowledge injection work today?". It describes `v5` as of +> this page's merge; the [Known gaps](#known-gaps--in-progress) section links +> the work that will change the memory half. + +Hive agents receive context from **three distinct layers**, and only one of +them is Hive's own: + +| Layer | Who owns it | Where it lives | Hive's role | +|---|---|---|---| +| **Knowledge** — curated facts about the repos and the fleet | Hive (`pkg/knowledge`) | File vaults under `/data/knowledge/…`, optional HTTP wiki layers | Selects a few facts per kick and **pushes** them into the kick prompt; also serves them for **pull** via API/MCP/`hivectl` | +| **Conversation** — the current chat in the CLI pane | The CLI | In the CLI process | Resets it before every kick (`/clear`) | +| **CLI memory** — the CLI's own persistent, cross-session notes (Claude Code auto-memory, Codex memories, Gemini `GEMINI.md`, …) | The CLI | Dot-directories under the agent's `HOME` | **None.** Hive neither feeds, reads, filters nor clears it; it only decides which agents share the directory | + +The confusion at the meeting was between layers 2 and 3: "every agent clears +its context before it runs" is true of the conversation and false of CLI +memory. + +## 1. Knowledge: the kick primer (push) + +``` + knowledge.vaults / git_sources / connectors / bead_synthesizer vault + (FileStore per source; HTTP wiki layers for personal/project/org/community) + │ + registerKnowledgeStores (cmd/hive/boot_knowledge_primer.go) + ▼ + ┌───────────────── Scheduler.primeKnowledge (pkg/scheduler/kick_priming.go) ───┐ + │ 1. extractKeywords ← titles + labels of the agent's first │ + │ maxIssuesToPrime (=5) queued issues │ + │ 2. scope ← knowledge.agent_scopes[] (or base agent for a replica) │ + │ 3. Primer.PrimeScoped (pkg/knowledge/primer.go) │ + │ search every layer/store → merge by layer precedence → │ + │ FilterByLifecycle (approved only) → scope.Filter → │ + │ Fisher-Rao rerank → type priority → cap at max_facts (25) → │ + │ expandWithGraph (never past the scope) → Context7 hints │ + │ 4. PrimedKnowledge.Render (primer_render.go): "# Relevant Knowledge", │ + │ grouped by type, whole facts dropped past max_bytes (4096), │ + │ "_… truncated: N facts / M bytes omitted (size cap)._" marker │ + └──────────────────────────────────────────────────────────────────────────────┘ + │ + templates.go: skills section + AGENTS.md section + knowledge section + substituted into the agent's kick template; lane kicks (lane_messages.go) + append the knowledge section for scanner / quality / architect / + ▼ + tmux: [/clear ⏎] then the kick text ⏎ (crux/agent/manager_kick.go) +``` + +**Where the facts come from.** Everything the primer searches is registered +in `registerKnowledgeStores` (`src/cmd/hive/boot_knowledge_primer.go`): +configured `knowledge.vaults` (personal layer), `knowledge.git_sources` +(layer from config), connector output, the **bead synthesizer vault** +(closed beads from every agent, classified into fact types — the main +organic source on a running hive), plus any HTTP wiki layers in +`knowledge.layers`. The repo's own docs are *not* a source unless you index +them through a git source, connector or code map +([knowledge-curator.md](knowledge-curator.md)). + +**How it is scoped.** + +- *By the work, not the repo:* the query is keywords from the titles and + labels of the agent's first five queued issues (`extractKeywords`, + `isNoiseLabel`). There is no automatic per-repo or per-issue filter; a fact + tagged with another repo can surface if the keywords match. +- *By agent:* `knowledge.agent_scopes.` is an allow-list of `layers`, + `repos`, `types`, `tags` (`KnowledgeAgentScope` in + `src/pkg/config/knowledge_config.go`). It is applied inside + `Primer.PrimeScoped` *after* the lifecycle filter and can only narrow; graph + expansion never pulls in an out-of-scope fact. A replica inherits its base + agent's entry (`Config.KnowledgeAgentScope`). No entry means unrestricted. +- *By lifecycle:* kicks carry **approved facts only**. The kick primer is built + with no `IncludeStates` (`newKnowledgePrimer` in `src/cmd/hive/main.go`), so + `draft`, `deprecated` and `superseded` never reach a kick; `agent_scopes. + include_states` only widens API/TOC reads that already asked for those + states ([knowledge-lifecycle.md](knowledge-lifecycle.md)). + +**Size caps** (all drop whole units, never clip, and leave a visible marker): + +| Block | Cap | Config | +|---|---|---| +| `# Relevant Knowledge` | 25 facts, 4096 bytes (floor 512) | `knowledge.primer.max_facts`, `knowledge.primer.max_bytes` | +| Repo `AGENTS.md` | 8192 bytes (floor 2048), whole sections | `project.agents_md_max_bytes` (`agentsmd.DefaultMaxInjectionBytes`) | +| Declared skills | 8192 bytes, whole skills | `maxSkillsInjectionBytes` in `kick_priming.go` (fixed) | + +**Switches.** `knowledge.enabled` gates the primer only — the bead synthesizer +still writes facts when it is off, they just reach no kick. The dashboard +knowledge toggle registers/unregisters the primer live +(`knowledgePrimerControl`). `knowledge.enabled: false` or an empty keyword set +means the kick simply has no knowledge section. + +## 2. Knowledge: pull access + +Agents (and external tools) can fetch what the primer did not push: +`GET /api/knowledge/toc` and `/api/knowledge/entry/{id}` (pass `agent=` to +apply that agent's scope), `GET /api/knowledge/effective?agent=` to see what +a given agent is allowed to see and why entries are excluded, the public +`POST /mcp/knowledge` MCP (`knowledge_toc`, `knowledge_get`), `hivectl +knowledge search|list`, and `bd kb search` inside the pod. See +[knowledge-toc.md](knowledge-toc.md) and +[public-knowledge-mcp.md](public-knowledge-mcp.md). + +## 3. AGENTS.md / CLAUDE.md: injected by Hive, not read natively + +Each agent's CLI starts in `/data/agents/` (`crux.DefaultWorkDir`), +not inside a repository checkout, so the CLIs' native instruction-file +discovery (`AGENTS.md` for Codex/Copilot, `CLAUDE.md` for Claude Code, +`GEMINI.md`) finds **nothing from the monitored repo** at launch. Repo +instructions reach agents only through the scheduler: `primeAgentsMd` parses +`//AGENTS.md` for the agent's *primary* repo and +prepends the rendered block to the knowledge section (`templates.go`). Without +`project.checkouts_dir` (or a matching `policies.local_dir`) the block is +empty — see [agents-md.md](agents-md.md). Skills declared on the agent are +prepended the same way ([skills.md](skills.md)); the hive-wide house style +precedes both ([house-style.md](house-style.md)). + +Two instruction files *are* read natively: a user-level +`~/.claude/CLAUDE.md` / `~/.gemini/GEMINI.md` if one exists — and because +those directories are fleet-shared today (next section), such a file applies +to every Claude or Gemini agent on the hive. + +## 4. CLI memory and conversation: what Hive does and does not touch + +**Conversation.** `clear_on_kick` defaults to `true` (`ClearOnKick`, +`src/pkg/config/config.go`). `deliverKickLocked` in +`src/crux/agent/manager_kick.go` types `/clear` ⏎ before the kick text for +every TUI backend (agy headless starts a new conversation instead). The one +exception is a PR follow-up delivered with `SendResumeKick` +(`pr_session_resume.go`), which deliberately continues the authoring +conversation. `/clear` empties the chat; it does not touch anything on disk. + +**Home layout** (`interactive_home.go`, `manager_env.go`, `manager_homes.go`): + +- Per-UID agents get `HOME=/data/home/agents/` (`AgentHome`, + `interactiveHomePath`) with per-agent `XDG_DATA_HOME`/`XDG_STATE_HOME` under + `~/.local` (`setupAgentXDGDirs`). `/data/home` is the persistent volume. +- `interactiveHomeBridgeDirs` symlinks `.claude`, `.config`, `.codex`, `.bob`, + `.gemini` and `.cache` from every per-agent home to the **same** directories + under `/data/home`, so one login authenticates the fleet. Anything a CLI + keeps *inside* those directories is therefore fleet-shared and persistent. +- Real per-agent directories with one narrow bridge: `~/.copilot` (only + `config.json` shared — `setupCopilotHome`, #9444), `~/.local`, + `.claude.json`, `.npm`, `.gitconfig` (#9478). Codex gets + `CODEX_HOME=/data/home/.codex-` with only `auth.json` symlinked to the + shared login (`setupCodexHome`, `codexHomePath`). +- `HIVE_SHARED_AGENT_HOME=1` forces the legacy single shared home for all. +- Inference (API-routed) agents use throwaway homes under + `/tmp/.claude-inference-home-` that vanish with the pod. +- Contributor containers stage only `.credentials.json` and `settings.json` + into the container (`CLAUDE_STAGE_ALLOWLIST` in the `justfile`, #7836); + transcripts, history and memory stay on the host. This is the one place + memory is isolated by construction. + +**No memory handling.** Nothing in Hive sets a CLI's "disable auto-memory" +option (`inferenceSettingsSeed` seeds permissions/onboarding/remote-control +keys only), prunes a `memory/` directory, or clears memory on kick, `/clear` +or restart. What each CLI remembers, it remembers across kicks and pod +restarts, and — for the bridged directories — across agents. + +### What persists across what + +Columns: a cadence **kick**; an explicit **`/clear`**; a **pod restart**; a +**new agent** added to the same hive. ✓ = survives / is visible, ✗ = gone / +not visible. + +| | Kick | `/clear` | Pod restart | New agent in same hive | +|---|---|---|---|---| +| **Knowledge** (`# Relevant Knowledge`, AGENTS.md, skills) | re-selected for *this* kick's issues | n/a — re-injected next kick | ✓ vaults on `/data` | ✓ same stores, filtered by its own `agent_scopes` | +| **Conversation** — all CLIs | ✗ (`/clear` first, unless `clear_on_kick: false` or a resume kick) | ✗ | ✗ (CLI relaunched) | ✗ | +| **Claude Code memory** (`~/.claude/projects//memory/`, `~/.claude/CLAUDE.md`, `history.jsonl`) | ✓ | ✓ | ✓ (`/data/home/.claude` is shared & persistent) | ✓ readable: the tree is fleet-shared; auto-memory is keyed by cwd `/data/agents/` so files are per-agent by path, but `CLAUDE.md`/history are one copy | +| **Codex memory** (`$CODEX_HOME`) | ✓ | ✓ | ✓ (`/data/home/.codex-`) | ✗ per-agent dir; only `auth.json` shared | +| **Copilot CLI memory/sessions** (`~/.copilot`) | ✓ | ✓ | ✓ | ✗ per-agent dir (#9444); only `config.json` shared | +| **Gemini memory** (`~/.gemini/GEMINI.md`, state) | ✓ | ✓ | ✓ | ✓ whole directory shared | +| **Bob memory/settings** (`~/.bob`) | ✓ | ✓ | ✓ | ✓ whole directory shared | + +The dashboard's cost accounting reads the shared Claude tree +(`data.claude_sessions_dir`, default `/data/home/.claude/projects`, and +attributes sessions by the `-data-agents-` project slug — +`src/pkg/tokens/claude_scanner.go`), which is why that tree is shared on +purpose today. + +## Known gaps / in progress + +Do not treat the following as current behaviour; follow the links. + +- [#17031](https://github.com/hivecommons/hive/issues/17031) — Claude/Gemini/Bob + persistent memory is shared across every agent on a hive. Fix in flight: + [PR #17039](https://github.com/hivecommons/hive/pull/17039) makes `.claude`, + `.gemini` and `.bob` real per-agent directories with only credential/settings + entries bridged (the `setupCopilotHome` pattern) and teaches the token + collector to scan `/data/home/agents/*/.claude/projects`. Until it merges, + the table above stands. +- [#17032](https://github.com/hivecommons/hive/issues/17032) — disable CLI + auto-memory by default and add a `clear_memory_on_kick` option alongside + `clear_on_kick`. +- [#17033](https://github.com/hivecommons/hive/issues/17033) — drain CLI + auto-memory into beads/knowledge at session end instead of a hidden + `MEMORY.md` (revives open question 5 of + [design/knowledge-system.md](design/knowledge-system.md)). +- Repo `AGENTS.md` injection still requires `project.checkouts_dir`; nearest- + file (`ParseNearest`) targeting is deferred ([agents-md.md](agents-md.md)). + +Closed history worth knowing: #7836 (contributor `~/.claude` staging), #9444 +(Copilot session-state isolation), #9478 (per-agent `.gitconfig`), #11201 +(`knowledge.agent_scopes`), #12124/#12129 (primer and AGENTS.md size caps), +#10524 (knowledge TOC, lifecycle, suggest). + +## Related + +[design/knowledge-system.md](design/knowledge-system.md) · +[knowledge-curator.md](knowledge-curator.md) · +[knowledge-lifecycle.md](knowledge-lifecycle.md) · +[knowledge-toc.md](knowledge-toc.md) · [agents-md.md](agents-md.md) · +[skills.md](skills.md) · [agent-configuration.md](agent-configuration.md) · +[contributor-relay.md](contributor-relay.md) diff --git a/src/docs/agents-md.md b/src/docs/agents-md.md index 827c9598cd..a060cb6f24 100644 --- a/src/docs/agents-md.md +++ b/src/docs/agents-md.md @@ -252,6 +252,10 @@ still wins. ## What to read next +- [agent-knowledge-and-memory.md](agent-knowledge-and-memory.md) — where this + block sits in the whole kick (skills, AGENTS.md, `# Relevant Knowledge`), why + the CLI does not read the repo's `AGENTS.md` natively, and what `/clear` + does and does not reset. - [house-style.md](house-style.md) — the short, hive-wide *how to change things* section that precedes this block in every kick; a repo overrides it with a root `HIVE_STYLE.md`, read from the same checkout root. diff --git a/src/docs/documentation-map.md b/src/docs/documentation-map.md index f9d3addf17..064e9efe9b 100644 --- a/src/docs/documentation-map.md +++ b/src/docs/documentation-map.md @@ -169,6 +169,7 @@ Documentation for the current Hive line (branch `v5`; the code and docs live und - [Custom dashboard stylesheets](custom-stylesheets.md) — operator-supplied CSS for the dashboard and public snapshot. - [Branding a hive](branding.md) — persistent per-deployment name, mark, and colours via `/branding/branding.json` and `custom.css` (`HIVE_BRANDING_JSON`/`HIVE_BRANDING_CSS`); distinct from the per-request `?style=` stylesheets above. - [Portable AgentDefinition format](../AGENT-DEFINITION.md) — standalone YAML schema for importing/exporting agent definitions. +- [How Hive injects knowledge and memory into agents](agent-knowledge-and-memory.md) — the three context layers (Hive knowledge primer, CLI conversation, CLI persistent memory), the kick-primer path and its scoping/caps, why repo `AGENTS.md` is injected rather than read natively, which per-agent home directories are fleet-shared, and what survives a kick, `/clear`, pod restart or a new agent. - [Knowledge connectors](knowledge-connectors.md) — **Owner configuration:** connect, validate, and sync external document/wiki sources; includes scope and API reference. - [Knowledge curator](knowledge-curator.md) — configure fact extraction, sources, vaults, document imports, code maps, and retention. - [Knowledge confidence scoring](knowledge-confidence.md) — interpret confidence badges and score explanations. diff --git a/src/docs/env-vars.md b/src/docs/env-vars.md index 527459f35c..adc585bfea 100644 --- a/src/docs/env-vars.md +++ b/src/docs/env-vars.md @@ -219,7 +219,7 @@ new value at the same time. | `HIVE_OUTPUT_LOG_DIR` | No | `/data/.hive/logs` | Directory (`0700`, files `0600`) where the hive keeps the `proxy`, `hive-output`, `litellm` and `requests` log sources read by the admin MCP `log_read` tool ([#11944](https://github.com/hivecommons/hive/issues/11944)); each keeps at most 5 files of 10 MiB and nothing older than 7 days. The `starts` and `crash` sources ([#11945](https://github.com/hivecommons/hive/issues/11945)) live here too: `starts.log` (newest 200 starts), `start-current.json` and at most 10 `crash-.log` files of 1 MiB. See [design/admin-mcp.md](design/admin-mcp.md#output-and-request-log-sources). | | `HIVE_PROXY_EGRESS_MARK` | No | `0x1112` | Packet mark exempted from the MITM egress redirect. | | `HIVE_PROXY_ADVISORY_OK` | No | `false` | Allows the spoke to start when the forced-proxy egress redirect cannot be installed (no `CAP_NET_ADMIN`/iptables). Enforcement becomes advisory-only — agents can bypass the proxy. Also gates whether the Go proxy trusts a self-asserted `Proxy-Authorization` header as agent identity when its UID map is unavailable (N7, #3841) — off by default, an unidentified caller is treated as `ADVISORY` (writes blocked) rather than whatever name it claims. See [security-model.md](security-model.md#5-network-and-credentials). | -| `HIVE_PROXY_ALLOW_PRIVATE_FORWARD` | No | `false` | Lifts the plain-HTTP forward destination guard ([#14082](https://github.com/hivecommons/hive/issues/14082)). Off by default: a non-CONNECT request through the proxy is dialled only when the resolved address is globally routable, so an agent cannot use the proxy to reach loopback services beside it, the link-local cloud metadata endpoint (`169.254.169.254`), RFC 1918/CGNAT cluster addresses or multicast, and the refusal is a `403` naming the class. Only the exact string `true` turns it on, for deployments that genuinely egress to a private network over plain HTTP (a LAN mirror on a native install). Loopback refusals stay at `DEBUG` and are not counted as violations, because agents are already handed `NO_PROXY` for loopback (#14076). | +| `HIVE_PROXY_ALLOW_PRIVATE_FORWARD` | No | `false` | Lifts the proxy's destination guard ([#14082](https://github.com/hivecommons/hive/issues/14082), [#16785](https://github.com/hivecommons/hive/pull/16785)). Off by default: a plain-HTTP (non-CONNECT) forward, a CONNECT to a host the proxy does not inspect, and a transparent-TLS passthrough are each dialled only when the resolved address is globally routable, so an agent cannot use the proxy to reach loopback services beside it, the link-local cloud metadata endpoint (`169.254.169.254`), RFC 1918/CGNAT cluster addresses or multicast, and the refusal is a `403` naming the class. Only the exact string `true` turns it on, and it lifts the guard for all three legs at once, for deployments that genuinely egress to a private network through the proxy (a LAN mirror on a native install); the proxy logs a boot-time warning when it is set. Loopback refusals stay at `DEBUG` and are not counted as violations, because agents are already handed `NO_PROXY` for loopback (#14076). | | `HIVE_PROXY_INJECT_GH_AUTH` | No | unset (off on every hive, including hosted) | Exact `true` enables proxy-side App-token injection. **Requires GitHub App auth. Refuses startup (exit 19) with `HIVE_PROXY_ADVISORY_OK=true`.** Other spellings do not enable it. Read [behavior and exceptions](#proxy-credential-injection). | | `HIVE_DEPLOYMENT_RUNTIME` | No | none (config `deployment.runtime` fallback; hub-registered Kubernetes hives are auto-detected) | Declares the deployment runtime (`kubernetes`, `podman-quadlet`, or `docker-compose`) so the dashboard can offer standalone upgrades. Unset/unrecognized means `unknown` and the upgrade button stays hidden. See [dashboard-standalone-upgrades.md](dashboard-standalone-upgrades.md). | | `HIVE_DEPLOYMENT_PODMAN_MODE` | Only when runtime is `podman-quadlet` | none (config `deployment.podman_mode` fallback) | `rootless` (`systemctl --user`) or `rootful` (system manager). Without an explicit mode the Podman/Quadlet upgrade path stays disabled. | diff --git a/src/docs/knowledge-curator.md b/src/docs/knowledge-curator.md index 73da40f1c0..2695cf2730 100644 --- a/src/docs/knowledge-curator.md +++ b/src/docs/knowledge-curator.md @@ -627,7 +627,8 @@ Maps an agent name to the `layers`, `repos`, `types`, `tags` and are restricted to. A missing agent or an empty field is unrestricted (the default). Unknown layer or lifecycle state names fail config load. See [Per-agent scopes](knowledge-toc.md#per-agent-scopes) for the intersection -rules. +rules, and [agent-knowledge-and-memory.md](agent-knowledge-and-memory.md) for +how the scope is applied on the kick path end to end. ```yaml knowledge: diff --git a/src/docs/manual-provisioning.md b/src/docs/manual-provisioning.md index fbf2ffb391..0b808a6662 100644 --- a/src/docs/manual-provisioning.md +++ b/src/docs/manual-provisioning.md @@ -1689,6 +1689,22 @@ provision. Two pools, one per method type: - **Hub-reachable-cluster** placeholders serve **public** methods — provisioned via the API (Path A) and left running at **`replicas: 1`**. +**Auto-pick on approve.** When an admin approves a provision request without +choosing a specific placeholder, the hub picks one from the pool matching the +request's `auth_method`: `private` → the heartbeat-only/GPU pool, anything else +→ the default hub-reachable pool (`hive-oke`). The **public preference is +soft**: if the default pool has no `available` placeholder, Auto-pick falls back +to any *other* cluster registered in `clusters.json` that has one — most +available first, ties broken by cluster ID — and logs one line +(`approve: pool hive-oke empty, falling back to lke648397`). Two pins stay +**hard** and never fall back: a `private` request only ever draws from the GPU +pool, and the GPU pool is never used to satisfy a public request. Clusters with +placeholders on disk but no `clusters.json` entry are skipped. The admin's +explicit pick in the approve modal is strict — if that placeholder is not +available the approve returns `409`, regardless of capacity elsewhere. +Only when every eligible pool is empty does Auto-pick return +`409 no available placeholder hive in pool "hive-oke" or any other registered pool`. + > **Gotcha — placeholder IDs are date + random suffix, NOT sequential integers.** > The live pool uses `hosted-available---`, where > `` is a short unique token (4 lowercase base36 chars) — e.g. diff --git a/src/docs/native-merge-queue.md b/src/docs/native-merge-queue.md index ebd97271be..be526e74c3 100644 --- a/src/docs/native-merge-queue.md +++ b/src/docs/native-merge-queue.md @@ -220,13 +220,16 @@ metadata, not a fabricated last-member PR event. in-progress); the queued run re-reads mutable PR state after the prior run finishes, preserving fail-closed semantics. Labels the policy never reads (`size/*`, `dco-signoff`, stage labels, ...) skip the wait: the job replays - the latest completed `native-queue-policy` `success` on the same head and - still checks the live PR for holds, draft and head moves. When there is no + the latest completed `native-queue-policy` `success` on the same head, + still checks the live PR for holds, draft and head moves, and re-validates + the five policy checks on that head — a provider re-run red since the + original verdict fails the job instead of being replayed. When there is no prior success to replay (never evaluated, or the last wait failed or timed - out) or GitHub returns a 5xx, the job falls through to the full wait so a - stale red verdict is never pinned onto later cosmetic flips. `hold`, - `needs-human`, `release-in-progress` and `do-not-merge*` flips always get - the full wait. + out), a policy provider is still pending or missing on the head, the DCO + provider reports UNKNOWN, or GitHub returns a 5xx, the job falls through to + the full wait so a stale verdict is never pinned onto later cosmetic flips. + `hold`, `needs-human`, `release-in-progress` and `do-not-merge*` flips + always get the full wait. - On a group it executes the event's trusted `base_sha` validator. It reads the live queue prefix ending at the event's exact synthetic head, verifies contiguous membership and the base/head commit chain, and checks diff --git a/src/docs/operator-reference/README.md b/src/docs/operator-reference/README.md index bef691a682..bdf0526242 100644 --- a/src/docs/operator-reference/README.md +++ b/src/docs/operator-reference/README.md @@ -320,7 +320,7 @@ Go callers keep reading `cfg.Agents`, `cfg.Governor.Gateways` and so on through | `knowledge.bead_synthesizer.publish` | Off (`enabled: false`). | Publishes synthesized facts into a repository through one PR on `hive/bead-synth-knowledge`: `enabled`, `repo` (empty = `project.primary_repo`), `path` (default `docs/knowledge/`; `.hive/knowledge/` is the alternative; relative, no `..`), `base_branch` (empty = repo default branch), `mode` (`pr` default or `commit`), `cadence` (`per_run` default or `daily`). ACMM L<=4 hives publish via PR only; `commit` is warned about and still opens a PR. See [knowledge-curator.md](../knowledge-curator.md#repository-publish-knowledgebead_synthesizerpublish). | | `variables.security.*` | Deny by default. | `allow_exec`, `allow_http`, and GitHub prompt-source allowlists are honored only from the trusted seed, not dashboard overlays. | | `agents..prompt_source` (size cap) | 96 KiB (`promptsrc.DefaultMaxPromptBytes`); `HIVE_PROMPT_SOURCE_MAX_BYTES` overrides it. | A fetched kick template over the cap is **rejected whole, not truncated** ([#12130](https://github.com/hivecommons/hive/issues/12130)): the kick uses the last successfully fetched copy if there is one, otherwise the ordinary `kick_template` → pack → convention → embedded-default chain, and the scheduler logs `prompt from GitHub exceeds size cap; REJECTED whole (not truncated), falling back` with `repo`, `path`, `size_bytes` and `cap_bytes`. Saving an agent whose source is over the cap fails in the dashboard with the same sizes rather than baking a cut-off policy. A half-policy is a policy with its closing rules missing; no remote policy is the safer failure. Every shipped template fits with >2× headroom (`TestShippedTemplatesFitUnderCap`). | -| `data.claude_sessions_dir` | `/data/home/.claude/projects` | Where the dashboard reads Claude Code session JSONL for per-agent token/cost accounting. Point it at the agents' real session directory if you relocate `HOME`. | +| `data.claude_sessions_dir` | `/data/home/.claude/projects` | Where the dashboard reads Claude Code session JSONL for per-agent token/cost accounting. Point it at the agents' real session directory if you relocate `HOME`. This shared tree is also where Claude Code keeps its persistent auto-memory; see [agent-knowledge-and-memory.md](../agent-knowledge-and-memory.md) for which per-agent home directories are fleet-shared and what `clear_on_kick` does and does not clear. | | `data.copilot_sessions_dir` | `/data/home/.copilot/session-state` | Same, for the Copilot CLI backend's session state. | ### Label-driven approval diff --git a/src/docs/security-model.md b/src/docs/security-model.md index 1d9a87d97d..9e9673f3ca 100644 --- a/src/docs/security-model.md +++ b/src/docs/security-model.md @@ -196,17 +196,23 @@ process cannot forge. Only `api.github.com` is inspected; other hosts are tunnelled without inspection. See [net-admin-requirement.md](net-admin-requirement.md). -**Plain-HTTP forwards.** The redirect covers port 443 only, so plain HTTP -reaches the proxy through its explicit proxy port. Those forwards are -identified like any other request and dialled only when the resolved address -is globally routable: loopback services beside the proxy, the link-local -metadata endpoint, RFC 1918/CGNAT cluster addresses and multicast are refused -with `403` and counted as violations (loopback excepted — agents are handed -`NO_PROXY` for it, so it is logged as the client misconfiguration it is). The -check runs at dial time on the resolved address, so a hostname pointed inward -is refused on the same evidence as a literal IP. Set -`HIVE_PROXY_ALLOW_PRIVATE_FORWARD=true` only where a private-network plain-HTTP -destination is intended. +**Plain-HTTP forwards and uninspected tunnels.** The redirect covers port 443 +only, so plain HTTP reaches the proxy through its explicit proxy port. Those +forwards are identified like any other request and dialled only when the +resolved address is globally routable: loopback services beside the proxy, the +link-local metadata endpoint, RFC 1918/CGNAT cluster addresses and multicast +are refused with `403` and counted as violations (loopback excepted — agents +are handed `NO_PROXY` for it, so it is logged as the client misconfiguration it +is). The same policy applies to the tunnels the proxy does not inspect: a +CONNECT to a host other than the inspected API and the transparent-TLS +passthrough for such a host are dialled through the same guarded dialer, so +`CONNECT 169.254.169.254:80` or an SNI naming a cluster address is refused on +the same terms rather than tunnelled from the proxy's own uid. The check runs +at dial time on the resolved address, so a hostname pointed inward is refused +on the same evidence as a literal IP. Set +`HIVE_PROXY_ALLOW_PRIVATE_FORWARD=true` only where a private-network +destination is intended; it lifts the guard for all three legs at once, and the +proxy warns at boot when it is set. **Hub-to-spoke channel.** Heartbeats carry a per-hive bearer key derived from the hub master secret and are compared in constant time. The payload is diff --git a/src/pkg/hub/autopick_pool_fallback_test.go b/src/pkg/hub/autopick_pool_fallback_test.go new file mode 100644 index 0000000000..970d7d68c7 --- /dev/null +++ b/src/pkg/hub/autopick_pool_fallback_test.go @@ -0,0 +1,246 @@ +package hub + +import ( + "log/slog" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" +) + +// Cluster IDs used by the fallback tests. fallbackClusterID stands in for the +// second public pool (lke648397 in production); unregisteredClusterID is a +// pool that has placeholders on disk but no clusters.json entry. +const ( + fallbackClusterID = "lke648397" + secondFallbackCluster = "aks-eu" + unregisteredClusterID = "ghost-cluster" +) + +// placeholderSpec is one available placeholder to seed for a test case. +type placeholderSpec struct { + id string + cluster string +} + +func seedPlaceholders(t *testing.T, specs []placeholderSpec) { + t.Helper() + for _, p := range specs { + if err := saveSaaSHive(&SaaSHive{ID: p.id, Owner: hubAdminUsername, Status: statusAvailable, ClusterID: p.cluster}); err != nil { + t.Fatal(err) + } + } +} + +func newFallbackTestHub() *HubServer { + return &HubServer{ + logger: slog.Default(), + hubSecret: testHubSecret, + saveCh: make(chan struct{}, 1), + clusters: map[string]ClusterConfig{ + defaultClusterID: *dynamicCluster(), + fallbackClusterID: {ID: fallbackClusterID, Name: "lke"}, + secondFallbackCluster: {ID: secondFallbackCluster, Name: "aks"}, + gpuClusterID: {ID: gpuClusterID, Name: "gpu", PullOnly: true}, + }, + } +} + +// TestAutoPickPlaceholderFallback is the table over (preferred pool empty / +// non-empty) x (other pools empty / non-empty) plus the hard pins that must +// never fall back. +func TestAutoPickPlaceholderFallback(t *testing.T) { + tests := []struct { + name string + preferred string + placeholders []placeholderSpec + wantID string + wantPool string + }{ + { + name: "preferred non-empty, others non-empty -> preferred wins", + preferred: defaultClusterID, + placeholders: []placeholderSpec{ + {"oke-1", defaultClusterID}, + {"lke-1", fallbackClusterID}, + {"lke-2", fallbackClusterID}, + }, + wantID: "oke-1", + wantPool: defaultClusterID, + }, + { + name: "preferred empty, others empty -> nothing", + preferred: defaultClusterID, + placeholders: nil, + wantID: "", + wantPool: defaultClusterID, + }, + { + name: "preferred empty, one other pool non-empty -> fall back", + preferred: defaultClusterID, + placeholders: []placeholderSpec{ + {"lke-2", fallbackClusterID}, + {"lke-1", fallbackClusterID}, + }, + wantID: "lke-1", + wantPool: fallbackClusterID, + }, + { + name: "preferred empty, several pools -> most available first", + preferred: defaultClusterID, + placeholders: []placeholderSpec{ + {"aks-1", secondFallbackCluster}, + {"lke-1", fallbackClusterID}, + {"lke-2", fallbackClusterID}, + }, + wantID: "lke-1", + wantPool: fallbackClusterID, + }, + { + name: "preferred empty, equal counts -> tie-break by cluster id", + preferred: defaultClusterID, + placeholders: []placeholderSpec{ + {"lke-1", fallbackClusterID}, + {"aks-1", secondFallbackCluster}, + }, + wantID: "aks-1", + wantPool: secondFallbackCluster, + }, + { + name: "preferred empty, only GPU pool has placeholders -> no fallback onto GPU", + preferred: defaultClusterID, + placeholders: []placeholderSpec{ + {"gpu-1", gpuClusterID}, + }, + wantID: "", + wantPool: defaultClusterID, + }, + { + name: "preferred empty, only unregistered cluster has placeholders -> skipped", + preferred: defaultClusterID, + placeholders: []placeholderSpec{ + {"ghost-1", unregisteredClusterID}, + }, + wantID: "", + wantPool: defaultClusterID, + }, + { + name: "private request: GPU pool empty, public pools full -> stays pinned", + preferred: gpuClusterID, + placeholders: []placeholderSpec{ + {"oke-1", defaultClusterID}, + {"lke-1", fallbackClusterID}, + }, + wantID: "", + wantPool: gpuClusterID, + }, + { + name: "private request: GPU pool non-empty -> GPU placeholder", + preferred: gpuClusterID, + placeholders: []placeholderSpec{ + {"gpu-1", gpuClusterID}, + {"oke-1", defaultClusterID}, + }, + wantID: "gpu-1", + wantPool: gpuClusterID, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + cleanup := helperSetupTempDirs(t) + defer cleanup() + mkUser(t, hubAdminUsername) + seedPlaceholders(t, tc.placeholders) + + s := newFallbackTestHub() + gotID, gotPool := s.autoPickPlaceholder(tc.preferred) + if gotID != tc.wantID || gotPool != tc.wantPool { + t.Errorf("autoPickPlaceholder(%q) = (%q, %q), want (%q, %q)", tc.preferred, gotID, gotPool, tc.wantID, tc.wantPool) + } + }) + } +} + +// TestHandleApproveProvisionPoolFallback drives the handler end to end: a +// public request with an empty default pool is served from another pool and +// the claimed hive keeps that pool's cluster_id; an admin's explicit pick is +// still strict (409 when the chosen placeholder is not available) and a fully +// empty fleet still 409s with a message that names the fallback scope. +func TestHandleApproveProvisionPoolFallback(t *testing.T) { + pending := func(user string) *ProvisionRequest { + return &ProvisionRequest{ + Username: user, Org: "acme", Repos: "repo", PrimaryRepo: "repo", ACMMLevel: 2, + AuthMethod: "public", RequestedAt: time.Now().UTC().Format(time.RFC3339), Status: provisionStatusPending, + } + } + + tests := []struct { + name string + placeholders []placeholderSpec + body string + wantCode int + wantHive string + wantCluster string + wantBodyHas string + }{ + { + name: "auto-pick falls back to the other pool", + placeholders: []placeholderSpec{{"lke-1", fallbackClusterID}}, + body: "", + wantCode: http.StatusOK, + wantHive: "lke-1", + wantCluster: fallbackClusterID, + }, + { + name: "auto-pick with every pool empty -> 409 naming the fallback scope", + placeholders: nil, + body: "", + wantCode: http.StatusConflict, + wantBodyHas: "or any other registered pool", + }, + { + name: "explicit pick of an unavailable placeholder stays strict even though another pool has capacity", + placeholders: []placeholderSpec{{"lke-1", fallbackClusterID}}, + body: `{"hive_id":"oke-missing"}`, + wantCode: http.StatusConflict, + wantBodyHas: "selected placeholder is not available", + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + cleanup := helperSetupTempDirs(t) + defer cleanup() + mkUser(t, hubAdminUsername) + seedPlaceholders(t, tc.placeholders) + if err := saveProvisionRequest(pending("bob")); err != nil { + t.Fatal(err) + } + + s := newFallbackTestHub() + rec := httptest.NewRecorder() + req := setPathValue(reqWithUser(http.MethodPut, "/approve-prov", tc.body, hubAdminUsername), "username", "bob") + s.handleApproveProvision(rec, req) + + if rec.Code != tc.wantCode { + t.Fatalf("status = %d, want %d (body=%s)", rec.Code, tc.wantCode, rec.Body.String()) + } + if tc.wantBodyHas != "" && !strings.Contains(rec.Body.String(), tc.wantBodyHas) { + t.Errorf("body = %s, want it to contain %q", rec.Body.String(), tc.wantBodyHas) + } + if tc.wantHive == "" { + return + } + got := loadSaaSHive(tc.wantHive) + if got == nil { + t.Fatalf("hive %q missing after approve", tc.wantHive) + } + if got.Owner != "bob" || got.Status != statusAssigned { + t.Errorf("hive %q owner/status = %q/%q, want bob/%s", tc.wantHive, got.Owner, got.Status, statusAssigned) + } + if got.ClusterID != tc.wantCluster { + t.Errorf("hive %q ClusterID = %q, want %q (fallback must keep the placeholder's real pool)", tc.wantHive, got.ClusterID, tc.wantCluster) + } + }) + } +} diff --git a/src/pkg/hub/saas_provisioning.go b/src/pkg/hub/saas_provisioning.go index 28cec59029..ae46e7d3a7 100644 --- a/src/pkg/hub/saas_provisioning.go +++ b/src/pkg/hub/saas_provisioning.go @@ -613,8 +613,11 @@ func (s *HubServer) handleApproveProvision(w http.ResponseWriter, r *http.Reques // Approving now ASSIGNS an available placeholder instead of just bumping // quota for a manual provision. Pick the pool from the request's auth_method - // (private → the heartbeat-only cluster / GPU pool, otherwise → the hub-reachable cluster / public pool). If no - // placeholder is available in that pool, tell the admin to provision more. + // (private → the heartbeat-only cluster / GPU pool, otherwise → the hub-reachable cluster / public pool). + // The public preference is soft: when that pool is empty, Auto-pick falls + // back to any other registered non-GPU pool with a placeholder (see + // autoPickPlaceholder). Only when every eligible pool is empty does the + // admin get told to provision more. pool := poolClusterForAuthMethod(pr.AuthMethod) hiveID := strings.TrimSpace(approveBody.HiveID) if hiveID != "" { @@ -626,10 +629,19 @@ func (s *HubServer) handleApproveProvision(w http.ResponseWriter, r *http.Reques return } } else { - hiveID = findAvailablePlaceholder(pool) + var pickedPool string + hiveID, pickedPool = s.autoPickPlaceholder(pool) + if hiveID != "" && pickedPool != pool { + s.logger.Info(fmt.Sprintf("approve: pool %s empty, falling back to %s", pool, pickedPool), "target", targetUsername, "hive", hiveID) + pool = pickedPool + } } if hiveID == "" { - http.Error(w, fmt.Sprintf(`{"error":"no available placeholder hive in pool %q — provision more placeholders"}`, pool), http.StatusConflict) + scope := "" + if pool != gpuClusterID { + scope = " or any other registered pool" + } + http.Error(w, fmt.Sprintf(`{"error":"no available placeholder hive in pool %q%s — provision more placeholders"}`, pool, scope), http.StatusConflict) return } @@ -904,6 +916,69 @@ func findAvailablePlaceholder(clusterID string) string { return "" } +// autoPickPlaceholder implements Auto-pick for handleApproveProvision: it +// returns the ID of an available placeholder and the pool it came from. +// +// The preferred pool is tried first. If it is empty AND the preference is +// soft (the public/default pool), the pick falls back to any OTHER cluster +// registered in clusters.json that has an available placeholder — ordered by +// most available first, tie-broken by cluster ID, so the choice is +// deterministic. Two pins stay hard and never fall back: a private request +// stays on the GPU pool (its spokes are heartbeat-only and the pool is a +// tenancy boundary), and the GPU pool is never used to satisfy a public +// request. Returns ("", preferredPool) when no eligible pool has a +// placeholder. +func (s *HubServer) autoPickPlaceholder(preferredPool string) (hiveID, pool string) { + if id := findAvailablePlaceholder(preferredPool); id != "" { + return id, preferredPool + } + if preferredPool == gpuClusterID { + return "", preferredPool + } + + type poolCount struct { + id string + count int + first string + } + byPool := map[string]*poolCount{} + for _, h := range listSaaSHives() { + if h.Status != statusAvailable || !isHubAdmin(h.Owner) { + continue + } + cid := clusterIDForHive(&h) + if cid == preferredPool || cid == gpuClusterID { + continue + } + if _, registered := s.clusters[cid]; !registered { + continue + } + pc := byPool[cid] + if pc == nil { + pc = &poolCount{id: cid, first: h.ID} + byPool[cid] = pc + } + pc.count++ + if h.ID < pc.first { + pc.first = h.ID + } + } + if len(byPool) == 0 { + return "", preferredPool + } + candidates := make([]*poolCount, 0, len(byPool)) + for _, pc := range byPool { + candidates = append(candidates, pc) + } + sort.Slice(candidates, func(i, j int) bool { + if candidates[i].count != candidates[j].count { + return candidates[i].count > candidates[j].count + } + return candidates[i].id < candidates[j].id + }) + return candidates[0].first, candidates[0].id +} + // AvailablePlaceholder is one row of the approve-picker dropdown: an available // placeholder hive the admin can assign a provision request to. type AvailablePlaceholder struct { diff --git a/src/proxy/package-lock.json b/src/proxy/package-lock.json index 86be927a62..f28302412e 100644 --- a/src/proxy/package-lock.json +++ b/src/proxy/package-lock.json @@ -12,7 +12,7 @@ "http-proxy-middleware": "^4.2.1" }, "devDependencies": { - "ws": "^8.21.3" + "ws": "^8.22.1" } }, "node_modules/accepts": { @@ -965,9 +965,9 @@ "license": "ISC" }, "node_modules/ws": { - "version": "8.21.3", - "resolved": "https://registry.npmjs.org/ws/-/ws-8.21.3.tgz", - "integrity": "sha512-201TZ/kPWxoPr/OKWjquZR1SWKXcvxdH+e1xrx89b3YbmzLMFCLfnaG1HFIgWzJOEWZ7MvpK++odZufgYR50Rw==", + "version": "8.22.1", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.22.1.tgz", + "integrity": "sha512-oyL6OPYme/CmiFvNuGzeoxCJ4nOrZE2O1FwM9MWM1AsGri6nLXB2pg+lNA8aLeXMSav0J6eIVS1j6vUPzUtK9g==", "dev": true, "license": "MIT", "engines": { diff --git a/src/proxy/package.json b/src/proxy/package.json index 812c1cb73b..7596b682e1 100644 --- a/src/proxy/package.json +++ b/src/proxy/package.json @@ -14,6 +14,6 @@ "http-proxy-middleware": "^4.2.1" }, "devDependencies": { - "ws": "^8.21.3" + "ws": "^8.22.1" } }