From 1258b26db80700c5a8a60508b1cfcda54569df9e Mon Sep 17 00:00:00 2001 From: Danathar Date: Wed, 28 Oct 2026 16:13:45 +0000 Subject: [PATCH 1/5] =?UTF-8?q?=E2=9C=A8=20feat:=20let=20a=20sandboxed=20r?= =?UTF-8?q?eviewer=20post=20reviews=20through=20a=20workspace=20spool?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A reviewer whose kicks run in a sandbox (Podman, or a Kubernetes Job on an accelerator node via sandbox.runtime: job) had no path to the pod-local review relay, so it could not post a verdict. Every sandbox kick now gets a review spool at .hive/review-requests in its workspace; hive-review writes there when HIVE_REVIEW_REQUEST_DIR is set. The executor registers the spool with a manifest under /sandbox-review-spools, outside every workspace and owner-only. The review-request watcher moves spooled requests into hive-owned staging through an os.Root at the workspace, runs them through the same handler as the pod-local drop box (per-head cap, pause, lane allowlist, sentinel block, audit unchanged), attributes them to the manifest's agent rather than the file's, and mirrors results back into the spool. A finished kick waits briefly for the relay to drain the spool before returning. Closes #14986 Hive-Run: hivecommons/hive#14986 Hive-Plan: issue-14986 proposal (relay accepts verdict files from the sandbox workspace) Hive-Spec: hivecommons/hive#14986#proposal-1-4 Co-Authored-By: Claude Opus 5.5 Signed-off-by: Danathar --- bin/hive-review.sh | 6 +- bin/test_hive_review.sh | 14 + .../added-14986-sandbox-review-spool.md | 1 + src/crux/agent/manager_sandbox.go | 4 + src/crux/agent/sandbox_executor.go | 19 + src/crux/agent/sandbox_review_spool.go | 148 +++++++ src/crux/agent/sandbox_review_spool_test.go | 229 ++++++++++ src/crux/sandbox_review_spool.go | 42 ++ src/docs/github-write-surface.md | 2 +- src/docs/sandbox-isolation.md | 30 ++ src/pkg/github/read_untrusted_other.go | 4 + src/pkg/github/read_untrusted_unix.go | 6 + .../github/review_request_sandbox_spool.go | 406 ++++++++++++++++++ .../review_request_sandbox_spool_test.go | 305 +++++++++++++ src/pkg/github/review_request_watcher.go | 36 +- 15 files changed, 1242 insertions(+), 10 deletions(-) create mode 100644 changelog.d/added-14986-sandbox-review-spool.md create mode 100644 src/crux/agent/sandbox_review_spool.go create mode 100644 src/crux/agent/sandbox_review_spool_test.go create mode 100644 src/crux/sandbox_review_spool.go create mode 100644 src/pkg/github/review_request_sandbox_spool.go create mode 100644 src/pkg/github/review_request_sandbox_spool_test.go diff --git a/bin/hive-review.sh b/bin/hive-review.sh index 419e0392cd..a763d03e2d 100755 --- a/bin/hive-review.sh +++ b/bin/hive-review.sh @@ -49,7 +49,11 @@ set -euo pipefail -REQ_DIR="/var/run/hive-metrics/review-requests" +# Inside a sandbox (Podman or a Kubernetes Job) the pod-local relay dir is not +# reachable; hive sets HIVE_REVIEW_REQUEST_DIR to the kick's workspace spool and +# relays from there, attributing the request to the kick's own agent whatever +# this script writes in "agent" (hivecommons/hive#14986). +REQ_DIR="${HIVE_REVIEW_REQUEST_DIR:-/var/run/hive-metrics/review-requests}" REPO=""; NUMBER=""; EVENT=""; BODY=""; BODY_FILE=""; THREAD=""; VERDICT_FILE=""; VERDICT=""; REVISE=0 while [ $# -gt 0 ]; do diff --git a/bin/test_hive_review.sh b/bin/test_hive_review.sh index c5cbe9faa7..d5c43b187b 100755 --- a/bin/test_hive_review.sh +++ b/bin/test_hive_review.sh @@ -163,6 +163,20 @@ else echo " PASS: no .tmp file remains (atomic replace)"; PASS=$((PASS + 1)) fi +# --- sandbox spool (#14986): HIVE_REVIEW_REQUEST_DIR redirects the request --- +# Runs the UNMODIFIED script: the override is the script's own behavior. +echo "" +echo "--- sandbox spool via HIVE_REVIEW_REQUEST_DIR ---" +SPOOL_DIR="$TMPDIR_ROOT/workspace/.hive/review-requests" +env HIVE_AGENT="spoolbot" HIVE_REVIEW_REQUEST_DIR="$SPOOL_DIR" bash "$SCRIPT" --repo "org/repo" 12 --comment --body "ran on the card" >/dev/null 2>&1 || true +SPOOL_FILES=("$SPOOL_DIR"/spoolbot-*.json) +if [ -f "${SPOOL_FILES[0]}" ]; then + check "spooled request is a comment" "comment" "$(field "${SPOOL_FILES[0]}" event)" + check "spooled request names the PR" "12" "$(field "${SPOOL_FILES[0]}" number)" +else + echo " FAIL: request not written to HIVE_REVIEW_REQUEST_DIR"; FAIL=$((FAIL + 2)) +fi + echo "" echo "=== $PASS passed, $FAIL failed ===" [ "$FAIL" -eq 0 ] diff --git a/changelog.d/added-14986-sandbox-review-spool.md b/changelog.d/added-14986-sandbox-review-spool.md new file mode 100644 index 0000000000..1f6f7601d0 --- /dev/null +++ b/changelog.d/added-14986-sandbox-review-spool.md @@ -0,0 +1 @@ +- A sandboxed reviewer can now post reviews ([#14986](https://github.com/hivecommons/hive/issues/14986)). Every sandbox kick (Podman or `sandbox.runtime: job`) gets a review spool at `.hive/review-requests` in its workspace, and `hive-review` inside the sandbox writes there via `HIVE_REVIEW_REQUEST_DIR`. The review relay picks spooled requests up through the same policy path as the pod-local drop box, attributing each one to the kick's agent from a hive-owned manifest under `/sandbox-review-spools`, never from the file's contents. A reviewer on an accelerator spoke can now run the change on the card node and post its verdict from there. diff --git a/src/crux/agent/manager_sandbox.go b/src/crux/agent/manager_sandbox.go index 3c2ff299c8..6d2fb856c1 100644 --- a/src/crux/agent/manager_sandbox.go +++ b/src/crux/agent/manager_sandbox.go @@ -8,6 +8,7 @@ import ( "strings" "time" + "github.com/hivecommons/hive/crux" "github.com/hivecommons/hive/crux/config" "github.com/hivecommons/hive/crux/effects" "github.com/hivecommons/hive/crux/kubejob" @@ -148,6 +149,9 @@ func (m *Manager) startSandboxKickLocked(agent *AgentProcess, message string) er EnvAllowlist: agent.Config.SandboxEnvAllowlist(m.sandboxConfig), NetworkMode: agent.Config.SandboxNetworkMode(m.sandboxConfig), Timeout: time.Duration(agent.Config.SandboxTimeoutS(m.sandboxConfig)) * time.Second, + // Every sandbox kick gets a review spool (hivecommons/hive#14986); + // whether the agent may post through it is the relay's decision. + ReviewSpoolDir: crux.SandboxReviewSpoolsDir(m.workDir), } runCtx, cancel := context.WithCancel(context.Background()) agent.cancel = cancel diff --git a/src/crux/agent/sandbox_executor.go b/src/crux/agent/sandbox_executor.go index baac6017b0..e3ae105352 100644 --- a/src/crux/agent/sandbox_executor.go +++ b/src/crux/agent/sandbox_executor.go @@ -69,6 +69,10 @@ type SandboxKickSpec struct { EnvAllowlist []string NetworkMode string Timeout time.Duration + // ReviewSpoolDir is where the kick registers its review spool with the + // relay (crux.SandboxReviewSpoolsDir). Empty disables spooling: the + // sandbox then has no way to post a review. + ReviewSpoolDir string } type configSnapshot struct { @@ -156,6 +160,17 @@ func (e *SandboxExecutor) Run(ctx context.Context, spec SandboxKickSpec) (Sandbo res.Error = err.Error() return res, err } + // A spool the relay cannot see is a lost review, not a failed kick: the + // rest of the run is still worth doing, so log and go on without one. + spool, spoolErr := e.openReviewSpool(spec.ReviewSpoolDir, spec.Agent, workspace) + if spoolErr != nil && e.Logger != nil { + e.Logger.Warn("sandbox review spool unavailable; the sandbox cannot post reviews this kick", "agent", spec.Agent, "error", spoolErr) + } + defer func() { + if err := spool.end(e.now()); err != nil && e.Logger != nil { + e.Logger.Warn("sandbox review spool: could not mark the kick ended", "agent", spec.Agent, "error", err) + } + }() runCtx, cancel := context.WithTimeout(ctx, spec.Timeout) defer cancel() sres, runErr := e.launcher().Run(runCtx, sandbox.LaunchSpec{ @@ -165,9 +180,13 @@ func (e *SandboxExecutor) Run(ctx context.Context, spec SandboxKickSpec) (Sandbo WorkspaceMount: sandbox.DefaultWorkspaceMount, WorkDir: sandbox.DefaultWorkspaceMount, Command: command, + Env: spool.sandboxEnv(), EnvAllowlist: spec.EnvAllowlist, NetworkMode: spec.NetworkMode, }) + if ctx.Err() == nil && !spool.awaitDrain(ctx) && e.Logger != nil { + e.Logger.Warn("sandbox review spool: requests still pending after the drain wait", "agent", spec.Agent, "spool", spool.entry.Spool) + } // SECURITY (#3940, CWE-532): the agent CLI (notably one supplied via a // direct launch_cmd override, which never passes through agent-launch.sh's // stderr scrubber) can emit token-shaped values on stdout/stderr — e.g. diff --git a/src/crux/agent/sandbox_review_spool.go b/src/crux/agent/sandbox_review_spool.go new file mode 100644 index 0000000000..e71c928cce --- /dev/null +++ b/src/crux/agent/sandbox_review_spool.go @@ -0,0 +1,148 @@ +package agent + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path" + "path/filepath" + "strings" + "time" + + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/crux/sandbox" +) + +// reviewSpoolDrainTimeout bounds how long a finished kick waits for the +// review relay to deliver requests the sandbox left in its spool. The wait +// matters most for a run-stage kick, whose worktree — spool included — is +// removed as soon as Run returns. Vars so tests can shorten them. +var ( + reviewSpoolDrainTimeout = 60 * time.Second + reviewSpoolDrainPoll = time.Second +) + +// reviewSpoolHandle is one kick's registration with the review relay +// (hivecommons/hive#14986): the spool inside the workspace and the manifest, +// outside it, that names the agent the relay attributes the spool to. +type reviewSpoolHandle struct { + manifest string + entry crux.SandboxReviewSpool +} + +// openReviewSpool creates the workspace spool and registers it. A nil handle +// with a nil error means spooling is not configured for this kick. +func (e *SandboxExecutor) openReviewSpool(manifestDir, agentName, workspace string) (*reviewSpoolHandle, error) { + if strings.TrimSpace(manifestDir) == "" { + return nil, nil + } + spool := filepath.Join(workspace, crux.SandboxReviewSpoolRelPath) + if err := os.MkdirAll(spool, 0o770); err != nil { + return nil, fmt.Errorf("create review spool: %w", err) + } + // Owner-only: the relay trusts a manifest's agent name, so nothing but + // hive may write here. The relay refuses a group- or world-writable dir. + if err := os.MkdirAll(manifestDir, 0o700); err != nil { + return nil, fmt.Errorf("create review spool manifest dir: %w", err) + } + if err := os.Chmod(manifestDir, 0o700); err != nil { + return nil, fmt.Errorf("secure review spool manifest dir: %w", err) + } + h := &reviewSpoolHandle{ + manifest: filepath.Join(manifestDir, fmt.Sprintf("%s-%d.json", sanitizeBranchPart(agentName), time.Now().UnixNano())), + entry: crux.SandboxReviewSpool{Agent: agentName, Spool: spool, StartedAt: e.now().UTC()}, + } + if err := h.write(); err != nil { + return nil, err + } + return h, nil +} + +// sandboxEnv is what the sandbox needs to find the spool: hive-review writes +// to HIVE_REVIEW_REQUEST_DIR when it is set. +func (h *reviewSpoolHandle) sandboxEnv() map[string]string { + if h == nil { + return nil + } + return map[string]string{crux.SandboxReviewSpoolEnv: path.Join(sandbox.DefaultWorkspaceMount, crux.SandboxReviewSpoolRelPath)} +} + +func (h *reviewSpoolHandle) write() error { + b, err := json.Marshal(h.entry) + if err != nil { + return err + } + f, err := os.CreateTemp(filepath.Dir(h.manifest), filepath.Base(h.manifest)+".tmp") + if err != nil { + return err + } + tmp := f.Name() + if _, err := f.Write(b); err != nil { + _ = f.Close() + _ = os.Remove(tmp) + return err + } + if err := f.Close(); err != nil { + _ = os.Remove(tmp) + return err + } + if err := os.Rename(tmp, h.manifest); err != nil { + _ = os.Remove(tmp) + return err + } + return nil +} + +// end marks the kick finished. The relay keeps delivering what is still in +// the spool and drops the manifest once nothing is left. +func (h *reviewSpoolHandle) end(now time.Time) error { + if h == nil { + return nil + } + at := now.UTC() + h.entry.EndedAt = &at + return h.write() +} + +// awaitDrain waits, bounded by reviewSpoolDrainTimeout, until the relay has +// consumed every request in the spool. A request the sandbox wrote in its +// last seconds would otherwise race the run-stage worktree removal. +func (h *reviewSpoolHandle) awaitDrain(ctx context.Context) bool { + if h == nil || !reviewSpoolPending(h.entry.Spool) { + return true + } + deadline := time.NewTimer(reviewSpoolDrainTimeout) + defer deadline.Stop() + tick := time.NewTicker(reviewSpoolDrainPoll) + defer tick.Stop() + for { + select { + case <-ctx.Done(): + return false + case <-deadline.C: + return !reviewSpoolPending(h.entry.Spool) + case <-tick.C: + if !reviewSpoolPending(h.entry.Spool) { + return true + } + } + } +} + +// reviewSpoolPending reports whether the spool still holds a request the +// relay has not consumed. Result files and quarantined requests do not +// count; neither does an in-progress ".json.tmp". +func reviewSpoolPending(spool string) bool { + entries, err := os.ReadDir(spool) + if err != nil { + return false + } + for _, e := range entries { + name := e.Name() + if !e.IsDir() && strings.HasSuffix(name, ".json") && !strings.HasSuffix(name, ".result.json") { + return true + } + } + return false +} diff --git a/src/crux/agent/sandbox_review_spool_test.go b/src/crux/agent/sandbox_review_spool_test.go new file mode 100644 index 0000000000..b7a7af73f9 --- /dev/null +++ b/src/crux/agent/sandbox_review_spool_test.go @@ -0,0 +1,229 @@ +package agent + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/crux/config" + "github.com/hivecommons/hive/crux/sandbox" +) + +// spoolProbeLauncher records what the sandbox would see of its review spool +// (hivecommons/hive#14986): the env hive-review reads, and the manifests +// registered while the sandbox is running. +type spoolProbeLauncher struct { + manifestDir string + mu *sync.Mutex + env *map[string]string + manifests *[]crux.SandboxReviewSpool + // request, when set, is written into the spool as the sandbox's last act. + request string +} + +func (l spoolProbeLauncher) Run(_ context.Context, spec sandbox.LaunchSpec) (sandbox.Result, error) { + l.mu.Lock() + defer l.mu.Unlock() + *l.env = spec.Env + *l.manifests = readSpoolManifests(l.manifestDir) + if l.request != "" { + spool := filepath.Join(spec.Workspace, crux.SandboxReviewSpoolRelPath) + if err := os.WriteFile(filepath.Join(spool, "r.json"), []byte(l.request), 0o660); err != nil { + return sandbox.Result{}, err + } + } + return sandbox.Result{Stdout: "ok"}, nil +} + +func readSpoolManifests(dir string) []crux.SandboxReviewSpool { + entries, _ := os.ReadDir(dir) + var out []crux.SandboxReviewSpool + for _, e := range entries { + b, err := os.ReadFile(filepath.Join(dir, e.Name())) + if err != nil { + continue + } + var m crux.SandboxReviewSpool + if json.Unmarshal(b, &m) == nil { + out = append(out, m) + } + } + return out +} + +func spoolProbe(manifestDir string) (spoolProbeLauncher, *map[string]string, *[]crux.SandboxReviewSpool) { + env := map[string]string{} + var manifests []crux.SandboxReviewSpool + return spoolProbeLauncher{manifestDir: manifestDir, mu: &sync.Mutex{}, env: &env, manifests: &manifests}, &env, &manifests +} + +// A kick registers its spool before the sandbox starts, tells the sandbox +// where it is, and marks it ended after. +func TestSandboxExecutorRegistersReviewSpool(t *testing.T) { + root := t.TempDir() + manifestDir := filepath.Join(t.TempDir(), "sandbox-review-spools") + launcher, env, during := spoolProbe(manifestDir) + exec := &SandboxExecutor{Runner: &sandboxFakeRunner{}, Launcher: launcher} + res, err := exec.Run(context.Background(), SandboxKickSpec{ + Agent: "reviewer-spyre", AgentConfig: configSnapshot{Backend: "claude"}, Message: "review #7", + Org: "torch-spyre", Repo: "spyre-inference", WorkspaceDir: root, Image: "card-image", + ReviewSpoolDir: manifestDir, + }) + if err != nil { + t.Fatalf("Run: %v", err) + } + + if got, want := (*env)[crux.SandboxReviewSpoolEnv], "/workspace/.hive/review-requests"; got != want { + t.Errorf("sandbox %s = %q, want %q", crux.SandboxReviewSpoolEnv, got, want) + } + spool := filepath.Join(res.Workspace, crux.SandboxReviewSpoolRelPath) + if fi, err := os.Stat(spool); err != nil || !fi.IsDir() { + t.Fatalf("spool not created in the workspace: %v", err) + } + if len(*during) != 1 { + t.Fatalf("manifests registered while the sandbox ran = %d, want 1", len(*during)) + } + if m := (*during)[0]; m.Agent != "reviewer-spyre" || m.Spool != spool || m.EndedAt != nil { + t.Errorf("manifest during the run = %+v, want agent reviewer-spyre, spool %s, not ended", m, spool) + } + if fi, err := os.Stat(manifestDir); err != nil || fi.Mode().Perm() != 0o700 { + t.Errorf("manifest dir must be owner-only: %v %v", fi.Mode(), err) + } + after := readSpoolManifests(manifestDir) + if len(after) != 1 || after[0].EndedAt == nil { + t.Errorf("manifest after the run = %+v, want one marked ended", after) + } +} + +// POSITIVE CONTROL: with no manifest dir the sandbox gets no spool and no +// env, so the assertions above test the registration and not a default. +func TestSandboxExecutorWithoutReviewSpoolDirRegistersNothing(t *testing.T) { + root := t.TempDir() + launcher, env, _ := spoolProbe(filepath.Join(t.TempDir(), "unused")) + exec := &SandboxExecutor{Runner: &sandboxFakeRunner{}, Launcher: launcher} + res, err := exec.Run(context.Background(), SandboxKickSpec{ + Agent: "scanner", AgentConfig: configSnapshot{Backend: "claude"}, Message: "fix", + Org: "o", Repo: "r", WorkspaceDir: root, Image: "img", + }) + if err != nil { + t.Fatalf("Run: %v", err) + } + if _, ok := (*env)[crux.SandboxReviewSpoolEnv]; ok { + t.Errorf("sandbox got %s without a manifest dir", crux.SandboxReviewSpoolEnv) + } + if _, err := os.Stat(filepath.Join(res.Workspace, crux.SandboxReviewSpoolRelPath)); !os.IsNotExist(err) { + t.Errorf("spool created without a manifest dir: %v", err) + } +} + +// A request the sandbox writes as it exits is waited for: Run does not +// return (and a run-stage worktree is not removed) until the relay has taken +// it, bounded by the drain timeout. +func TestSandboxExecutorWaitsForTheRelayToDrainTheSpool(t *testing.T) { + oldTimeout, oldPoll := reviewSpoolDrainTimeout, reviewSpoolDrainPoll + reviewSpoolDrainTimeout, reviewSpoolDrainPoll = 10*time.Second, 5*time.Millisecond + t.Cleanup(func() { reviewSpoolDrainTimeout, reviewSpoolDrainPoll = oldTimeout, oldPoll }) + + root := t.TempDir() + manifestDir := filepath.Join(t.TempDir(), "sandbox-review-spools") + launcher, _, _ := spoolProbe(manifestDir) + launcher.request = `{"repo":"o/r","number":7,"event":"comment","body":"x"}` + var consumed atomic.Bool + go func() { + // The relay: take the request once it appears. The flag is set + // BEFORE the remove, so a Run that returned only after the spool + // drained always sees it. + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + matches, _ := filepath.Glob(filepath.Join(root, "*", crux.SandboxReviewSpoolRelPath, "r.json")) + if len(matches) == 1 { + time.Sleep(50 * time.Millisecond) + consumed.Store(true) + _ = os.Remove(matches[0]) + return + } + time.Sleep(time.Millisecond) + } + }() + exec := &SandboxExecutor{Runner: &sandboxFakeRunner{}, Launcher: launcher} + res, err := exec.Run(context.Background(), SandboxKickSpec{ + Agent: "reviewer-spyre", AgentConfig: configSnapshot{Backend: "claude"}, Message: "review", + Org: "o", Repo: "r", WorkspaceDir: root, Image: "img", ReviewSpoolDir: manifestDir, + }) + if err != nil { + t.Fatalf("Run: %v", err) + } + if !consumed.Load() { + t.Fatal("Run returned before the relay consumed the spooled request") + } + if reviewSpoolPending(filepath.Join(res.Workspace, crux.SandboxReviewSpoolRelPath)) { + t.Error("spool still pending after Run") + } +} + +func TestReviewSpoolAwaitDrainGivesUpAtTheTimeout(t *testing.T) { + oldTimeout, oldPoll := reviewSpoolDrainTimeout, reviewSpoolDrainPoll + reviewSpoolDrainTimeout, reviewSpoolDrainPoll = 30*time.Millisecond, 5*time.Millisecond + t.Cleanup(func() { reviewSpoolDrainTimeout, reviewSpoolDrainPoll = oldTimeout, oldPoll }) + + spool := t.TempDir() + if err := os.WriteFile(filepath.Join(spool, "r.json"), []byte("{}"), 0o600); err != nil { + t.Fatal(err) + } + // Results and quarantined requests are not pending. + for _, name := range []string{"done.result.json", "x.json.bad", "y.json.tmp"} { + if err := os.WriteFile(filepath.Join(spool, name), []byte("{}"), 0o600); err != nil { + t.Fatal(err) + } + } + h := &reviewSpoolHandle{entry: crux.SandboxReviewSpool{Spool: spool}} + if h.awaitDrain(context.Background()) { + t.Fatal("awaitDrain reported drained with a request still in the spool") + } + if err := os.Remove(filepath.Join(spool, "r.json")); err != nil { + t.Fatal(err) + } + if !h.awaitDrain(context.Background()) { + t.Fatal("awaitDrain reported pending with only results and quarantined files left") + } +} + +// The manager gives every sandbox kick — Job runtime included — a spool +// registered under its own work dir, where the relay looks. +func TestJobRuntimeKickRegistersReviewSpoolUnderWorkDir(t *testing.T) { + workDir := t.TempDir() + t.Setenv("HIVE_WORK_DIR", workDir) + manifestDir := crux.SandboxReviewSpoolsDir(workDir) + launcher, env, during := spoolProbe(manifestDir) + + yes := true + cfg := map[string]config.AgentConfig{"reviewer": {Backend: "claude", Sandbox: &config.AgentSandboxOverride{ + Enabled: &yes, Runtime: config.SandboxRuntimeJob}}} + m := NewManager(cfg, quietTestLogger(), ProjectContext{Org: "torch-spyre", Repos: []string{"spyre-inference"}}) + m.SetSandboxConfig(config.AgentSandboxConfig{Enabled: true, Image: "card-image", WorkspaceDir: t.TempDir(), + Job: &config.SandboxJobConfig{WorkspaceClaim: "hive-data"}}) + m.setSandboxJobLauncherFactoryForTest(func(config.SandboxJobConfig) sandbox.Launcher { return launcher }) + m.setSandboxRunnerForTest(&sandboxFakeRunner{}) + if err := m.Start(context.Background(), "reviewer"); err != nil { + t.Fatalf("Start: %v", err) + } + if err := m.SendKick("reviewer", "review #7 on the card"); err != nil { + t.Fatalf("SendKick: %v", err) + } + waitIdle(t, m, "reviewer") + + launcher.mu.Lock() + defer launcher.mu.Unlock() + if (*env)[crux.SandboxReviewSpoolEnv] == "" { + t.Errorf("job sandbox got no %s", crux.SandboxReviewSpoolEnv) + } + if len(*during) != 1 || (*during)[0].Agent != "reviewer" { + t.Errorf("manifests under %s during the job = %+v, want one for reviewer", manifestDir, *during) + } +} diff --git a/src/crux/sandbox_review_spool.go b/src/crux/sandbox_review_spool.go new file mode 100644 index 0000000000..743f5844ed --- /dev/null +++ b/src/crux/sandbox_review_spool.go @@ -0,0 +1,42 @@ +package crux + +import ( + "path/filepath" + "time" +) + +// A sandboxed kick has no path to the pod-local review relay +// (/var/run/hive-metrics/review-requests): the container or Job sees only its +// own workspace. So hive gives every sandbox kick a review SPOOL inside that +// workspace, and registers it with a MANIFEST the relay reads +// (hivecommons/hive#14986). The manifest lives under workDir, outside every +// workspace, so the sandbox can neither see nor forge it — which is why the +// relay attributes a spooled request to the manifest's agent and never to +// the request file's own "agent" field. +const ( + // SandboxReviewSpoolRelPath is the spool, relative to the kick + // workspace. hive-review writes there when HIVE_REVIEW_REQUEST_DIR + // names it. + SandboxReviewSpoolRelPath = ".hive/review-requests" + // SandboxReviewSpoolEnv is the environment variable hive sets inside the + // sandbox to the spool's in-sandbox path. + SandboxReviewSpoolEnv = "HIVE_REVIEW_REQUEST_DIR" + sandboxReviewSpoolsDir = "sandbox-review-spools" +) + +// SandboxReviewSpoolsDir is where the sandbox executor writes one manifest +// per kick and the review relay reads them. Host and runtime share it +// through /data, exactly as they share SandboxRunsDir. +func SandboxReviewSpoolsDir(workDir string) string { + return filepath.Join(workDir, sandboxReviewSpoolsDir) +} + +// SandboxReviewSpool is one kick's manifest. Spool is the host-side path of +// the workspace spool. EndedAt is set once the sandbox has exited; the relay +// drops the manifest after that when nothing is left to deliver. +type SandboxReviewSpool struct { + Agent string `json:"agent"` + Spool string `json:"spool"` + StartedAt time.Time `json:"started_at"` + EndedAt *time.Time `json:"ended_at,omitempty"` +} diff --git a/src/docs/github-write-surface.md b/src/docs/github-write-surface.md index 858eceaf28..e77482e9e4 100644 --- a/src/docs/github-write-surface.md +++ b/src/docs/github-write-surface.md @@ -44,7 +44,7 @@ Each relay authorizes a request in this order. It stops at the first refusal. | `close_issue` | `issue_request_watcher.go` kind `close` | `hive-open-issue close` | `agent_issue_closed` (`pr_closed` when the number is a PR) | yes | yes / issue or PR number | | `label` | `issue_request_watcher.go` kind `label` (`.../issue-requests`) | `hive-open-issue label` (also reached by `gh issue edit --add-label`/`--remove-label` and `gh pr edit --add-label`/`--remove-label`, which the `gh` wrapper translates into this relay) | `agent_label_applied` | yes | yes / issue or PR number | | `request_review` | `issue_request_watcher.go` kind `request_review` (`.../issue-requests`) | `hive-open-issue request-review` (also reached by `gh pr edit --add-reviewer`, which the `gh` wrapper translates into this relay) | `agent_review_requested` | yes | yes / PR number | -| `review` | `review_request_watcher.go` events `approve`, `request_changes`, `comment`, `record_verdict`, and thread replies (`.../review-requests`) | `hive-review` | `agent_pr_reviewed` | yes | yes / PR number | +| `review` | `review_request_watcher.go` events `approve`, `request_changes`, `comment`, `record_verdict`, and thread replies (`.../review-requests`, plus sandbox review spools via `review_request_sandbox_spool.go`, #14986) | `hive-review` | `agent_pr_reviewed` | yes | yes / PR number | | `resolve_thread` | `review_request_watcher.go` event `resolve_thread` | `hive-review` | `agent_pr_reviewed` (`state=thread_resolved`) | yes | yes / PR number | | `merge_pr` | `merge_request_watcher.go` (`.../merge-requests`) | `hive-merge` | `pr_merged` (`path=relay`, via `MergePR`) | yes | yes / PR number | | `push_branch` | `push_branch_request_watcher.go` (`.../push-requests`) | `hive-push-branch` | `agent_branch_pushed` (branch and commit in the detail) | yes | yes / none (a branch push has no number) | diff --git a/src/docs/sandbox-isolation.md b/src/docs/sandbox-isolation.md index 5e8083d26a..11340455f2 100644 --- a/src/docs/sandbox-isolation.md +++ b/src/docs/sandbox-isolation.md @@ -173,6 +173,36 @@ agents: Hive warns at boot and on reload (`config.AgentSandboxGateWarnings`) when a job-runtime agent resolves no `workspace_claim` or names a runtime hive does not have, and refuses the kick with the same message rather than failing it after the agent is marked running. +### Posting reviews from a sandbox + +A reviewer can run in a sandbox too, so a review of an accelerator change can actually run it on the card node ([#14986](https://github.com/hivecommons/hive/issues/14986)). `sandbox.runtime: job` is a legal setting for a `reviewer`-base agent; the review runs on the node the Job lands on. + +```yaml +agents: + reviewer: + sandbox: + enabled: true + runtime: job + image: registry.example/project-dev:latest # must include hive-review + job: + node_selector: { accelerator: spyre } + resources: + limits: { "vendor.com/device": "1" } +``` + +A sandbox cannot reach the pod-local review relay (`/var/run/hive-metrics/review-requests`), and it never holds a GitHub credential. Instead, every sandbox kick (Podman or Job) gets a **review spool**: + +1. Before launch, hive creates `.hive/review-requests` in the kick workspace and sets `HIVE_REVIEW_REQUEST_DIR=/workspace/.hive/review-requests` inside the sandbox. `hive-review` writes its request there instead of the pod-local dir, in the same JSON schema. +2. Hive registers the spool with a manifest under `/sandbox-review-spools/` (`work_dir` is `HIVE_WORK_DIR`, default `/data/agents`). The manifest names the kick's agent. It lives outside every workspace, in a directory only hive's uid may write, so the sandbox can neither see nor forge it. +3. The review-request watcher moves each spooled request into a hive-owned staging directory beside the manifest and runs it through the same handler as the pod-local drop box. `review.max_reviews_per_head`, repo pause, the lane write allowlist, the sentinel approval block, the contributor COMMENT-only guard, the canary scan and the `agent_pr_reviewed` audit all apply unchanged. The request is **attributed to the manifest's agent**, whatever its own `agent` field says; the per-agent authorization (ACMM `CanPush` or the `converse` capability) is checked for that agent. +4. The result is copied back to `.result.json` in the spool, where the sandboxed agent reads it exactly as it would in-pod. A bad `--verdict-file` is refused with the validator's message, so the agent can fix the verdict and resubmit within the same kick. + +While any spool is registered, the watcher scans on its short poll interval: a Job writes its spool over a shared volume, often from another node, where no file-change notification reaches the hive pod. After the sandbox exits, the kick waits up to a minute for the relay to take any request still in the spool, because a run-stage worktree is removed when the kick returns. The manifest is dropped once the kick has ended and nothing is left to deliver, or after 24 hours. + +The relay touches the spool only through an `os.Root` opened at the workspace. A symlink the sandbox plants can't redirect a read, rename or write outside its own workspace, and a request file that is a symlink, FIFO or oversized is set aside as `.rejected`. The relay ignores a manifest directory or manifest that is not owned by its own uid or is group- or world-writable. In the sidecar runtime (`CRUX_MODE=sidecar`), the runtime and hive must therefore share both `/data` and a uid. + +Out of scope: the quality and deps lanes are unchanged, and nothing inside a Job writes to GitHub directly. + ## Health and metrics for sandboxed agents A sandboxed agent (Podman or Job runtime) has no tmux session, no in-pod process and no `pluk` log between kicks: the manager parks it in the `idle` state and each kick runs to completion in a container or Job. The dashboard therefore must not read "not running" as "session down" for these agents, and the token collector cannot find their CLI session files because the CLI's home directory lives in the sandbox, not in the hive pod. diff --git a/src/pkg/github/read_untrusted_other.go b/src/pkg/github/read_untrusted_other.go index 74970b0f23..1ba3b2c129 100644 --- a/src/pkg/github/read_untrusted_other.go +++ b/src/pkg/github/read_untrusted_other.go @@ -45,3 +45,7 @@ func readUntrustedFile(path string, maxBytes int64) ([]byte, fs.FileInfo, error) } return data, fi, nil } + +// untrustedRootOpenFlags: see read_untrusted_unix.go. Off unix O_NONBLOCK is +// not portable; the os.Root confinement and the type checks remain. +const untrustedRootOpenFlags = 0 diff --git a/src/pkg/github/read_untrusted_unix.go b/src/pkg/github/read_untrusted_unix.go index 29657fd033..dfe8ff6f10 100644 --- a/src/pkg/github/read_untrusted_unix.go +++ b/src/pkg/github/read_untrusted_unix.go @@ -46,3 +46,9 @@ func readUntrustedFile(path string, maxBytes int64) ([]byte, fs.FileInfo, error) } return data, fi, nil } + +// untrustedRootOpenFlags are added to every open of a file in a +// sandbox-owned tree through an os.Root. The Root keeps the open, symlinks +// included, inside the tree; O_NONBLOCK keeps a FIFO swapped in after the +// caller's type check from blocking the relay. +const untrustedRootOpenFlags = syscall.O_NONBLOCK diff --git a/src/pkg/github/review_request_sandbox_spool.go b/src/pkg/github/review_request_sandbox_spool.go new file mode 100644 index 0000000000..e70876a6c8 --- /dev/null +++ b/src/pkg/github/review_request_sandbox_spool.go @@ -0,0 +1,406 @@ +package github + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "io/fs" + "log/slog" + "os" + "path/filepath" + "strings" + "time" + + "github.com/hivecommons/hive/crux" +) + +// Sandbox review spools (hivecommons/hive#14986). A reviewer whose kicks run +// in a sandbox — a Podman container or, on a hardware spoke, a Kubernetes Job +// on the accelerator node — cannot reach ReviewRequestDir. The sandbox +// executor gives every kick a spool inside its workspace and registers it +// with a manifest under crux.SandboxReviewSpoolsDir; this file is the relay's +// half. Each pass it: +// +// 1. moves every request out of the spool into a hive-owned staging dir +// next to the manifest, +// 2. runs the staging dir through the SAME handleOneReviewRequest the +// pod-local drop box uses, so per-head caps, repo pause, the lane write +// allowlist, the sentinel block and the audit trail apply unchanged, and +// 3. copies each result file back into the spool, where the sandboxed agent +// reads it exactly as it would in-pod. +// +// The spool is sandbox-writable, so the relay touches it only through an +// os.Root opened at the workspace: a symlink the sandbox plants can never +// steer a read, rename or write outside its own workspace. Everything that +// renames, quarantines or retries happens in staging, which only hive writes. +// +// Attribution differs from the pod-local path. There a request names its +// agent and the file's owning UID proves it. Inside a sandbox the UID proves +// nothing (the Job runs as whatever its image says) and the "agent" field is +// the sandbox's to write, so a spooled request is attributed to the +// manifest's agent and only that. The manifest is trusted because the +// sandbox cannot reach it: it lives outside every workspace, in an +// owner-only directory, and the relay ignores one that is not owned by its +// own uid or that anyone else could write. + +// sandboxReviewSpoolsDirForTest lets tests point the relay at a temp dir. +var sandboxReviewSpoolsDirForTest string + +func sandboxReviewSpoolsDir() string { + if sandboxReviewSpoolsDirForTest != "" { + return sandboxReviewSpoolsDirForTest + } + return crux.SandboxReviewSpoolsDir(crux.WorkDir()) +} + +// sandboxReviewSpoolMaxAge drops a manifest whatever its state. A kick never +// outlives it, so an older manifest belongs to a run whose executor died +// before marking it ended. +const sandboxReviewSpoolMaxAge = 24 * time.Hour + +// sandboxReviewManifestMaxBytes caps a manifest read; a real one is a few +// hundred bytes. +const sandboxReviewManifestMaxBytes = 16 << 10 + +// reviewRequestSource says who a request file is attributed to. The zero +// value is the pod-local drop box: the request's own agent field, checked +// against the file's owning UID. +type reviewRequestSource struct { + // agent, when set, is the manifest-attributed owner of a sandbox spool. + // It replaces the request's agent field, and the UID check is skipped. + agent string +} + +type sandboxReviewSpool struct { + path string + crux.SandboxReviewSpool +} + +// staging is the hive-owned dir a spool's requests are processed in. +func (s sandboxReviewSpool) staging() string { + return strings.TrimSuffix(s.path, ".json") + ".d" +} + +// workspace is the kick workspace the spool lives in, and rel the spool's +// path inside it. +func (s sandboxReviewSpool) workspace() (dir, rel string) { + rel = filepath.FromSlash(crux.SandboxReviewSpoolRelPath) + return strings.TrimSuffix(s.Spool, string(filepath.Separator)+rel), rel +} + +// processSandboxReviewSpools delivers every registered spool, then drops the +// manifests of kicks that have ended and left nothing to deliver. +func (c *Client) processSandboxReviewSpools(ctx context.Context, nowFn func() time.Time) { + for _, s := range c.loadSandboxReviewSpools(nowFn()) { + if ctx.Err() != nil { + return + } + staging := s.staging() + if err := os.MkdirAll(staging, 0o700); err != nil { + c.logger.Warn("review-request watcher: cannot create sandbox spool staging dir", + slog.String("dir", staging), slog.String("error", err.Error())) + continue + } + // A missing workspace (a run-stage worktree is removed when its kick + // returns) leaves nothing to ingest or reply to, but whatever was + // already staged is still delivered. + ws, rel := s.workspace() + root, err := os.OpenRoot(ws) + if err == nil { + c.ingestSandboxSpool(root, rel, staging, nowFn()) + } + c.processReviewRequestDir(ctx, staging, reviewRequestSource{agent: s.Agent}, nowFn) + spoolPending := false + if root != nil { + mirrorSandboxResults(root, rel, staging) + spoolPending = rootSpoolPending(root, rel) + _ = root.Close() + } + if s.EndedAt != nil && !spoolPending && !dirHasReviewRequest(staging) { + _ = os.RemoveAll(staging) + _ = os.Remove(s.path) + } + } +} + +// hasSandboxReviewSpools reports whether any manifest is registered in dir. +// While one is, the watcher keeps its short poll cadence: a Job writes its +// spool over a shared volume from another node, where a change notification +// never fires. +func hasSandboxReviewSpools(dir string) bool { + entries, err := os.ReadDir(dir) + if err != nil { + return false + } + for _, e := range entries { + if !e.IsDir() && strings.HasSuffix(e.Name(), ".json") { + return true + } + } + return false +} + +// loadSandboxReviewSpools returns the registered spools, pruning manifests +// that are stale or malformed. +func (c *Client) loadSandboxReviewSpools(now time.Time) []sandboxReviewSpool { + dir := sandboxReviewSpoolsDir() + fi, err := os.Lstat(dir) + if err != nil { + return nil + } + if reason := untrustedOwnerReason(fi); reason != "" { + c.logger.Warn("review-request watcher: ignoring sandbox review spools; manifest dir is not hive-only", + slog.String("dir", dir), slog.String("reason", reason)) + return nil + } + entries, err := os.ReadDir(dir) + if err != nil { + return nil + } + var out []sandboxReviewSpool + for _, e := range entries { + name := e.Name() + if e.IsDir() || !strings.HasSuffix(name, ".json") { + continue + } + path := filepath.Join(dir, name) + data, mfi, err := readUntrustedFile(path, sandboxReviewManifestMaxBytes) + if err != nil { + if errors.Is(err, errDropBoxFileRejected) { + _ = os.Remove(path) + } + continue + } + if reason := untrustedOwnerReason(mfi); reason != "" { + c.logger.Warn("review-request watcher: ignoring sandbox review manifest", + slog.String("path", path), slog.String("reason", reason)) + continue + } + s := sandboxReviewSpool{path: path} + if err := json.Unmarshal(data, &s.SandboxReviewSpool); err != nil { + // Manifests are written by rename, so a parse failure past the + // in-flight grace is corruption, not a torn write. + if quarantinable(path, now) { + _ = os.RemoveAll(s.staging()) + _ = os.Remove(path) + } + continue + } + if reason := sandboxSpoolRefusal(s.SandboxReviewSpool, now); reason != "" { + c.logger.Warn("review-request watcher: dropping sandbox review manifest", + slog.String("path", path), slog.String("agent", s.Agent), slog.String("reason", reason)) + _ = os.RemoveAll(s.staging()) + _ = os.Remove(path) + continue + } + out = append(out, s) + } + return out +} + +// untrustedOwnerReason is why fi cannot be trusted to carry attribution, or +// "" when it is a regular file or directory owned by this process's uid that +// no one else may write. +func untrustedOwnerReason(fi fs.FileInfo) string { + switch { + case fi.Mode()&fs.ModeSymlink != 0: + return "symbolic link" + case !fi.IsDir() && !fi.Mode().IsRegular(): + return "not a regular file or directory" + case fileOwnerUID(fi) != os.Geteuid(): + return "not owned by the relay's uid" + case fi.Mode().Perm()&0o022 != 0: + return "group- or world-writable" + } + return "" +} + +// sandboxSpoolRefusal is why a manifest must be dropped, or "" to keep it. +func sandboxSpoolRefusal(m crux.SandboxReviewSpool, now time.Time) string { + suffix := string(filepath.Separator) + filepath.FromSlash(crux.SandboxReviewSpoolRelPath) + switch { + case strings.TrimSpace(m.Agent) == "": + return "manifest names no agent" + case !filepath.IsAbs(m.Spool) || filepath.Clean(m.Spool) != m.Spool || !strings.HasSuffix(m.Spool, suffix): + return "spool is not /" + crux.SandboxReviewSpoolRelPath + case !m.StartedAt.IsZero() && now.Sub(m.StartedAt) > sandboxReviewSpoolMaxAge: + return "manifest is older than " + sandboxReviewSpoolMaxAge.String() + } + return "" +} + +// ingestSandboxSpool moves each request out of the spool into staging. A +// name already staged (a request still retrying) waits in the spool for a +// later pass rather than replacing it. A file that is not yet valid JSON is +// left alone for inFlightGrace, in case the sandbox is still writing it; past +// that it is staged as-is and fails validation there, so the agent gets a +// result file that says why. +func (c *Client) ingestSandboxSpool(root *os.Root, rel, staging string, now time.Time) { + d, err := root.Open(rel) + if err != nil { + return + } + entries, err := d.ReadDir(-1) + _ = d.Close() + if err != nil { + return + } + for _, e := range entries { + if !isReviewRequestFile(e) { + continue + } + name := e.Name() + dst := filepath.Join(staging, name) + if _, err := os.Lstat(dst); err == nil { + continue + } + src := filepath.Join(rel, name) + data, fi, err := readRootFile(root, src, requestMaxBytes) + if err != nil { + if errors.Is(err, errDropBoxFileRejected) { + _ = root.Rename(src, src+".rejected") + c.logger.Warn("review-request watcher: REJECTED sandbox spool file (unsafe file)", + slog.String("path", src), slog.String("reason", err.Error())) + } + continue + } + if !json.Valid(data) && now.Sub(fi.ModTime()) < inFlightGrace { + continue + } + if err := writeRequestFile(dst, data); err != nil { + continue + } + _ = root.Remove(src) + } +} + +// mirrorSandboxResults copies each staged result into the spool. A result +// whose request has left staging is final, so its staged copy is dropped +// once mirrored; one still retrying is mirrored again on the next pass. +func mirrorSandboxResults(root *os.Root, rel, staging string) { + entries, err := os.ReadDir(staging) + if err != nil { + return + } + for _, e := range entries { + name := e.Name() + if e.IsDir() || !strings.HasSuffix(name, ".result.json") { + continue + } + staged := filepath.Join(staging, name) + data, err := os.ReadFile(staged) + if err != nil { + continue + } + if err := writeRootFile(root, filepath.Join(rel, name), data); err != nil { + continue + } + request := strings.TrimSuffix(staged, ".result.json") + ".json" + if _, err := os.Lstat(request); errors.Is(err, fs.ErrNotExist) { + _ = os.Remove(staged) + } + } +} + +// rootSpoolPending reports whether the spool still holds a request. +func rootSpoolPending(root *os.Root, rel string) bool { + d, err := root.Open(rel) + if err != nil { + return false + } + defer d.Close() + entries, err := d.ReadDir(-1) + if err != nil { + return false + } + for _, e := range entries { + if isReviewRequestFile(e) { + return true + } + } + return false +} + +// dirHasReviewRequest reports whether dir holds a request the relay has not +// consumed (result files and quarantined requests do not count). +func dirHasReviewRequest(dir string) bool { + entries, err := os.ReadDir(dir) + if err != nil { + return false + } + for _, e := range entries { + if isReviewRequestFile(e) { + return true + } + } + return false +} + +// isReviewRequestFile is the drop-box filter: a ".json" request, not +// its ".result.json" reply and not a directory. +func isReviewRequestFile(e fs.DirEntry) bool { + name := e.Name() + return !e.IsDir() && strings.HasSuffix(name, ".json") && !strings.HasSuffix(name, ".result.json") +} + +// readRootFile is readUntrustedFile for a file inside a sandbox-owned tree. +// The Lstat names a symlink or FIFO as such so it can be set aside; the +// open flags close the window in which one could be swapped in after it. +func readRootFile(root *os.Root, name string, maxBytes int64) ([]byte, fs.FileInfo, error) { + lfi, err := root.Lstat(name) + if err != nil { + return nil, nil, err + } + if !lfi.Mode().IsRegular() { + return nil, lfi, dropBoxRejectedError(fmt.Sprintf("not a regular file (%s)", lfi.Mode().Type())) + } + f, err := root.OpenFile(name, os.O_RDONLY|untrustedRootOpenFlags, 0) + if err != nil { + return nil, nil, err + } + defer f.Close() + fi, err := f.Stat() + if err != nil { + return nil, nil, err + } + if !fi.Mode().IsRegular() { + return nil, fi, dropBoxRejectedError(fmt.Sprintf("not a regular file (%s)", fi.Mode().Type())) + } + if fi.Size() > maxBytes { + return nil, fi, dropBoxRejectedError(fmt.Sprintf("%d bytes exceeds cap of %d", fi.Size(), maxBytes)) + } + data, err := io.ReadAll(io.LimitReader(f, maxBytes+1)) + if err != nil { + return nil, fi, err + } + if int64(len(data)) > maxBytes { + return nil, fi, dropBoxRejectedError(fmt.Sprintf("grew past cap of %d bytes during read", maxBytes)) + } + return data, fi, nil +} + +// writeRootFile publishes name inside root atomically, the os.Root analogue +// of writeRequestFile: the agent never sees a half-written result. +func writeRootFile(root *os.Root, name string, data []byte) error { + tmp := fmt.Sprintf("%s.%d.tmp", name, time.Now().UnixNano()) + f, err := root.OpenFile(tmp, os.O_WRONLY|os.O_CREATE|os.O_EXCL|untrustedRootOpenFlags, 0o660) + if err != nil { + return err + } + if _, err := f.Write(data); err != nil { + _ = f.Close() + _ = root.Remove(tmp) + return err + } + if err := f.Close(); err != nil { + _ = root.Remove(tmp) + return err + } + if err := root.Rename(tmp, name); err != nil { + _ = root.Remove(tmp) + return err + } + return nil +} diff --git a/src/pkg/github/review_request_sandbox_spool_test.go b/src/pkg/github/review_request_sandbox_spool_test.go new file mode 100644 index 0000000000..82fbb606ae --- /dev/null +++ b/src/pkg/github/review_request_sandbox_spool_test.go @@ -0,0 +1,305 @@ +package github + +import ( + "context" + "encoding/json" + "errors" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/hivecommons/hive/crux" +) + +// withSandboxSpools points the relay at an owner-only manifest dir and +// returns it. +func withSandboxSpools(t *testing.T) string { + t.Helper() + dir := filepath.Join(t.TempDir(), "sandbox-review-spools") + if err := os.Mkdir(dir, 0o700); err != nil { + t.Fatal(err) + } + old := sandboxReviewSpoolsDirForTest + sandboxReviewSpoolsDirForTest = dir + t.Cleanup(func() { sandboxReviewSpoolsDirForTest = old }) + return dir +} + +// newSandboxSpool creates a kick workspace with its spool and registers it +// for agent, returning the spool and the manifest path. +func newSandboxSpool(t *testing.T, manifests, agent string, ended bool) (spool, manifest string) { + t.Helper() + workspace := t.TempDir() + spool = filepath.Join(workspace, filepath.FromSlash(crux.SandboxReviewSpoolRelPath)) + if err := os.MkdirAll(spool, 0o770); err != nil { + t.Fatal(err) + } + m := crux.SandboxReviewSpool{Agent: agent, Spool: spool, StartedAt: time.Now().UTC()} + if ended { + at := time.Now().UTC() + m.EndedAt = &at + } + manifest = writeSandboxManifest(t, manifests, agent, m) + return spool, manifest +} + +func writeSandboxManifest(t *testing.T, manifests, agent string, m crux.SandboxReviewSpool) string { + t.Helper() + b, err := json.Marshal(m) + if err != nil { + t.Fatal(err) + } + path := filepath.Join(manifests, agent+"-1.json") + if err := os.WriteFile(path, b, 0o600); err != nil { + t.Fatal(err) + } + return path +} + +func readReviewResult(t *testing.T, reqPath string) ReviewResponse { + t.Helper() + b, err := os.ReadFile(strings.TrimSuffix(reqPath, ".json") + ".result.json") + if err != nil { + t.Fatalf("no result file mirrored into the spool: %v", err) + } + var resp ReviewResponse + if err := json.Unmarshal(b, &resp); err != nil { + t.Fatal(err) + } + return resp +} + +// A sandboxed reviewer's spooled comment is posted, attributed to the agent +// the MANIFEST names — not the one the file claims — and its result lands +// back in the spool where the sandbox can read it. +func TestSandboxSpool_PostsReviewAttributedToManifestAgent(t *testing.T) { + reviewed := 0 + var apiEvent string + srv := newReviewMockServer(t, &reviewed, &apiEvent) + defer srv.Close() + c := reviewTestClient(t, srv.URL) + withReviewDir(t) + manifests := withSandboxSpools(t) + spool, _ := newSandboxSpool(t, manifests, "reviewer-spyre", false) + + var authzAgent string + authzUID := -1 + c.reviewAuthz = func(agent string, uid int) error { authzAgent, authzUID = agent, uid; return nil } + var auditAgent string + c.SetAttributionAudit(func(action, detail, agent string) { + if action == AuditActionPRReviewed { + auditAgent = agent + } + }) + + reqPath, err := WriteReviewRequest(spool, ReviewRequest{Repo: "o/r", Number: 7, Event: "comment", Body: "card run passed", Agent: "someone-else"}) + if err != nil { + t.Fatal(err) + } + c.ProcessReviewRequestsOnce(context.Background()) + + if reviewed != 1 || apiEvent != "COMMENT" { + t.Fatalf("reviews posted = %d (event %q), want 1 COMMENT", reviewed, apiEvent) + } + if authzAgent != "reviewer-spyre" { + t.Errorf("authorized as %q, want the manifest's agent reviewer-spyre", authzAgent) + } + if authzUID != 0 { + t.Errorf("authorizer got uid %d, want 0: a staged file's owner proves nothing", authzUID) + } + if auditAgent != "reviewer-spyre" { + t.Errorf("audited as %q, want reviewer-spyre", auditAgent) + } + if _, err := os.Stat(reqPath); !errors.Is(err, os.ErrNotExist) { + t.Errorf("request still in the spool after delivery: %v", err) + } + if resp := readReviewResult(t, reqPath); !resp.OK || resp.State != "commented" { + t.Errorf("mirrored result = %+v, want ok commented", resp) + } +} + +// Relay policy is the same code path: a denial reaches the sandbox as a +// result file and nothing is posted. +func TestSandboxSpool_DenialIsMirroredAndNothingPosted(t *testing.T) { + reviewed := 0 + srv := newReviewMockServer(t, &reviewed, nil) + defer srv.Close() + c := reviewTestClient(t, srv.URL) + c.reviewAuthz = func(agent string, uid int) error { return errors.New("not push-capable") } + withReviewDir(t) + manifests := withSandboxSpools(t) + spool, _ := newSandboxSpool(t, manifests, "reviewer-spyre", false) + + reqPath, err := WriteReviewRequest(spool, ReviewRequest{Repo: "o/r", Number: 7, Event: "comment", Body: "x", Agent: "reviewer-spyre"}) + if err != nil { + t.Fatal(err) + } + c.ProcessReviewRequestsOnce(context.Background()) + + if reviewed != 0 { + t.Fatalf("denied request posted %d reviews", reviewed) + } + resp := readReviewResult(t, reqPath) + if resp.OK || !strings.Contains(resp.Error, "not push-capable") { + t.Errorf("mirrored result = %+v, want the denial", resp) + } +} + +// The manifest is the whole of the trust: a manifest dir others can write is +// ignored, so nothing in it is delivered. +func TestSandboxSpool_IgnoresWritableManifestDir(t *testing.T) { + reviewed := 0 + srv := newReviewMockServer(t, &reviewed, nil) + defer srv.Close() + c := reviewTestClient(t, srv.URL) + withReviewDir(t) + manifests := withSandboxSpools(t) + spool, _ := newSandboxSpool(t, manifests, "reviewer-spyre", false) + if err := os.Chmod(manifests, 0o777); err != nil { + t.Fatal(err) + } + reqPath, err := WriteReviewRequest(spool, ReviewRequest{Repo: "o/r", Number: 7, Event: "comment", Body: "x"}) + if err != nil { + t.Fatal(err) + } + c.ProcessReviewRequestsOnce(context.Background()) + + if reviewed != 0 { + t.Fatalf("world-writable manifest dir was trusted: %d reviews posted", reviewed) + } + if _, err := os.Stat(reqPath); err != nil { + t.Errorf("request should be left untouched: %v", err) + } +} + +// The sandbox owns its workspace and can swap the spool for a symlink. The +// relay reads the spool through an os.Root at the workspace, so a link out +// of it is never followed: the request outside is neither posted nor moved. +func TestSandboxSpool_SymlinkedSpoolCannotEscapeWorkspace(t *testing.T) { + reviewed := 0 + srv := newReviewMockServer(t, &reviewed, nil) + defer srv.Close() + c := reviewTestClient(t, srv.URL) + withReviewDir(t) + manifests := withSandboxSpools(t) + spool, _ := newSandboxSpool(t, manifests, "reviewer-spyre", false) + + outside := t.TempDir() + victim, err := WriteReviewRequest(outside, ReviewRequest{Repo: "o/r", Number: 7, Event: "comment", Body: "x"}) + if err != nil { + t.Fatal(err) + } + if err := os.Remove(spool); err != nil { + t.Fatal(err) + } + if err := os.Symlink(outside, spool); err != nil { + t.Fatal(err) + } + c.ProcessReviewRequestsOnce(context.Background()) + + if reviewed != 0 { + t.Fatalf("relay followed the spool symlink out of the workspace: %d reviews posted", reviewed) + } + if _, err := os.Stat(victim); err != nil { + t.Errorf("file outside the workspace was moved: %v", err) + } + if _, err := os.Stat(strings.TrimSuffix(victim, ".json") + ".result.json"); !errors.Is(err, os.ErrNotExist) { + t.Errorf("relay wrote a result outside the workspace: %v", err) + } +} + +// A request file that is itself a symlink is set aside, never read. +func TestSandboxSpool_RejectsSymlinkedRequest(t *testing.T) { + reviewed := 0 + srv := newReviewMockServer(t, &reviewed, nil) + defer srv.Close() + c := reviewTestClient(t, srv.URL) + withReviewDir(t) + manifests := withSandboxSpools(t) + spool, _ := newSandboxSpool(t, manifests, "reviewer-spyre", false) + + target, err := WriteReviewRequest(filepath.Dir(spool), ReviewRequest{Repo: "o/r", Number: 7, Event: "comment", Body: "x"}) + if err != nil { + t.Fatal(err) + } + link := filepath.Join(spool, "link.json") + if err := os.Symlink(target, link); err != nil { + t.Fatal(err) + } + c.ProcessReviewRequestsOnce(context.Background()) + + if reviewed != 0 { + t.Fatalf("symlinked request was delivered: %d reviews posted", reviewed) + } + if _, err := os.Lstat(link + ".rejected"); err != nil { + t.Errorf("symlinked request should be set aside as .rejected: %v", err) + } +} + +// Once the kick has ended and nothing is left to deliver, the manifest and +// its staging dir go; while the kick runs they stay. +func TestSandboxSpool_PrunesOnlyEndedAndDrainedManifests(t *testing.T) { + c := reviewTestClient(t, "http://127.0.0.1:0") + withReviewDir(t) + manifests := withSandboxSpools(t) + + _, running := newSandboxSpool(t, manifests, "running", false) + _, ended := newSandboxSpool(t, manifests, "ended", true) + if !hasSandboxReviewSpools(manifests) { + t.Fatal("hasSandboxReviewSpools = false with two manifests registered") + } + c.ProcessReviewRequestsOnce(context.Background()) + + if _, err := os.Stat(running); err != nil { + t.Errorf("running kick's manifest was pruned: %v", err) + } + if _, err := os.Stat(ended); !errors.Is(err, os.ErrNotExist) { + t.Errorf("ended, drained manifest was kept: %v", err) + } + if _, err := os.Stat(strings.TrimSuffix(ended, ".json") + ".d"); !errors.Is(err, os.ErrNotExist) { + t.Errorf("ended manifest's staging dir was kept: %v", err) + } +} + +// A manifest whose spool is not a workspace spool, or that outlived any +// kick, is dropped without reading anything. +func TestSandboxSpool_DropsInvalidAndStaleManifests(t *testing.T) { + now := time.Now().UTC() + for _, tc := range []struct { + name string + m crux.SandboxReviewSpool + }{ + {"no agent", crux.SandboxReviewSpool{Spool: "/data/agents/sandbox/x/.hive/review-requests", StartedAt: now}}, + {"not a workspace spool", crux.SandboxReviewSpool{Agent: "a", Spool: "/var/run/hive-metrics/review-requests", StartedAt: now}}, + {"relative", crux.SandboxReviewSpool{Agent: "a", Spool: "x/.hive/review-requests", StartedAt: now}}, + {"stale", crux.SandboxReviewSpool{Agent: "a", Spool: "/data/agents/sandbox/x/.hive/review-requests", StartedAt: now.Add(-sandboxReviewSpoolMaxAge - time.Hour)}}, + } { + t.Run(tc.name, func(t *testing.T) { + c := reviewTestClient(t, "http://127.0.0.1:0") + withReviewDir(t) + manifests := withSandboxSpools(t) + path := writeSandboxManifest(t, manifests, "m", tc.m) + if got := c.loadSandboxReviewSpools(now); len(got) != 0 { + t.Fatalf("loaded %d spools, want 0", len(got)) + } + if _, err := os.Stat(path); !errors.Is(err, os.ErrNotExist) { + t.Errorf("invalid manifest was kept: %v", err) + } + }) + } +} + +func TestHasSandboxReviewSpools_EmptyAndMissing(t *testing.T) { + if hasSandboxReviewSpools(filepath.Join(t.TempDir(), "missing")) { + t.Error("missing manifest dir reported spools") + } + empty := t.TempDir() + if err := os.Mkdir(filepath.Join(empty, "staging.d"), 0o700); err != nil { + t.Fatal(err) + } + if hasSandboxReviewSpools(empty) { + t.Error("manifest dir holding only a staging dir reported spools") + } +} diff --git a/src/pkg/github/review_request_watcher.go b/src/pkg/github/review_request_watcher.go index a57e8d8adb..64c6074dd9 100644 --- a/src/pkg/github/review_request_watcher.go +++ b/src/pkg/github/review_request_watcher.go @@ -164,7 +164,7 @@ func (c *Client) StartReviewRequestWatcher(ctx context.Context, authz ReviewRequ // them (the race detector flagged exactly that on v4 CI). Reading them // here is sequenced with the caller. poll, fallback := reviewRequestPollInterval, reviewRequestFallbackInterval - dir := reviewRequestDir() + dir, spools := reviewRequestDir(), sandboxReviewSpoolsDir() // Event-driven since #13378: a request-file drop (fsnotify) or a // review-shaped webhook delivery (reviewRequestWakeEvent) wakes the // loop; the timer is a reconcile that runs every fallback while webhooks @@ -176,7 +176,7 @@ func (c *Client) StartReviewRequestWatcher(ctx context.Context, authz ReviewRequ go func() { defer close(done) defer stopNotify() - t := time.NewTimer(reviewRequestScanInterval(dirNotify != nil, poll, fallback)) + t := time.NewTimer(reviewRequestScanInterval(dirNotify != nil && !hasSandboxReviewSpools(spools), poll, fallback)) defer t.Stop() for { select { @@ -200,7 +200,10 @@ func (c *Client) StartReviewRequestWatcher(ctx context.Context, authz ReviewRequ default: } } - t.Reset(reviewRequestScanInterval(dirNotify != nil, poll, fallback)) + // A registered sandbox spool keeps the poll cadence: its writes + // land on a shared volume, often from another node, where no + // change notification reaches this pod. + t.Reset(reviewRequestScanInterval(dirNotify != nil && !hasSandboxReviewSpools(spools), poll, fallback)) } }() c.logger.Info("review-request watcher started", slog.String("dir", dir), @@ -220,17 +223,23 @@ func (c *Client) reviewRequestTick(ctx context.Context, nowFn func() time.Time) c.processReviewRequests(WithPacingClass(ctx, PacingLow), nowFn) } +// processReviewRequests drains the pod-local drop box, then every sandbox +// review spool (review_request_sandbox_spool.go). func (c *Client) processReviewRequests(ctx context.Context, nowFn func() time.Time) { - entries, err := os.ReadDir(reviewRequestDir()) + c.processReviewRequestDir(ctx, reviewRequestDir(), reviewRequestSource{}, nowFn) + c.processSandboxReviewSpools(ctx, nowFn) +} + +func (c *Client) processReviewRequestDir(ctx context.Context, dir string, src reviewRequestSource, nowFn func() time.Time) { + entries, err := os.ReadDir(dir) if err != nil { return } for _, e := range entries { - name := e.Name() - if e.IsDir() || !strings.HasSuffix(name, ".json") || strings.HasSuffix(name, ".result.json") { + if !isReviewRequestFile(e) { continue } - c.handleOneReviewRequest(ctx, filepath.Join(reviewRequestDir(), name), nowFn) + c.handleOneReviewRequest(ctx, filepath.Join(dir, e.Name()), src, nowFn) } } @@ -242,7 +251,7 @@ func (c *Client) ProcessReviewRequestsOnce(ctx context.Context) { c.processReviewRequests(ctx, time.Now) } -func (c *Client) handleOneReviewRequest(ctx context.Context, path string, nowFn func() time.Time) { +func (c *Client) handleOneReviewRequest(ctx context.Context, path string, src reviewRequestSource, nowFn func() time.Time) { if !c.reviewRetries.allows(path, nowFn()) { return } @@ -271,6 +280,11 @@ func (c *Client) handleOneReviewRequest(ctx context.Context, path string, nowFn slog.String("path", path), slog.String("error", err.Error())) return } + // A sandbox spool's requests belong to the agent its manifest names, + // whatever the file says (hivecommons/hive#14986). + if src.agent != "" { + req.Agent = src.agent + } apiEvent, state, okEvent := reviewEventToAPI(req.Event) resolveThread := isResolveThreadEvent(req.Event) @@ -332,6 +346,12 @@ func (c *Client) handleOneReviewRequest(ctx context.Context, path string, nowFn // Authorize: forge-resistance (file UID must be the claimed agent) + the same // push-capability gate as opening a PR. A nil authorizer fails closed. fileUID := statUID(data, path) + if src.agent != "" { + // A staged spool file is owned by hive, not the agent: its owning + // UID proves nothing, and the manifest already did the proving. 0 + // is the authorizer's "no owning UID" value. + fileUID = 0 + } if c.reviewAuthz == nil { c.denyReviewRequest(path, req, "no authorizer configured (fail closed)", nowFn) return From eebe28300cfb5ccb3a5c97b141fce17ba113a7a1 Mon Sep 17 00:00:00 2001 From: Danathar Date: Wed, 28 Oct 2026 16:20:53 +0000 Subject: [PATCH 2/5] =?UTF-8?q?=F0=9F=90=9B=20test:=20drop=20duplicate=20r?= =?UTF-8?q?eadReviewResult=20helper=20from=20the=20spool=20test?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit review_request_thread_test.go already declares readReviewResult with the same signature and behaviour, so the package failed to compile. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Danathar --- src/pkg/github/review_request_sandbox_spool_test.go | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/src/pkg/github/review_request_sandbox_spool_test.go b/src/pkg/github/review_request_sandbox_spool_test.go index 82fbb606ae..5bf4793d25 100644 --- a/src/pkg/github/review_request_sandbox_spool_test.go +++ b/src/pkg/github/review_request_sandbox_spool_test.go @@ -58,19 +58,6 @@ func writeSandboxManifest(t *testing.T, manifests, agent string, m crux.SandboxR return path } -func readReviewResult(t *testing.T, reqPath string) ReviewResponse { - t.Helper() - b, err := os.ReadFile(strings.TrimSuffix(reqPath, ".json") + ".result.json") - if err != nil { - t.Fatalf("no result file mirrored into the spool: %v", err) - } - var resp ReviewResponse - if err := json.Unmarshal(b, &resp); err != nil { - t.Fatal(err) - } - return resp -} - // A sandboxed reviewer's spooled comment is posted, attributed to the agent // the MANIFEST names — not the one the file claims — and its result lands // back in the spool where the sandbox can read it. From 8192c8115bf4a6446369343fdd66d6e62075fb17 Mon Sep 17 00:00:00 2001 From: Danathar Date: Wed, 28 Oct 2026 16:27:14 +0000 Subject: [PATCH 3/5] =?UTF-8?q?=F0=9F=90=9B=20test:=20wait=20on=20conditio?= =?UTF-8?q?ns=20instead=20of=20fixed=20sleeps=20in=20sandbox=20tests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TestSleepRatchet failed: the spool drain test added two sleeps. Its relay goroutine now takes the request path from the fake launcher over a channel instead of polling the workspace; the one remaining hold is a deliberate negative wait and says so. The four poll loops in sandbox_executor_test.go use the package's ticker-based waitFor, and the ratchet baseline drops from 198 to 195. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Danathar --- src/crux/agent/sandbox_executor_test.go | 55 ++++++--------------- src/crux/agent/sandbox_review_spool_test.go | 36 ++++++++------ src/internal/testutil/sleep_ratchet_test.go | 2 +- 3 files changed, 38 insertions(+), 55 deletions(-) diff --git a/src/crux/agent/sandbox_executor_test.go b/src/crux/agent/sandbox_executor_test.go index 00e18c5547..ce2f41db90 100644 --- a/src/crux/agent/sandbox_executor_test.go +++ b/src/crux/agent/sandbox_executor_test.go @@ -316,17 +316,11 @@ func TestManagerSandboxPauseCancelsRunningKick(t *testing.T) { if err := m.Pause("scanner", "test", "pause while running"); err != nil { t.Fatalf("Pause: %v", err) } - deadline := time.Now().Add(time.Second) - for time.Now().Before(deadline) { + waitFor(t, "sandbox agent to stay paused after cancellation", func() bool { m.mu.RLock() - done := m.agents["scanner"].cancel == nil && m.agents["scanner"].State == StatePaused - m.mu.RUnlock() - if done { - return - } - time.Sleep(10 * time.Millisecond) - } - t.Fatalf("sandbox agent did not stay paused after cancellation: %+v", m.AllStatuses()["scanner"]) + defer m.mu.RUnlock() + return m.agents["scanner"].cancel == nil && m.agents["scanner"].State == StatePaused + }) } func TestManagerSandboxResumeWhileCancelDrainsReturnsIdle(t *testing.T) { @@ -350,17 +344,11 @@ func TestManagerSandboxResumeWhileCancelDrainsReturnsIdle(t *testing.T) { if err := m.Resume(context.Background(), "scanner", "test", "resume immediately"); err != nil { t.Fatalf("Resume: %v", err) } - deadline := time.Now().Add(time.Second) - for time.Now().Before(deadline) { + waitFor(t, "sandbox agent to return idle after resumed cancellation", func() bool { m.mu.RLock() - done := m.agents["scanner"].cancel == nil && m.agents["scanner"].State == StateIdle - m.mu.RUnlock() - if done { - return - } - time.Sleep(10 * time.Millisecond) - } - t.Fatalf("sandbox agent did not return idle after resumed cancellation: %+v", m.AllStatuses()["scanner"]) + defer m.mu.RUnlock() + return m.agents["scanner"].cancel == nil && m.agents["scanner"].State == StateIdle + }) } func TestManagerSandboxRunningSkippedByTmuxCrashDetector(t *testing.T) { @@ -387,17 +375,11 @@ func TestManagerSandboxRunningSkippedByTmuxCrashDetector(t *testing.T) { if err := m.Pause("scanner", "test", "cleanup"); err != nil { t.Fatalf("Pause cleanup: %v", err) } - deadline := time.Now().Add(time.Second) - for time.Now().Before(deadline) { + waitFor(t, "sandbox cleanup to finish", func() bool { m.mu.RLock() - done := m.agents["scanner"].cancel == nil - m.mu.RUnlock() - if done { - return - } - time.Sleep(10 * time.Millisecond) - } - t.Fatal("sandbox cleanup did not finish") + defer m.mu.RUnlock() + return m.agents["scanner"].cancel == nil + }) } func TestManagerSandboxDoesNotPushWhenModeCannotPush(t *testing.T) { @@ -419,17 +401,10 @@ func TestManagerSandboxDoesNotPushWhenModeCannotPush(t *testing.T) { if err := m.SendKick("scanner", "fix"); err != nil { t.Fatalf("SendKick: %v", err) } - deadline := time.Now().Add(time.Second) - for time.Now().Before(deadline) { - if got := m.AllStatuses()["scanner"].State; got == StateIdle { - if runner.pushed || pr.called { - t.Fatalf("non-push mode pushed=%v pr=%v", runner.pushed, pr.called) - } - return - } - time.Sleep(10 * time.Millisecond) + waitFor(t, "sandbox agent to complete", func() bool { return m.AllStatuses()["scanner"].State == StateIdle }) + if runner.pushed || pr.called { + t.Fatalf("non-push mode pushed=%v pr=%v", runner.pushed, pr.called) } - t.Fatalf("sandbox agent did not complete: %+v", m.AllStatuses()["scanner"]) } func TestAgentSandboxConfigGating(t *testing.T) { diff --git a/src/crux/agent/sandbox_review_spool_test.go b/src/crux/agent/sandbox_review_spool_test.go index b7a7af73f9..f781ba5e20 100644 --- a/src/crux/agent/sandbox_review_spool_test.go +++ b/src/crux/agent/sandbox_review_spool_test.go @@ -25,6 +25,8 @@ type spoolProbeLauncher struct { manifests *[]crux.SandboxReviewSpool // request, when set, is written into the spool as the sandbox's last act. request string + // written, when set, receives the request's path once it is in the spool. + written chan string } func (l spoolProbeLauncher) Run(_ context.Context, spec sandbox.LaunchSpec) (sandbox.Result, error) { @@ -34,9 +36,13 @@ func (l spoolProbeLauncher) Run(_ context.Context, spec sandbox.LaunchSpec) (san *l.manifests = readSpoolManifests(l.manifestDir) if l.request != "" { spool := filepath.Join(spec.Workspace, crux.SandboxReviewSpoolRelPath) - if err := os.WriteFile(filepath.Join(spool, "r.json"), []byte(l.request), 0o660); err != nil { + path := filepath.Join(spool, "r.json") + if err := os.WriteFile(path, []byte(l.request), 0o660); err != nil { return sandbox.Result{}, err } + if l.written != nil { + l.written <- path + } } return sandbox.Result{Stdout: "ok"}, nil } @@ -134,22 +140,24 @@ func TestSandboxExecutorWaitsForTheRelayToDrainTheSpool(t *testing.T) { manifestDir := filepath.Join(t.TempDir(), "sandbox-review-spools") launcher, _, _ := spoolProbe(manifestDir) launcher.request = `{"repo":"o/r","number":7,"event":"comment","body":"x"}` + launcher.written = make(chan string, 1) var consumed atomic.Bool go func() { - // The relay: take the request once it appears. The flag is set - // BEFORE the remove, so a Run that returned only after the spool - // drained always sees it. - deadline := time.Now().Add(5 * time.Second) - for time.Now().Before(deadline) { - matches, _ := filepath.Glob(filepath.Join(root, "*", crux.SandboxReviewSpoolRelPath, "r.json")) - if len(matches) == 1 { - time.Sleep(50 * time.Millisecond) - consumed.Store(true) - _ = os.Remove(matches[0]) - return - } - time.Sleep(time.Millisecond) + // The relay: take the request once the sandbox has written it. The + // flag is set BEFORE the remove, so a Run that returned only after + // the spool drained always sees it. + var path string + select { + case path = <-launcher.written: + case <-time.After(5 * time.Second): + return } + // Deliberate hold, not a wait on a condition: it gives Run time to + // return if it did NOT wait for the drain, so that regression fails + // this test instead of racing the remove below. + time.Sleep(50 * time.Millisecond) + consumed.Store(true) + _ = os.Remove(path) }() exec := &SandboxExecutor{Runner: &sandboxFakeRunner{}, Launcher: launcher} res, err := exec.Run(context.Background(), SandboxKickSpec{ diff --git a/src/internal/testutil/sleep_ratchet_test.go b/src/internal/testutil/sleep_ratchet_test.go index 2320c9affa..c19e350dd5 100644 --- a/src/internal/testutil/sleep_ratchet_test.go +++ b/src/internal/testutil/sleep_ratchet_test.go @@ -21,7 +21,7 @@ import ( // channel / WaitGroup when a goroutine you own can signal directly); a fixed // sleep is only acceptable for a deliberate "nothing happens during this // window" negative wait, and that still needs a comment saying so. -const sleepBaseline = 198 +const sleepBaseline = 195 // sleepCall is the literal the ratchet counts. Kept as a constant so the // message and the count cannot drift apart. From 75c71f4af358b665d450aa4d1a095f83dbbfef8c Mon Sep 17 00:00:00 2001 From: Danathar Date: Wed, 28 Oct 2026 14:02:27 -0400 Subject: [PATCH 4/5] =?UTF-8?q?=F0=9F=93=96=20docs:=20repoint=20the=20scal?= =?UTF-8?q?e-envelope=20RankActionableIssues=20citation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The citation drifted on v5 (client.go grew; RankActionableIssues now spans 4780-4790), so check-docs-citations.py failed the "relative links and anchors resolve" job on this PR and on the v5 tip alike. Not caused by this PR's diff; fixed here so the PR can go green. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BQYnPGFJx81c8JXR3o1sg1 Signed-off-by: Danathar --- src/docs/scale-envelope.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/docs/scale-envelope.md b/src/docs/scale-envelope.md index b270ef9fc3..5278d7936c 100644 --- a/src/docs/scale-envelope.md +++ b/src/docs/scale-envelope.md @@ -34,7 +34,7 @@ ones to: 2. **Truncation must preserve the prioritization signal, not cut an arbitrary slice.** This holds today and is easy to assume backwards: - Issues are sorted by actionable rank tier, then **oldest first** within each tier before any cap is applied - (`RankActionableIssues`, `src/pkg/github/client.go:4772-4782`, descending `AgeMinutes`). + (`RankActionableIssues`, `src/pkg/github/client.go:4780-4790`, descending `AgeMinutes`). - PRs are sorted by **review class then oldest-first within class** (`SortPullRequestsForReview`, `src/pkg/github/review_priority.go:334`; fixes → refactors/docs → tests). From 59acb97fbc482028622a46eaa10e06c980cdf753 Mon Sep 17 00:00:00 2001 From: Danathar Date: Wed, 28 Oct 2026 14:28:48 -0400 Subject: [PATCH 5/5] =?UTF-8?q?=F0=9F=94=92=20fix(review-spool):=20require?= =?UTF-8?q?=20a=20per-run=20MAC=20on=20spooled=20review=20requests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The relay attributed a spooled request to the manifest's agent and skipped the file-owner forge check (fileUID=0). The manifest proves which workspace a request came from, not who wrote it: the spool is 0o770, so any agent sharing hive's group could drop a request there and have it posted as the sandboxed reviewer. Each sandbox kick now mints a random 256-bit spool key. The executor keeps it in the hive-only manifest and passes it to the sandbox only through HIVE_REVIEW_SPOOL_KEY; it is never written to the workspace. hive-review wraps a spooled request as {"request": , "mac": hex HMAC-SHA256(key, request)}. The relay stages only a request whose MAC matches the run's key and that the spool has not already accepted (an O_EXCL marker per MAC in the hive-owned staging dir). An unsigned request, one MACed with another run's key, or a copy of an accepted one is renamed .rejected, logged at WARN and answered with a "rejected:" result. A manifest without a key is dropped. The Podman launcher now passes explicit env by name (`--env K`) with the value in podman's own environment, so the key never shows up in a world-readable /proc//cmdline. Refs #14986 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BQYnPGFJx81c8JXR3o1sg1 Signed-off-by: Danathar --- bin/hive-review.sh | 14 +- bin/test_hive_review.sh | 24 +++ .../added-14986-sandbox-review-spool.md | 2 +- src/crux/agent/sandbox_review_spool.go | 22 ++- src/crux/agent/sandbox_review_spool_test.go | 42 ++++++ src/crux/sandbox/sandbox.go | 8 +- src/crux/sandbox/sandbox_test.go | 17 +++ src/crux/sandbox_review_spool.go | 33 ++++- src/docs/sandbox-isolation.md | 5 +- .../github/review_request_sandbox_spool.go | 93 +++++++++++- .../review_request_sandbox_spool_test.go | 138 ++++++++++++++++-- src/pkg/github/review_request_watcher.go | 6 +- 12 files changed, 370 insertions(+), 34 deletions(-) diff --git a/bin/hive-review.sh b/bin/hive-review.sh index a763d03e2d..f6490b911f 100755 --- a/bin/hive-review.sh +++ b/bin/hive-review.sh @@ -52,7 +52,10 @@ set -euo pipefail # Inside a sandbox (Podman or a Kubernetes Job) the pod-local relay dir is not # reachable; hive sets HIVE_REVIEW_REQUEST_DIR to the kick's workspace spool and # relays from there, attributing the request to the kick's own agent whatever -# this script writes in "agent" (hivecommons/hive#14986). +# this script writes in "agent" (hivecommons/hive#14986). The spool is +# group-writable, so hive also sets HIVE_REVIEW_SPOOL_KEY, a per-run key, and +# accepts only a request MACed with it: the request is then written as +# {"request": "", "mac": ""}. REQ_DIR="${HIVE_REVIEW_REQUEST_DIR:-/var/run/hive-metrics/review-requests}" REPO=""; NUMBER=""; EVENT=""; BODY=""; BODY_FILE=""; THREAD=""; VERDICT_FILE=""; VERDICT=""; REVISE=0 @@ -202,8 +205,15 @@ if verdict.strip(): % (i, obj.get("repo"), obj.get("number"), repo, number)) raise SystemExit(2) req["report"] = verdict +data = json.dumps(req) +# Read from the environment, never argv: argv is visible to every user. +key = os.environ.get("HIVE_REVIEW_SPOOL_KEY", "") +if key: + import hashlib, hmac + mac = hmac.new(key.encode(), data.encode(), hashlib.sha256).hexdigest() + data = json.dumps({"request": data, "mac": mac}) with open(temporary, "w") as fh: - json.dump(req, fh) + fh.write(data) os.replace(temporary, path) PY diff --git a/bin/test_hive_review.sh b/bin/test_hive_review.sh index d5c43b187b..67fb8995e6 100755 --- a/bin/test_hive_review.sh +++ b/bin/test_hive_review.sh @@ -177,6 +177,30 @@ else echo " FAIL: request not written to HIVE_REVIEW_REQUEST_DIR"; FAIL=$((FAIL + 2)) fi +# --- sandbox spool key: the request is MACed with HIVE_REVIEW_SPOOL_KEY --- +echo "" +echo "--- sandbox spool signed with HIVE_REVIEW_SPOOL_KEY ---" +SIGNED_DIR="$TMPDIR_ROOT/signed/.hive/review-requests" +SPOOL_KEY="0123456789abcdef0123456789abcdef" +env HIVE_AGENT="spoolbot" HIVE_REVIEW_REQUEST_DIR="$SIGNED_DIR" HIVE_REVIEW_SPOOL_KEY="$SPOOL_KEY" bash "$SCRIPT" --repo "org/repo" 12 --comment --body "signed" >/dev/null 2>&1 || true +SIGNED_FILES=("$SIGNED_DIR"/spoolbot-*.json) +if [ -f "${SIGNED_FILES[0]}" ]; then + # The relay recomputes HMAC-SHA256(key, request) over the exact request text. + check "envelope MAC verifies under the run key" "ok" "$(python3 -c ' +import hashlib, hmac, json, sys +env = json.load(open(sys.argv[1])) +want = hmac.new(sys.argv[2].encode(), env["request"].encode(), hashlib.sha256).hexdigest() +print("ok" if hmac.compare_digest(env["mac"], want) else "bad")' "${SIGNED_FILES[0]}" "$SPOOL_KEY")" + check "signed request is a comment" "comment" "$(python3 -c 'import json,sys; print(json.loads(json.load(open(sys.argv[1]))["request"])["event"])' "${SIGNED_FILES[0]}")" + if grep -q "$SPOOL_KEY" "${SIGNED_FILES[0]}"; then + echo " FAIL: the spool key itself was written into the spool"; FAIL=$((FAIL + 1)) + else + echo " PASS: the spool key is not in the spool file"; PASS=$((PASS + 1)) + fi +else + echo " FAIL: signed request not written to HIVE_REVIEW_REQUEST_DIR"; FAIL=$((FAIL + 3)) +fi + echo "" echo "=== $PASS passed, $FAIL failed ===" [ "$FAIL" -eq 0 ] diff --git a/changelog.d/added-14986-sandbox-review-spool.md b/changelog.d/added-14986-sandbox-review-spool.md index 1f6f7601d0..f3e70495ea 100644 --- a/changelog.d/added-14986-sandbox-review-spool.md +++ b/changelog.d/added-14986-sandbox-review-spool.md @@ -1 +1 @@ -- A sandboxed reviewer can now post reviews ([#14986](https://github.com/hivecommons/hive/issues/14986)). Every sandbox kick (Podman or `sandbox.runtime: job`) gets a review spool at `.hive/review-requests` in its workspace, and `hive-review` inside the sandbox writes there via `HIVE_REVIEW_REQUEST_DIR`. The review relay picks spooled requests up through the same policy path as the pod-local drop box, attributing each one to the kick's agent from a hive-owned manifest under `/sandbox-review-spools`, never from the file's contents. A reviewer on an accelerator spoke can now run the change on the card node and post its verdict from there. +- A sandboxed reviewer can now post reviews ([#14986](https://github.com/hivecommons/hive/issues/14986)). Every sandbox kick (Podman or `sandbox.runtime: job`) gets a review spool at `.hive/review-requests` in its workspace, and `hive-review` inside the sandbox writes there via `HIVE_REVIEW_REQUEST_DIR`. The review relay picks spooled requests up through the same policy path as the pod-local drop box, attributing each one to the kick's agent from a hive-owned manifest under `/sandbox-review-spools`, never from the file's contents. Because the spool is group-writable, each kick also gets its own random spool key (`HIVE_REVIEW_SPOOL_KEY`, passed only through the sandbox environment), and the relay accepts only requests that carry a valid HMAC under that key and haven't been accepted before, so another agent can't post a review as the sandboxed reviewer by dropping a file into its spool. A reviewer on an accelerator spoke can now run the change on the card node and post its verdict from there. diff --git a/src/crux/agent/sandbox_review_spool.go b/src/crux/agent/sandbox_review_spool.go index e71c928cce..f45cfa2873 100644 --- a/src/crux/agent/sandbox_review_spool.go +++ b/src/crux/agent/sandbox_review_spool.go @@ -2,6 +2,8 @@ package agent import ( "context" + "crypto/rand" + "encoding/hex" "encoding/json" "fmt" "os" @@ -49,9 +51,17 @@ func (e *SandboxExecutor) openReviewSpool(manifestDir, agentName, workspace stri if err := os.Chmod(manifestDir, 0o700); err != nil { return nil, fmt.Errorf("secure review spool manifest dir: %w", err) } + // The run's spool key: the only thing that lets the relay tell this + // sandbox's requests from ones another agent drops into the + // group-writable spool. It reaches the sandbox only through its + // environment, never through the workspace. + key := make([]byte, 32) + if _, err := rand.Read(key); err != nil { + return nil, fmt.Errorf("mint review spool key: %w", err) + } h := &reviewSpoolHandle{ manifest: filepath.Join(manifestDir, fmt.Sprintf("%s-%d.json", sanitizeBranchPart(agentName), time.Now().UnixNano())), - entry: crux.SandboxReviewSpool{Agent: agentName, Spool: spool, StartedAt: e.now().UTC()}, + entry: crux.SandboxReviewSpool{Agent: agentName, Spool: spool, Key: hex.EncodeToString(key), StartedAt: e.now().UTC()}, } if err := h.write(); err != nil { return nil, err @@ -59,13 +69,17 @@ func (e *SandboxExecutor) openReviewSpool(manifestDir, agentName, workspace stri return h, nil } -// sandboxEnv is what the sandbox needs to find the spool: hive-review writes -// to HIVE_REVIEW_REQUEST_DIR when it is set. +// sandboxEnv is what the sandbox needs to use the spool: hive-review writes +// to HIVE_REVIEW_REQUEST_DIR when it is set, and MACs each request with +// HIVE_REVIEW_SPOOL_KEY. func (h *reviewSpoolHandle) sandboxEnv() map[string]string { if h == nil { return nil } - return map[string]string{crux.SandboxReviewSpoolEnv: path.Join(sandbox.DefaultWorkspaceMount, crux.SandboxReviewSpoolRelPath)} + return map[string]string{ + crux.SandboxReviewSpoolEnv: path.Join(sandbox.DefaultWorkspaceMount, crux.SandboxReviewSpoolRelPath), + crux.SandboxReviewSpoolKeyEnv: h.entry.Key, + } } func (h *reviewSpoolHandle) write() error { diff --git a/src/crux/agent/sandbox_review_spool_test.go b/src/crux/agent/sandbox_review_spool_test.go index f781ba5e20..055dff766d 100644 --- a/src/crux/agent/sandbox_review_spool_test.go +++ b/src/crux/agent/sandbox_review_spool_test.go @@ -5,6 +5,7 @@ import ( "encoding/json" "os" "path/filepath" + "strings" "sync" "sync/atomic" "testing" @@ -105,6 +106,47 @@ func TestSandboxExecutorRegistersReviewSpool(t *testing.T) { if len(after) != 1 || after[0].EndedAt == nil { t.Errorf("manifest after the run = %+v, want one marked ended", after) } + // The run's spool key reaches the sandbox through its env only: it is + // in the manifest the relay reads, never in the workspace. + key := (*during)[0].Key + if len(key) != 64 { + t.Fatalf("manifest spool key = %q, want 256 bits hex-encoded", key) + } + if got := (*env)[crux.SandboxReviewSpoolKeyEnv]; got != key { + t.Errorf("sandbox %s = %q, want the manifest's key", crux.SandboxReviewSpoolKeyEnv, got) + } + if sandbox.IsCredentialName(crux.SandboxReviewSpoolKeyEnv) { + t.Errorf("%s would be stripped by the launchers' credential filter", crux.SandboxReviewSpoolKeyEnv) + } + _ = filepath.WalkDir(res.Workspace, func(p string, d os.DirEntry, err error) error { + if err == nil && !d.IsDir() { + if b, rerr := os.ReadFile(p); rerr == nil && strings.Contains(string(b), key) { + t.Errorf("spool key written into the workspace at %s", p) + } + } + return nil + }) +} + +// Each kick mints its own key, so a request signed in one run is refused in +// any other. +func TestSandboxExecutorMintsAKeyPerKick(t *testing.T) { + manifestDir := filepath.Join(t.TempDir(), "sandbox-review-spools") + keys := map[string]bool{} + for i := 0; i < 2; i++ { + launcher, env, _ := spoolProbe(manifestDir) + exec := &SandboxExecutor{Runner: &sandboxFakeRunner{}, Launcher: launcher} + if _, err := exec.Run(context.Background(), SandboxKickSpec{ + Agent: "reviewer-spyre", AgentConfig: configSnapshot{Backend: "claude"}, Message: "review #7", + Org: "o", Repo: "r", WorkspaceDir: t.TempDir(), Image: "img", ReviewSpoolDir: manifestDir, + }); err != nil { + t.Fatalf("Run: %v", err) + } + keys[(*env)[crux.SandboxReviewSpoolKeyEnv]] = true + } + if len(keys) != 2 || keys[""] { + t.Errorf("spool keys across two kicks = %v, want two distinct keys", keys) + } } // POSITIVE CONTROL: with no manifest dir the sandbox gets no spool and no diff --git a/src/crux/sandbox/sandbox.go b/src/crux/sandbox/sandbox.go index c9c93c30b2..64393effa9 100644 --- a/src/crux/sandbox/sandbox.go +++ b/src/crux/sandbox/sandbox.go @@ -110,9 +110,13 @@ func PodmanArgs(spec LaunchSpec) ([]string, error) { args = append(args, "--userns=keep-id") args = append(args, "-v", filepath.Clean(spec.Workspace)+":"+mount+":Z") args = append(args, "--workdir", workdir) - for k, v := range spec.Env { + // Name only: podman takes the value from its own environment, which Run + // sets from the same spec.Env. A value on the command line would be + // readable by every user on the host through /proc//cmdline, and + // some values (the review spool key, #14986) are per-run secrets. + for k := range spec.Env { if !isCredentialName(k) { - args = append(args, "--env", k+"="+v) + args = append(args, "--env", k) } } args = append(args, spec.Image) diff --git a/src/crux/sandbox/sandbox_test.go b/src/crux/sandbox/sandbox_test.go index 45d78c4d26..f30a33b148 100644 --- a/src/crux/sandbox/sandbox_test.go +++ b/src/crux/sandbox/sandbox_test.go @@ -24,6 +24,23 @@ func TestPodmanArgsUseCredentialFreeNetworklessDefaults(t *testing.T) { } } +// Explicit env goes to podman by name only; the value travels in podman's +// own environment, never its argv, which every host user can read through +// /proc//cmdline (the review spool key is a per-run secret, #14986). +func TestPodmanArgsKeepEnvValuesOffTheCommandLine(t *testing.T) { + args, err := PodmanArgs(LaunchSpec{Image: "agent:latest", Workspace: "/w", Command: []string{"true"}, Env: map[string]string{"HIVE_REVIEW_SPOOL_KEY": "per-run-key-value"}}) + if err != nil { + t.Fatal(err) + } + joined := strings.Join(args, " ") + if strings.Contains(joined, "per-run-key-value") { + t.Fatalf("env value on podman argv: %q", joined) + } + if !strings.Contains(joined, "--env HIVE_REVIEW_SPOOL_KEY ") { + t.Fatalf("args %q do not pass HIVE_REVIEW_SPOOL_KEY by name", joined) + } +} + func TestSanitizedEnvAllowlistDropsCredentials(t *testing.T) { env := SanitizedEnv([]string{"PATH=/bin", "HOME=/home/dev", "GITHUB_TOKEN=secret", "MY_TOKEN=secret"}, []string{"PATH", "HOME", "GITHUB_TOKEN", "MY_TOKEN"}, map[string]string{"SAFE": "1", "GH_TOKEN": "nope"}) if !slices.Contains(env, "PATH=/bin") || !slices.Contains(env, "HOME=/home/dev") || !slices.Contains(env, "SAFE=1") { diff --git a/src/crux/sandbox_review_spool.go b/src/crux/sandbox_review_spool.go index 743f5844ed..6fde6bcf1d 100644 --- a/src/crux/sandbox_review_spool.go +++ b/src/crux/sandbox_review_spool.go @@ -1,6 +1,9 @@ package crux import ( + "crypto/hmac" + "crypto/sha256" + "encoding/hex" "path/filepath" "time" ) @@ -20,8 +23,12 @@ const ( SandboxReviewSpoolRelPath = ".hive/review-requests" // SandboxReviewSpoolEnv is the environment variable hive sets inside the // sandbox to the spool's in-sandbox path. - SandboxReviewSpoolEnv = "HIVE_REVIEW_REQUEST_DIR" - sandboxReviewSpoolsDir = "sandbox-review-spools" + SandboxReviewSpoolEnv = "HIVE_REVIEW_REQUEST_DIR" + // SandboxReviewSpoolKeyEnv carries the kick's spool key into the + // sandbox. The name must not contain TOKEN or SECRET: the launchers + // strip those (sandbox.IsCredentialName). + SandboxReviewSpoolKeyEnv = "HIVE_REVIEW_SPOOL_KEY" + sandboxReviewSpoolsDir = "sandbox-review-spools" ) // SandboxReviewSpoolsDir is where the sandbox executor writes one manifest @@ -34,9 +41,31 @@ func SandboxReviewSpoolsDir(workDir string) string { // SandboxReviewSpool is one kick's manifest. Spool is the host-side path of // the workspace spool. EndedAt is set once the sandbox has exited; the relay // drops the manifest after that when nothing is left to deliver. +// +// Key is the kick's spool key: random, minted by the executor for this run +// only, and handed to the sandbox solely through its environment. The spool +// is group-writable, so knowing WHICH workspace a request came from does not +// say WHO wrote it; the relay accepts only a request MACed with this key +// (SandboxReviewMAC), which nothing outside this run's sandbox holds. type SandboxReviewSpool struct { Agent string `json:"agent"` Spool string `json:"spool"` + Key string `json:"key"` StartedAt time.Time `json:"started_at"` EndedAt *time.Time `json:"ended_at,omitempty"` } + +// SandboxReviewEnvelope is what a sandbox writes to its spool: the request +// JSON, verbatim, and its MAC under the kick's spool key. +type SandboxReviewEnvelope struct { + Request string `json:"request"` + MAC string `json:"mac"` +} + +// SandboxReviewMAC is hex(HMAC-SHA256(key, request)), with the key's text as +// the HMAC key. bin/hive-review.sh computes the same value. +func SandboxReviewMAC(key string, request []byte) string { + m := hmac.New(sha256.New, []byte(key)) + m.Write(request) + return hex.EncodeToString(m.Sum(nil)) +} diff --git a/src/docs/sandbox-isolation.md b/src/docs/sandbox-isolation.md index 7c689e7972..b4b4d32307 100644 --- a/src/docs/sandbox-isolation.md +++ b/src/docs/sandbox-isolation.md @@ -214,11 +214,12 @@ A sandbox cannot reach the pod-local review relay (`/var/run/hive-metrics/review 1. Before launch, hive creates `.hive/review-requests` in the kick workspace and sets `HIVE_REVIEW_REQUEST_DIR=/workspace/.hive/review-requests` inside the sandbox. `hive-review` writes its request there instead of the pod-local dir, in the same JSON schema. 2. Hive registers the spool with a manifest under `/sandbox-review-spools/` (`work_dir` is `HIVE_WORK_DIR`, default `/data/agents`). The manifest names the kick's agent. It lives outside every workspace, in a directory only hive's uid may write, so the sandbox can neither see nor forge it. 3. The review-request watcher moves each spooled request into a hive-owned staging directory beside the manifest and runs it through the same handler as the pod-local drop box. `review.max_reviews_per_head`, repo pause, the lane write allowlist, the sentinel approval block, the contributor COMMENT-only guard, the canary scan and the `agent_pr_reviewed` audit all apply unchanged. The request is **attributed to the manifest's agent**, whatever its own `agent` field says; the per-agent authorization (ACMM `CanPush` or the `converse` capability) is checked for that agent. -4. The result is copied back to `.result.json` in the spool, where the sandboxed agent reads it exactly as it would in-pod. A bad `--verdict-file` is refused with the validator's message, so the agent can fix the verdict and resubmit within the same kick. +4. The manifest says which workspace a request came from, not who wrote it: the spool is group-writable, so any agent sharing hive's group could drop a file into it. So hive also mints a random 256-bit **spool key** for each kick, keeps it in the manifest, and hands it to the sandbox only through `HIVE_REVIEW_SPOOL_KEY` in its environment (the Podman launcher passes env by name, so the value never appears on a command line). `hive-review` writes `{"request": "", "mac": ""}`. The relay stages only a request whose MAC matches this run's key and that this spool has not already accepted. An unsigned request, one signed with another run's key, or a copy of an accepted one is set aside as `.rejected`, logged at WARN, and answered with a `rejected:` result file. This stands in for the file-owner check the pod-local relay does, which can't apply here because a Job's uid is not an agent's. +5. The result is copied back to `.result.json` in the spool, where the sandboxed agent reads it exactly as it would in-pod. A bad `--verdict-file` is refused with the validator's message, so the agent can fix the verdict and resubmit within the same kick. While any spool is registered, the watcher scans on its short poll interval: a Job writes its spool over a shared volume, often from another node, where no file-change notification reaches the hive pod. After the sandbox exits, the kick waits up to a minute for the relay to take any request still in the spool, because a run-stage worktree is removed when the kick returns. The manifest is dropped once the kick has ended and nothing is left to deliver, or after 24 hours. -The relay touches the spool only through an `os.Root` opened at the workspace. A symlink the sandbox plants can't redirect a read, rename or write outside its own workspace, and a request file that is a symlink, FIFO or oversized is set aside as `.rejected`. The relay ignores a manifest directory or manifest that is not owned by its own uid or is group- or world-writable. In the sidecar runtime (`CRUX_MODE=sidecar`), the runtime and hive must therefore share both `/data` and a uid. +The relay touches the spool only through an `os.Root` opened at the workspace. A symlink the sandbox plants can't redirect a read, rename or write outside its own workspace, and a request file that is a symlink, FIFO or oversized is set aside as `.rejected`. The relay ignores a manifest directory or manifest that is not owned by its own uid or is group- or world-writable, and a manifest without a spool key. In the sidecar runtime (`CRUX_MODE=sidecar`), the runtime and hive must therefore share both `/data` and a uid. On `sandbox.runtime: job`, the key is in the Job's pod spec, so anything that can read Jobs or Pods in the namespace through the Kubernetes API can read it; agents must not hold that permission. A group member can still delete or alter a request in the spool, which loses the review but cannot post one. Out of scope: the quality and deps lanes are unchanged, and nothing inside a Job writes to GitHub directly. diff --git a/src/pkg/github/review_request_sandbox_spool.go b/src/pkg/github/review_request_sandbox_spool.go index e70876a6c8..e38c84e4c1 100644 --- a/src/pkg/github/review_request_sandbox_spool.go +++ b/src/pkg/github/review_request_sandbox_spool.go @@ -2,6 +2,7 @@ package github import ( "context" + "crypto/hmac" "encoding/json" "errors" "fmt" @@ -44,6 +45,16 @@ import ( // sandbox cannot reach it: it lives outside every workspace, in an // owner-only directory, and the relay ignores one that is not owned by its // own uid or that anyone else could write. +// +// The manifest says which workspace a request came from, not who wrote it: +// the spool is group-writable, so any agent sharing hive's group could drop +// a file there. So the request must also carry a MAC under the run's spool +// key (crux.SandboxReviewEnvelope), which the executor mints per kick, keeps +// in the manifest and hands to the sandbox only through its environment. A +// request without a valid MAC, or one whose MAC this spool has already +// accepted (a copied request), is set aside as .rejected and never staged. +// That is what stands in for the file-owner forge check the pod-local path +// relies on. // sandboxReviewSpoolsDirForTest lets tests point the relay at a temp dir. var sandboxReviewSpoolsDirForTest string @@ -60,6 +71,10 @@ func sandboxReviewSpoolsDir() string { // before marking it ended. const sandboxReviewSpoolMaxAge = 24 * time.Hour +// sandboxReviewSpoolKeyMinLen is the shortest spool key a manifest may carry: +// 128 bits, hex-encoded. The executor mints 256. +const sandboxReviewSpoolKeyMinLen = 32 + // sandboxReviewManifestMaxBytes caps a manifest read; a real one is a few // hundred bytes. const sandboxReviewManifestMaxBytes = 16 << 10 @@ -109,7 +124,7 @@ func (c *Client) processSandboxReviewSpools(ctx context.Context, nowFn func() ti ws, rel := s.workspace() root, err := os.OpenRoot(ws) if err == nil { - c.ingestSandboxSpool(root, rel, staging, nowFn()) + c.ingestSandboxSpool(root, rel, staging, s.Key, nowFn()) } c.processReviewRequestDir(ctx, staging, reviewRequestSource{agent: s.Agent}, nowFn) spoolPending := false @@ -225,6 +240,8 @@ func sandboxSpoolRefusal(m crux.SandboxReviewSpool, now time.Time) string { return "manifest names no agent" case !filepath.IsAbs(m.Spool) || filepath.Clean(m.Spool) != m.Spool || !strings.HasSuffix(m.Spool, suffix): return "spool is not /" + crux.SandboxReviewSpoolRelPath + case len(m.Key) < sandboxReviewSpoolKeyMinLen: + return "manifest carries no spool key" case !m.StartedAt.IsZero() && now.Sub(m.StartedAt) > sandboxReviewSpoolMaxAge: return "manifest is older than " + sandboxReviewSpoolMaxAge.String() } @@ -234,10 +251,10 @@ func sandboxSpoolRefusal(m crux.SandboxReviewSpool, now time.Time) string { // ingestSandboxSpool moves each request out of the spool into staging. A // name already staged (a request still retrying) waits in the spool for a // later pass rather than replacing it. A file that is not yet valid JSON is -// left alone for inFlightGrace, in case the sandbox is still writing it; past -// that it is staged as-is and fails validation there, so the agent gets a -// result file that says why. -func (c *Client) ingestSandboxSpool(root *os.Root, rel, staging string, now time.Time) { +// left alone for inFlightGrace, in case the sandbox is still writing it. +// Only the request inside an envelope MACed with this run's key is staged; +// anything else is set aside (sandboxSpoolEnvelope). +func (c *Client) ingestSandboxSpool(root *os.Root, rel, staging, key string, now time.Time) { d, err := root.Open(rel) if err != nil { return @@ -269,13 +286,77 @@ func (c *Client) ingestSandboxSpool(root *os.Root, rel, staging string, now time if !json.Valid(data) && now.Sub(fi.ModTime()) < inFlightGrace { continue } - if err := writeRequestFile(dst, data); err != nil { + request, mac, reason := sandboxSpoolEnvelope(data, key) + if reason != "" { + c.rejectSandboxSpoolFile(root, src, reason, now) + continue + } + marker, err := markSpoolMACSeen(staging, mac) + if errors.Is(err, fs.ErrExist) { + c.rejectSandboxSpoolFile(root, src, "replay of a request this spool already accepted", now) + continue + } + if err != nil { + continue // left in the spool for the next pass + } + if err := writeRequestFile(dst, request); err != nil { + _ = os.Remove(marker) continue } _ = root.Remove(src) } } +// sandboxSpoolEnvelope returns the request inside data and its MAC, or why +// data is not a request this run's sandbox wrote. A request without the key +// is indistinguishable from one another agent planted, so it is refused. +func sandboxSpoolEnvelope(data []byte, key string) (request []byte, mac, reason string) { + var env crux.SandboxReviewEnvelope + if err := json.Unmarshal(data, &env); err != nil || env.Request == "" || env.MAC == "" { + return nil, "", "not a signed spool envelope" + } + request = []byte(env.Request) + want := crux.SandboxReviewMAC(key, request) + if !hmac.Equal([]byte(env.MAC), []byte(want)) { + return nil, "", "MAC does not match this run's spool key" + } + return request, want, "" +} + +// markSpoolMACSeen records that this spool accepted a request with mac, so +// a copy of it (the spool is group-readable) cannot be submitted twice. It +// returns the marker path; an fs.ErrExist error means a replay. +func markSpoolMACSeen(staging, mac string) (string, error) { + seen := filepath.Join(staging, "seen") + if err := os.MkdirAll(seen, 0o700); err != nil { + return "", err + } + marker := filepath.Join(seen, mac) + f, err := os.OpenFile(marker, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) + if err != nil { + return "", err + } + _ = f.Close() + return marker, nil +} + +// rejectSandboxSpoolFile sets an unsigned, mis-signed or replayed request +// aside as .rejected and leaves a result file saying why, so a sandbox +// running an old hive-review is not left waiting. +func (c *Client) rejectSandboxSpoolFile(root *os.Root, src, reason string, now time.Time) { + _ = root.Rename(src, src+".rejected") + c.logger.Warn("review-request watcher: REJECTED sandbox spool file (not from this run's sandbox)", + slog.String("path", src), slog.String("reason", reason)) + result := strings.TrimSuffix(src, ".json") + ".result.json" + if _, err := root.Lstat(result); err == nil { + return // never overwrite a real result with this one + } + resp := ReviewResponse{OK: false, Error: "rejected: " + reason, At: now.UTC().Format(time.RFC3339)} + if b, err := json.Marshal(resp); err == nil { + _ = writeRootFile(root, result, b) + } +} + // mirrorSandboxResults copies each staged result into the spool. A result // whose request has left staging is final, so its staged copy is dropped // once mirrored; one still retrying is mirrored again on the next pass. diff --git a/src/pkg/github/review_request_sandbox_spool_test.go b/src/pkg/github/review_request_sandbox_spool_test.go index 5bf4793d25..eeafb446e3 100644 --- a/src/pkg/github/review_request_sandbox_spool_test.go +++ b/src/pkg/github/review_request_sandbox_spool_test.go @@ -36,7 +36,7 @@ func newSandboxSpool(t *testing.T, manifests, agent string, ended bool) (spool, if err := os.MkdirAll(spool, 0o770); err != nil { t.Fatal(err) } - m := crux.SandboxReviewSpool{Agent: agent, Spool: spool, StartedAt: time.Now().UTC()} + m := crux.SandboxReviewSpool{Agent: agent, Spool: spool, Key: testSpoolKey(agent), StartedAt: time.Now().UTC()} if ended { at := time.Now().UTC() m.EndedAt = &at @@ -45,6 +45,28 @@ func newSandboxSpool(t *testing.T, manifests, agent string, ended bool) (spool, return spool, manifest } +// testSpoolKey is the spool key a test manifest for agent carries. +func testSpoolKey(agent string) string { return crux.SandboxReviewMAC("test-key", []byte(agent)) } + +// writeSignedSpoolRequest writes req into spool as hive-review does inside a +// sandbox: the request JSON in an envelope MACed with key. +func writeSignedSpoolRequest(t *testing.T, spool, name, key string, req ReviewRequest) string { + t.Helper() + raw, err := json.Marshal(req) + if err != nil { + t.Fatal(err) + } + b, err := json.Marshal(crux.SandboxReviewEnvelope{Request: string(raw), MAC: crux.SandboxReviewMAC(key, raw)}) + if err != nil { + t.Fatal(err) + } + path := filepath.Join(spool, name) + if err := os.WriteFile(path, b, 0o660); err != nil { + t.Fatal(err) + } + return path +} + func writeSandboxManifest(t *testing.T, manifests, agent string, m crux.SandboxReviewSpool) string { t.Helper() b, err := json.Marshal(m) @@ -81,10 +103,7 @@ func TestSandboxSpool_PostsReviewAttributedToManifestAgent(t *testing.T) { } }) - reqPath, err := WriteReviewRequest(spool, ReviewRequest{Repo: "o/r", Number: 7, Event: "comment", Body: "card run passed", Agent: "someone-else"}) - if err != nil { - t.Fatal(err) - } + reqPath := writeSignedSpoolRequest(t, spool, "r.json", testSpoolKey("reviewer-spyre"), ReviewRequest{Repo: "o/r", Number: 7, Event: "comment", Body: "card run passed", Agent: "someone-else"}) c.ProcessReviewRequestsOnce(context.Background()) if reviewed != 1 || apiEvent != "COMMENT" { @@ -119,10 +138,7 @@ func TestSandboxSpool_DenialIsMirroredAndNothingPosted(t *testing.T) { manifests := withSandboxSpools(t) spool, _ := newSandboxSpool(t, manifests, "reviewer-spyre", false) - reqPath, err := WriteReviewRequest(spool, ReviewRequest{Repo: "o/r", Number: 7, Event: "comment", Body: "x", Agent: "reviewer-spyre"}) - if err != nil { - t.Fatal(err) - } + reqPath := writeSignedSpoolRequest(t, spool, "r.json", testSpoolKey("reviewer-spyre"), ReviewRequest{Repo: "o/r", Number: 7, Event: "comment", Body: "x", Agent: "reviewer-spyre"}) c.ProcessReviewRequestsOnce(context.Background()) if reviewed != 0 { @@ -134,6 +150,100 @@ func TestSandboxSpool_DenialIsMirroredAndNothingPosted(t *testing.T) { } } +// The spool is group-writable, so another agent sharing hive's group can +// drop a request into it. Without a MAC under this run's key — unsigned, or +// signed with another run's key (a request copied from a different spool) — +// it is refused, set aside as .rejected and answered, and nothing is posted. +func TestSandboxSpool_RefusesRequestNotSignedForThisRun(t *testing.T) { + for _, tc := range []struct { + name string + write func(t *testing.T, spool string) string + }{ + {"unsigned, as another agent would write it", func(t *testing.T, spool string) string { + p, err := WriteReviewRequest(spool, ReviewRequest{Repo: "o/r", Number: 7, Event: "approve", Agent: "reviewer-spyre"}) + if err != nil { + t.Fatal(err) + } + return p + }}, + {"signed for another run", func(t *testing.T, spool string) string { + return writeSignedSpoolRequest(t, spool, "r.json", testSpoolKey("other-run"), ReviewRequest{Repo: "o/r", Number: 7, Event: "approve"}) + }}, + {"MAC over different bytes", func(t *testing.T, spool string) string { + raw, _ := json.Marshal(ReviewRequest{Repo: "o/r", Number: 7, Event: "comment", Body: "harmless"}) + forged, _ := json.Marshal(ReviewRequest{Repo: "o/r", Number: 7, Event: "approve"}) + b, _ := json.Marshal(crux.SandboxReviewEnvelope{Request: string(forged), MAC: crux.SandboxReviewMAC(testSpoolKey("reviewer-spyre"), raw)}) + p := filepath.Join(spool, "r.json") + if err := os.WriteFile(p, b, 0o660); err != nil { + t.Fatal(err) + } + return p + }}, + } { + t.Run(tc.name, func(t *testing.T) { + reviewed := 0 + srv := newReviewMockServer(t, &reviewed, nil) + defer srv.Close() + c := reviewTestClient(t, srv.URL) + authorized := false + c.reviewAuthz = func(agent string, uid int) error { authorized = true; return nil } + withReviewDir(t) + manifests := withSandboxSpools(t) + spool, _ := newSandboxSpool(t, manifests, "reviewer-spyre", false) + + reqPath := tc.write(t, spool) + c.ProcessReviewRequestsOnce(context.Background()) + + if reviewed != 0 || authorized { + t.Fatalf("request not signed for this run reached the relay: %d reviews posted, authorized=%v", reviewed, authorized) + } + if _, err := os.Lstat(reqPath + ".rejected"); err != nil { + t.Errorf("request should be set aside as .rejected: %v", err) + } + if resp := readReviewResult(t, reqPath); resp.OK || !strings.Contains(resp.Error, "rejected") { + t.Errorf("result = %+v, want a rejection", resp) + } + }) + } +} + +// A copy of a request this spool already accepted — the spool is +// group-readable — is a replay, refused even under a fresh name. +func TestSandboxSpool_RefusesReplayOfAcceptedRequest(t *testing.T) { + reviewed := 0 + srv := newReviewMockServer(t, &reviewed, nil) + defer srv.Close() + c := reviewTestClient(t, srv.URL) + c.reviewAuthz = func(agent string, uid int) error { return nil } + withReviewDir(t) + manifests := withSandboxSpools(t) + spool, _ := newSandboxSpool(t, manifests, "reviewer-spyre", false) + + req := ReviewRequest{Repo: "o/r", Number: 7, Event: "comment", Body: "card run passed"} + first := writeSignedSpoolRequest(t, spool, "first.json", testSpoolKey("reviewer-spyre"), req) + copied, err := os.ReadFile(first) + if err != nil { + t.Fatal(err) + } + c.ProcessReviewRequestsOnce(context.Background()) + if reviewed != 1 { + t.Fatalf("signed request: %d reviews posted, want 1", reviewed) + } + + replay := filepath.Join(spool, "replay.json") + if err := os.WriteFile(replay, copied, 0o660); err != nil { + t.Fatal(err) + } + c.ProcessReviewRequestsOnce(context.Background()) + + if reviewed != 1 { + t.Fatalf("replayed request was posted: %d reviews, want 1", reviewed) + } + if _, err := os.Lstat(replay + ".rejected"); err != nil { + t.Errorf("replay should be set aside as .rejected: %v", err) + } +} + // The manifest is the whole of the trust: a manifest dir others can write is // ignored, so nothing in it is delivered. func TestSandboxSpool_IgnoresWritableManifestDir(t *testing.T) { @@ -258,10 +368,12 @@ func TestSandboxSpool_DropsInvalidAndStaleManifests(t *testing.T) { name string m crux.SandboxReviewSpool }{ - {"no agent", crux.SandboxReviewSpool{Spool: "/data/agents/sandbox/x/.hive/review-requests", StartedAt: now}}, - {"not a workspace spool", crux.SandboxReviewSpool{Agent: "a", Spool: "/var/run/hive-metrics/review-requests", StartedAt: now}}, - {"relative", crux.SandboxReviewSpool{Agent: "a", Spool: "x/.hive/review-requests", StartedAt: now}}, - {"stale", crux.SandboxReviewSpool{Agent: "a", Spool: "/data/agents/sandbox/x/.hive/review-requests", StartedAt: now.Add(-sandboxReviewSpoolMaxAge - time.Hour)}}, + {"no agent", crux.SandboxReviewSpool{Spool: "/data/agents/sandbox/x/.hive/review-requests", Key: testSpoolKey("a"), StartedAt: now}}, + {"not a workspace spool", crux.SandboxReviewSpool{Agent: "a", Spool: "/var/run/hive-metrics/review-requests", Key: testSpoolKey("a"), StartedAt: now}}, + {"relative", crux.SandboxReviewSpool{Agent: "a", Spool: "x/.hive/review-requests", Key: testSpoolKey("a"), StartedAt: now}}, + {"stale", crux.SandboxReviewSpool{Agent: "a", Spool: "/data/agents/sandbox/x/.hive/review-requests", Key: testSpoolKey("a"), StartedAt: now.Add(-sandboxReviewSpoolMaxAge - time.Hour)}}, + {"no spool key", crux.SandboxReviewSpool{Agent: "a", Spool: "/data/agents/sandbox/x/.hive/review-requests", StartedAt: now}}, + {"short spool key", crux.SandboxReviewSpool{Agent: "a", Spool: "/data/agents/sandbox/x/.hive/review-requests", Key: "0123456789abcdef", StartedAt: now}}, } { t.Run(tc.name, func(t *testing.T) { c := reviewTestClient(t, "http://127.0.0.1:0") diff --git a/src/pkg/github/review_request_watcher.go b/src/pkg/github/review_request_watcher.go index cf1b17c0c9..80c2693ca1 100644 --- a/src/pkg/github/review_request_watcher.go +++ b/src/pkg/github/review_request_watcher.go @@ -349,8 +349,10 @@ func (c *Client) handleOneReviewRequest(ctx context.Context, path string, src re fileUID := statUID(data, path) if src.agent != "" { // A staged spool file is owned by hive, not the agent: its owning - // UID proves nothing, and the manifest already did the proving. 0 - // is the authorizer's "no owning UID" value. + // UID proves nothing. The proving was done at ingest instead: the + // manifest names the agent, and the request's MAC under the run's + // spool key shows that agent's sandbox wrote it + // (ingestSandboxSpool). 0 is the authorizer's "no owning UID" value. fileUID = 0 } if c.reviewAuthz == nil {