From 5ce7de00e3d21048d4dc4ffcdafe91545cb425f6 Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Mon, 16 Nov 2026 16:37:42 -0500 Subject: [PATCH 1/3] =?UTF-8?q?=E2=9C=A8=20Supply=20hive's=20sandbox=20rel?= =?UTF-8?q?ays=20(hive-review)=20on=20every=20sandbox=20launch?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Job runtime: an init container from the running hive image (by digest, read from hive's own pod) copies hive-review into an emptyDir that the kick container mounts read-only at /opt/hive/relays. The init container is restricted-SCC safe: runAsNonRoot, read-only root filesystem, no privilege escalation, all capabilities dropped; the uid is left to the SCC on OpenShift and is 65534 elsewhere. job.relay_image and job.relay_run_as_user override, relay_image: none opts out. Podman: hive stages its own hive-review in a private temp dir and bind-mounts it read-only. Both wrap the kick command in a /bin/sh preflight that fails the kick (exit 78) with a clear message when bash or python3 is missing, and prepends the relay dir to PATH. Fixes #18512 Signed-off-by: Andy Anderson --- changelog.d/added-18512-sandbox-relays.md | 1 + src/crux/agent/manager_sandbox.go | 2 + src/crux/agent/sandbox_executor.go | 3 + src/crux/config/agent_config.go | 9 + src/crux/kubejob/kubejob.go | 48 ++- src/crux/kubejob/kubejob_test.go | 11 + src/crux/kubejob/relays.go | 212 ++++++++++++ src/crux/kubejob/relays_test.go | 303 ++++++++++++++++++ src/crux/sandbox/relays.go | 142 ++++++++ src/crux/sandbox/relays_test.go | 198 ++++++++++++ src/crux/sandbox/sandbox.go | 22 +- src/docs/sandbox-isolation.md | 16 +- .../dashboard/settings_parity_allowlist.txt | 4 + 13 files changed, 966 insertions(+), 5 deletions(-) create mode 100644 changelog.d/added-18512-sandbox-relays.md create mode 100644 src/crux/kubejob/relays.go create mode 100644 src/crux/kubejob/relays_test.go create mode 100644 src/crux/sandbox/relays.go create mode 100644 src/crux/sandbox/relays_test.go diff --git a/changelog.d/added-18512-sandbox-relays.md b/changelog.d/added-18512-sandbox-relays.md new file mode 100644 index 0000000000..a70e8b20fd --- /dev/null +++ b/changelog.d/added-18512-sandbox-relays.md @@ -0,0 +1 @@ +- Sandboxed agent kicks now get hive's own `hive-review` relay on `PATH`. The Job runtime copies it in from the running hive image by digest, through a restricted-SCC-safe init container. Podman bind-mounts it read-only. Project images no longer need to include hive's scripts, only bash and python3, and a kick on an image without them fails with a clear message (#18512). diff --git a/src/crux/agent/manager_sandbox.go b/src/crux/agent/manager_sandbox.go index fae687f9e1..7c6c8f66bb 100644 --- a/src/crux/agent/manager_sandbox.go +++ b/src/crux/agent/manager_sandbox.go @@ -447,6 +447,8 @@ func JobOptionsFromConfig(job config.SandboxJobConfig) kubejob.Options { SchedulerName: job.SchedulerName, EnvFromSecrets: append([]string(nil), job.EnvFromSecrets...), TTLSeconds: job.TTLSeconds, + RelayImage: job.RelayImage, + RelayRunAsUser: job.RelayRunAsUser, Resources: kubejob.Resources{ Limits: job.Resources.Limits, Requests: job.Resources.Requests, diff --git a/src/crux/agent/sandbox_executor.go b/src/crux/agent/sandbox_executor.go index 3e5671cf6b..bf445950e8 100644 --- a/src/crux/agent/sandbox_executor.go +++ b/src/crux/agent/sandbox_executor.go @@ -229,6 +229,9 @@ func (e *SandboxExecutor) Run(ctx context.Context, spec SandboxKickSpec) (Sandbo EnvAllowlist: spec.EnvAllowlist, NetworkMode: spec.NetworkMode, ClineProvider: provider, + // hive-review and the other sandbox-safe relays come from hive, not + // the project image (#18512). + Relays: true, }) if provider != nil { sres.Stdout = provider.Redact(sres.Stdout) diff --git a/src/crux/config/agent_config.go b/src/crux/config/agent_config.go index 3f54e513cb..f808cfe1a1 100644 --- a/src/crux/config/agent_config.go +++ b/src/crux/config/agent_config.go @@ -44,6 +44,15 @@ type SandboxJobConfig struct { EnvFromSecrets []string `yaml:"env_from_secrets,omitempty" json:"env_from_secrets,omitempty"` Volumes []SandboxJobVolume `yaml:"volumes,omitempty" json:"volumes,omitempty"` TTLSeconds int `yaml:"ttl_seconds,omitempty" json:"ttl_seconds,omitempty"` + // RelayImage is the image of the init container that copies hive's + // sandbox relays (hive-review) into the Job (#18512). Empty resolves the + // running hive image by digest from hive's own pod; "none" skips the + // relays entirely. + RelayImage string `yaml:"relay_image,omitempty" json:"relay_image,omitempty"` + // RelayRunAsUser pins the relay init container's uid. Unset leaves it to + // the cluster on OpenShift (the SCC assigns one from the namespace range) + // and uses 65534 elsewhere, since the hive image's default user is root. + RelayRunAsUser *int64 `yaml:"relay_run_as_user,omitempty" json:"relay_run_as_user,omitempty"` } // SandboxJobToleration mirrors the subset of a Kubernetes toleration an diff --git a/src/crux/kubejob/kubejob.go b/src/crux/kubejob/kubejob.go index bc24f16b9b..7cd2bd2d2e 100644 --- a/src/crux/kubejob/kubejob.go +++ b/src/crux/kubejob/kubejob.go @@ -135,6 +135,14 @@ type Options struct { Network string // PollInterval overrides how often the Job status is read (tests). PollInterval time.Duration + // RelayImage is the image of the init container that supplies hive's + // sandbox relays to a LaunchSpec with Relays set (#18512). Empty resolves + // the running hive image, by digest, from hive's own pod; RelayImageNone + // skips the relays. + RelayImage string + // RelayRunAsUser pins the relay init container's uid. Nil leaves it to + // the cluster on OpenShift and uses DefaultRelayRunAsUser elsewhere. + RelayRunAsUser *int64 } // network is Options.Network with the empty default applied. @@ -196,6 +204,9 @@ type Launcher struct { NamespacePath string HTTPClient *http.Client Now func() time.Time + // PodName is hive's own pod, read to resolve the relay image; empty + // falls back to $POD_NAME, then $HOSTNAME, then os.Hostname. + PodName string } var _ sandbox.Launcher = (*Launcher)(nil) @@ -214,6 +225,9 @@ func (o Options) Validate() error { return fmt.Errorf("kubejob: volumes[%d] needs both claim and mount_path", i) } } + if o.RelayRunAsUser != nil && *o.RelayRunAsUser <= 0 { + return errors.New("kubejob: relay_run_as_user must be a positive (non-root) uid") + } switch o.network() { case NetworkDefault, NetworkNone, NetworkHost: default: @@ -248,8 +262,14 @@ func (l *Launcher) Run(ctx context.Context, spec sandbox.LaunchSpec) (sandbox.Re if err != nil { return res, err } + var relays *relayPlan + if spec.Relays { + if relays, err = l.resolveRelays(ctx, ns); err != nil { + return res, err + } + } name := jobName(spec.Name, l.now()) - body, err := json.Marshal(l.manifest(ns, name, subPath, spec)) + body, err := json.Marshal(l.jobManifest(ns, name, subPath, spec, relays)) if err != nil { return res, err } @@ -493,6 +513,12 @@ func (l *Launcher) delete(ctx context.Context, jobPath string) error { // manifest renders the batch/v1 Job. Kept as plain maps rather than client-go // types: the repo has no k8s.io dependency and this is the only writer. func (l *Launcher) manifest(ns, name, subPath string, spec sandbox.LaunchSpec) map[string]any { + return l.jobManifest(ns, name, subPath, spec, nil) +} + +// jobManifest is manifest plus, when relays is non-nil, the relay init +// container, its emptyDir and the agent container's read-only mount of it. +func (l *Launcher) jobManifest(ns, name, subPath string, spec sandbox.LaunchSpec, relays *relayPlan) map[string]any { mount := spec.WorkspaceMount if mount == "" { mount = sandbox.DefaultWorkspaceMount @@ -557,8 +583,18 @@ func (l *Launcher) manifest(ns, name, subPath string, spec sandbox.LaunchSpec) m "envFrom": envFrom, "volumeMounts": mounts, } - if len(spec.Command) > 0 { - container["command"] = spec.Command + command := spec.Command + if relays != nil { + command = sandbox.RelayCommand(command) + mounts = append(mounts, map[string]any{"name": relayVolumeName, "mountPath": sandbox.RelayMountPath, "readOnly": true}) + container["volumeMounts"] = mounts + volumes = append(volumes, map[string]any{ + "name": relayVolumeName, + "emptyDir": map[string]string{"sizeLimit": relayVolumeSizeLimit}, + }) + } + if len(command) > 0 { + container["command"] = command } if len(l.Options.Resources.Limits) > 0 || len(l.Options.Resources.Requests) > 0 { container["resources"] = l.Options.Resources @@ -574,6 +610,12 @@ func (l *Launcher) manifest(ns, name, subPath string, spec sandbox.LaunchSpec) m "seccompProfile": map[string]string{"type": "RuntimeDefault"}, }, } + if relays != nil { + podSpec["initContainers"] = []map[string]any{relays.initContainer()} + if len(relays.pullSecrets) > 0 { + podSpec["imagePullSecrets"] = relays.pullSecrets + } + } if len(l.Options.NodeSelector) > 0 { podSpec["nodeSelector"] = l.Options.NodeSelector } diff --git a/src/crux/kubejob/kubejob_test.go b/src/crux/kubejob/kubejob_test.go index a7ce49835b..6ad2c70eed 100644 --- a/src/crux/kubejob/kubejob_test.go +++ b/src/crux/kubejob/kubejob_test.go @@ -43,6 +43,8 @@ type fakeAPI struct { policyErr int // when >0, the NetworkPolicy POST fails with this status policyDeleted bool order []string // "policy", "job" in creation order + selfPod string // JSON body of hive's own pod (testSelfPod), "" = 404 + selfPodGets int } func newFakeAPI() *fakeAPI { @@ -120,6 +122,14 @@ func (f *fakeAPI) handler(t *testing.T) http.HandlerFunc { f.onPodLogAfterComplete() } _, _ = w.Write([]byte(f.logBody)) + case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/pods/"+testSelfPod): + f.selfPodGets++ + if f.selfPod == "" { + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte(`{"kind":"Status","message":"pods \"` + testSelfPod + `\" not found"}`)) + return + } + _, _ = w.Write([]byte(f.selfPod)) case r.Method == http.MethodGet && strings.Contains(r.URL.Path, "/pods/"): state := `{"terminated":{"exitCode":` + itoa(f.exitCode) + `}}` if f.podStatus != "terminated" { @@ -155,6 +165,7 @@ func newLauncher(srv *httptest.Server, opts Options) *Launcher { HTTPClient: srv.Client(), TokenPath: filepath.Join("/nonexistent", "token"), Now: func() time.Time { return time.Unix(1700000000, 0) }, + PodName: testSelfPod, } } diff --git a/src/crux/kubejob/relays.go b/src/crux/kubejob/relays.go new file mode 100644 index 0000000000..0bb8ed76b1 --- /dev/null +++ b/src/crux/kubejob/relays.go @@ -0,0 +1,212 @@ +package kubejob + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "os" + "path" + "strings" + + "github.com/hivecommons/hive/crux/sandbox" +) + +// Sandbox relays on the Job runtime (#18512). +// +// The project image knows nothing about hive, so an init container from the +// running hive image (by digest, so the relay matches the hive that reads its +// spool) copies sandbox.RelayNames into an emptyDir. The kick container mounts +// that emptyDir read-only at sandbox.RelayMountPath and runs its command +// through sandbox.RelayCommand, which checks for bash and python3 and puts the +// relays first on PATH. +// +// The init container is built to pass OpenShift's restricted SCC and the +// Kubernetes "restricted" Pod Security Standard: non-root, read-only root +// filesystem, no privilege escalation, every capability dropped, RuntimeDefault +// seccomp. Its uid is left for the SCC to assign on OpenShift (a fixed uid +// outside the namespace range is rejected there); elsewhere the hive image's +// default user is root, which runAsNonRoot refuses, so it runs as +// DefaultRelayRunAsUser unless Options.RelayRunAsUser says otherwise. + +const ( + // RelayImageNone in Options.RelayImage skips the relays. + RelayImageNone = "none" + // DefaultRelayRunAsUser is the relay init container's uid off OpenShift + // (nobody). It only reads the relays and writes the emptyDir. + DefaultRelayRunAsUser int64 = 65534 + + relayVolumeName = "hive-relays" + relayInitContainerName = "hive-relays" + // relayVolumeSizeLimit bounds the emptyDir; the relays are a few KiB. + relayVolumeSizeLimit = "16Mi" + // relaySourceDir is where the hive image installs its relays (Dockerfile). + relaySourceDir = "/usr/local/bin" + // hiveContainerName is the hive container in hive's own Deployment. + hiveContainerName = "hive" + // openshiftSCCAnnotation is set by OpenShift on every pod it admits. + openshiftSCCAnnotation = "openshift.io/scc" + pullIfNotPresent = "IfNotPresent" +) + +// relayPlan is what the manifest needs to add the relay init container. +type relayPlan struct { + image string + runAsUser *int64 + pullSecrets []map[string]string +} + +func (p *relayPlan) initContainer() map[string]any { + sc := map[string]any{ + "allowPrivilegeEscalation": false, + "capabilities": map[string]any{"drop": []string{"ALL"}}, + "seccompProfile": map[string]string{"type": "RuntimeDefault"}, + "runAsNonRoot": true, + "readOnlyRootFilesystem": true, + } + if p.runAsUser != nil { + sc["runAsUser"] = *p.runAsUser + } + command := []string{"cp"} + for _, name := range sandbox.RelayNames { + command = append(command, path.Join(relaySourceDir, name)) + } + command = append(command, sandbox.RelayMountPath+"/") + return map[string]any{ + "name": relayInitContainerName, + "image": p.image, + "imagePullPolicy": pullIfNotPresent, + "command": command, + "securityContext": sc, + "volumeMounts": []map[string]any{{"name": relayVolumeName, "mountPath": sandbox.RelayMountPath}}, + } +} + +// selfPod is the part of hive's own pod the relay plan reads. +type selfPod struct { + Metadata struct { + Annotations map[string]string `json:"annotations"` + } `json:"metadata"` + Spec struct { + Containers []struct { + Name string `json:"name"` + Image string `json:"image"` + } `json:"containers"` + ImagePullSecrets []map[string]string `json:"imagePullSecrets"` + } `json:"spec"` + Status struct { + ContainerStatuses []struct { + Name string `json:"name"` + Image string `json:"image"` + ImageID string `json:"imageID"` + } `json:"containerStatuses"` + } `json:"status"` +} + +// resolveRelays builds the relay plan, or nil when relays are switched off. +// Hive's own pod is read only when the image or the uid is not configured. +func (l *Launcher) resolveRelays(ctx context.Context, ns string) (*relayPlan, error) { + image := strings.TrimSpace(l.Options.RelayImage) + if image == RelayImageNone { + return nil, nil + } + plan := &relayPlan{image: image, runAsUser: l.Options.RelayRunAsUser} + if image != "" && plan.runAsUser != nil { + return plan, nil + } + pod, err := l.selfPod(ctx, ns) + if err != nil { + return nil, fmt.Errorf("kubejob: reading hive's own pod to supply the sandbox relays (hive-review): %w; set job.relay_image and job.relay_run_as_user, or job.relay_image: %s to skip the relays", err, RelayImageNone) + } + if plan.image == "" { + if plan.image = pod.hiveImage(); plan.image == "" { + return nil, fmt.Errorf("kubejob: hive's own pod reports no image to supply the sandbox relays from; set job.relay_image, or job.relay_image: %s to skip the relays", RelayImageNone) + } + // The init container pulls the hive image, so it needs the + // credentials hive's own pod pulled it with. + plan.pullSecrets = pod.Spec.ImagePullSecrets + } + if plan.runAsUser == nil { + if _, openshift := pod.Metadata.Annotations[openshiftSCCAnnotation]; !openshift { + uid := DefaultRelayRunAsUser + plan.runAsUser = &uid + } + } + return plan, nil +} + +func (l *Launcher) selfPod(ctx context.Context, ns string) (selfPod, error) { + var pod selfPod + name, err := l.podName() + if err != nil { + return pod, err + } + raw, err := l.do(ctx, http.MethodGet, fmt.Sprintf("/api/v1/namespaces/%s/pods/%s", ns, name), nil) + if err != nil { + return pod, err + } + if err := json.Unmarshal(raw, &pod); err != nil { + return pod, fmt.Errorf("decoding pod %s: %w", name, err) + } + return pod, nil +} + +func (l *Launcher) podName() (string, error) { + for _, candidate := range []string{l.PodName, os.Getenv("POD_NAME"), os.Getenv("HOSTNAME")} { + if c := strings.TrimSpace(candidate); c != "" { + return c, nil + } + } + host, err := os.Hostname() + if err != nil || strings.TrimSpace(host) == "" { + return "", errors.New("cannot determine hive's own pod name") + } + return strings.TrimSpace(host), nil +} + +// hiveImage is the hive container's running image by digest when the runtime +// reports a pullable one, else the image as written in the status or spec. +// The container named "hive" is preferred; failing that, the first one. +func (p selfPod) hiveImage() string { + statuses := p.Status.ContainerStatuses + for i := range statuses { + if statuses[i].Name == hiveContainerName { + statuses = statuses[i : i+1] + break + } + } + if len(statuses) > 0 { + cs := statuses[0] + if ref := pullableDigest(cs.ImageID); ref != "" { + return ref + } + if img := strings.TrimSpace(cs.Image); img != "" && !strings.HasPrefix(img, "sha256:") { + return img + } + } + containers := p.Spec.Containers + for i := range containers { + if containers[i].Name == hiveContainerName { + return strings.TrimSpace(containers[i].Image) + } + } + if len(containers) > 0 { + return strings.TrimSpace(containers[0].Image) + } + return "" +} + +// pullableDigest turns a container status imageID into a pullable +// repo@sha256 reference, or "" when it is only a bare image ID. +func pullableDigest(imageID string) string { + ref := strings.TrimSpace(imageID) + if _, rest, ok := strings.Cut(ref, "://"); ok { + ref = rest + } + repo, digest, ok := strings.Cut(ref, "@") + if !ok || repo == "" || !strings.HasPrefix(digest, "sha256:") { + return "" + } + return ref +} diff --git a/src/crux/kubejob/relays_test.go b/src/crux/kubejob/relays_test.go new file mode 100644 index 0000000000..f0af8fea51 --- /dev/null +++ b/src/crux/kubejob/relays_test.go @@ -0,0 +1,303 @@ +package kubejob + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/hivecommons/hive/crux/sandbox" +) + +const ( + testSelfPod = "hive-7d9f-abcde" + testHiveDigest = "ghcr.io/hivecommons/hive@sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + vanillaSelfPod = `{"metadata":{"annotations":{}},"spec":{"containers":[{"name":"hive","image":"ghcr.io/hivecommons/hive:stable"}],"imagePullSecrets":[{"name":"ghcr-pull"}]},"status":{"containerStatuses":[{"name":"sidecar","image":"other:1","imageID":"other@sha256:aa"},{"name":"hive","image":"ghcr.io/hivecommons/hive:stable","imageID":"docker-pullable://` + testHiveDigest + `"}]}}` + openshiftSelfPod = `{"metadata":{"annotations":{"openshift.io/scc":"restricted-v2"}},"spec":{"containers":[{"name":"hive","image":"ghcr.io/hivecommons/hive:stable"}]},"status":{"containerStatuses":[{"name":"hive","image":"ghcr.io/hivecommons/hive:stable","imageID":"` + testHiveDigest + `"}]}}` +) + +func relaySpec() sandbox.LaunchSpec { + spec := specFor("/data/agents/sandbox/reviewer-1") + spec.Relays = true + return spec +} + +// runRelayJob runs one relay kick against the fake API and returns the Job +// manifest's pod spec. +func runRelayJob(t *testing.T, api *fakeAPI, opts Options) map[string]any { + t.Helper() + api.statuses = []string{statusComplete} + srv := httptest.NewServer(api.handler(t)) + defer srv.Close() + if _, err := newLauncher(srv, opts).Run(context.Background(), relaySpec()); err != nil { + t.Fatalf("Run: %v", err) + } + return podSpecOf(t, api.created) +} + +func podSpecOf(t *testing.T, job map[string]any) map[string]any { + t.Helper() + raw, _ := json.Marshal(job) + var m struct { + Spec struct { + Template struct { + Spec map[string]any `json:"spec"` + } `json:"template"` + } `json:"spec"` + } + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + return m.Spec.Template.Spec +} + +func asJSON(v any) string { raw, _ := json.Marshal(v); return string(raw) } + +func TestRelayJobSpecOnVanillaCluster(t *testing.T) { + api := newFakeAPI() + api.selfPod = vanillaSelfPod + pod := runRelayJob(t, api, Options{}) + + inits, _ := pod["initContainers"].([]any) + if len(inits) != 1 { + t.Fatalf("initContainers = %s, want one relay init container", asJSON(pod["initContainers"])) + } + initC := asJSON(inits[0]) + for _, want := range []string{ + `"name":"hive-relays"`, + `"image":"` + testHiveDigest + `"`, // the running digest, not the :stable tag + `"imagePullPolicy":"IfNotPresent"`, + `"command":["cp","/usr/local/bin/hive-review","/opt/hive/relays/"]`, + `"runAsNonRoot":true`, + `"readOnlyRootFilesystem":true`, + `"allowPrivilegeEscalation":false`, + `"capabilities":{"drop":["ALL"]}`, + `"seccompProfile":{"type":"RuntimeDefault"}`, + `"runAsUser":65534`, + `"mountPath":"/opt/hive/relays","name":"hive-relays"`, + } { + if !strings.Contains(initC, want) { + t.Errorf("relay init container missing %s\n%s", want, initC) + } + } + if strings.Contains(initC, `"privileged"`) || strings.Contains(initC, `"add"`) { + t.Errorf("relay init container must not ask for privileges: %s", initC) + } + + vols := asJSON(pod["volumes"]) + if !strings.Contains(vols, `"emptyDir":{"sizeLimit":"16Mi"},"name":"hive-relays"`) { + t.Errorf("volumes missing the relay emptyDir: %s", vols) + } + if got := asJSON(pod["imagePullSecrets"]); got != `[{"name":"ghcr-pull"}]` { + t.Errorf("imagePullSecrets = %s, want hive's own pull secret for the hive image", got) + } + + containers, _ := pod["containers"].([]any) + kick := asJSON(containers[0]) + if !strings.Contains(kick, `"mountPath":"/opt/hive/relays","name":"hive-relays","readOnly":true`) { + t.Errorf("kick container must mount the relays read-only: %s", kick) + } + var c struct { + Command []string `json:"command"` + } + _ = json.Unmarshal([]byte(kick), &c) + want := append([]string(nil), sandbox.RelayCommand(relaySpec().Command)...) + if strings.Join(c.Command, "\x00") != strings.Join(want, "\x00") { + t.Errorf("kick command = %q, want the relay preflight wrapper %q", c.Command, want) + } + if c.Command[len(c.Command)-1] != "@.hive/kick-prompt.txt" { + t.Errorf("the agent command must still run last: %q", c.Command) + } +} + +func TestRelayJobOnOpenShiftLeavesUIDToTheSCC(t *testing.T) { + api := newFakeAPI() + api.selfPod = openshiftSelfPod + pod := runRelayJob(t, api, Options{}) + initC := asJSON(pod["initContainers"]) + if strings.Contains(initC, "runAsUser") { + t.Errorf("on OpenShift the SCC assigns the uid; a fixed one is rejected: %s", initC) + } + if !strings.Contains(initC, `"runAsNonRoot":true`) || !strings.Contains(initC, `"image":"`+testHiveDigest+`"`) { + t.Errorf("relay init container = %s", initC) + } + if _, ok := pod["imagePullSecrets"]; ok { + t.Errorf("no pull secrets on hive's pod means none on the Job: %s", asJSON(pod["imagePullSecrets"])) + } +} + +func TestRelayJobConfiguredImageAndUIDSkipThePodRead(t *testing.T) { + api := newFakeAPI() + uid := int64(1000680000) + pod := runRelayJob(t, api, Options{RelayImage: "registry.internal/hive@sha256:beef", RelayRunAsUser: &uid}) + if api.selfPodGets != 0 { + t.Errorf("hive's pod was read %d times; both values were configured", api.selfPodGets) + } + initC := asJSON(pod["initContainers"]) + if !strings.Contains(initC, `"image":"registry.internal/hive@sha256:beef"`) || !strings.Contains(initC, `"runAsUser":1000680000`) { + t.Errorf("relay init container = %s, want the configured image and uid", initC) + } +} + +func TestRelayJobConfiguredImageStillDetectsOpenShift(t *testing.T) { + api := newFakeAPI() + api.selfPod = openshiftSelfPod + pod := runRelayJob(t, api, Options{RelayImage: "registry.internal/hive:v5"}) + initC := asJSON(pod["initContainers"]) + if !strings.Contains(initC, `"image":"registry.internal/hive:v5"`) || strings.Contains(initC, "runAsUser") { + t.Errorf("relay init container = %s", initC) + } + if _, ok := pod["imagePullSecrets"]; ok { + t.Error("a configured relay image does not borrow hive's pull secrets") + } +} + +func TestRelayJobNoneSkipsRelays(t *testing.T) { + api := newFakeAPI() + pod := runRelayJob(t, api, Options{RelayImage: RelayImageNone}) + if _, ok := pod["initContainers"]; ok || strings.Contains(asJSON(pod), "hive-relays") { + t.Errorf("relay_image: none must add nothing: %s", asJSON(pod)) + } + if api.selfPodGets != 0 { + t.Error("relay_image: none must not read hive's pod") + } +} + +func TestJobWithoutRelaysIsUnchanged(t *testing.T) { + api := newFakeAPI() + api.statuses = []string{statusComplete} + srv := httptest.NewServer(api.handler(t)) + defer srv.Close() + if _, err := newLauncher(srv, Options{}).Run(context.Background(), specFor("/data/agents/sandbox/deps-1")); err != nil { + t.Fatal(err) + } + if s := asJSON(api.created); strings.Contains(s, "initContainers") || strings.Contains(s, "hive-relays") { + t.Errorf("a launch without Relays must not get relays: %s", s) + } +} + +func TestRelayJobFailsClearlyWhenHivePodUnreadable(t *testing.T) { + api := newFakeAPI() // selfPod "" → 404 + api.statuses = []string{statusComplete} + srv := httptest.NewServer(api.handler(t)) + defer srv.Close() + _, err := newLauncher(srv, Options{}).Run(context.Background(), relaySpec()) + if err == nil || !strings.Contains(err.Error(), "relay_image") || !strings.Contains(err.Error(), "hive-review") { + t.Fatalf("Run error = %v, want a message naming hive-review and job.relay_image", err) + } + if api.created != nil { + t.Error("no Job may be created when the relays cannot be resolved") + } +} + +func TestRelayJobFailsWhenHivePodHasNoImage(t *testing.T) { + api := newFakeAPI() + api.selfPod = `{"metadata":{},"spec":{},"status":{}}` + api.statuses = []string{statusComplete} + srv := httptest.NewServer(api.handler(t)) + defer srv.Close() + _, err := newLauncher(srv, Options{}).Run(context.Background(), relaySpec()) + if err == nil || !strings.Contains(err.Error(), "no image") { + t.Fatalf("Run error = %v, want the no-image message", err) + } +} + +func TestRelayJobBadPodJSON(t *testing.T) { + api := newFakeAPI() + api.selfPod = `{not json` + api.statuses = []string{statusComplete} + srv := httptest.NewServer(api.handler(t)) + defer srv.Close() + _, err := newLauncher(srv, Options{}).Run(context.Background(), relaySpec()) + if err == nil || !strings.Contains(err.Error(), "decoding pod") { + t.Fatalf("Run error = %v, want a decode error", err) + } +} + +func TestValidateRejectsRootRelayUID(t *testing.T) { + for _, uid := range []int64{0, -1} { + u := uid + if err := (Options{WorkspaceClaim: "c", RelayRunAsUser: &u}).Validate(); err == nil || !strings.Contains(err.Error(), "relay_run_as_user") { + t.Errorf("uid %d: Validate = %v, want a relay_run_as_user error", uid, err) + } + } +} + +func TestHiveImageSelection(t *testing.T) { + cases := []struct { + name, pod, want string + }{ + {"digest from imageID", vanillaSelfPod, testHiveDigest}, + {"bare image ID falls back to status image", `{"status":{"containerStatuses":[{"name":"hive","image":"ghcr.io/x:v1","imageID":"sha256:abc"}]}}`, "ghcr.io/x:v1"}, + {"status image that is only an ID falls back to spec", `{"spec":{"containers":[{"name":"hive","image":"ghcr.io/x:spec"}]},"status":{"containerStatuses":[{"name":"hive","image":"sha256:abc","imageID":""}]}}`, "ghcr.io/x:spec"}, + {"no hive container: first status", `{"status":{"containerStatuses":[{"name":"main","imageID":"containerd://reg/x@sha256:ff"}]}}`, "reg/x@sha256:ff"}, + {"no status: first spec container", `{"spec":{"containers":[{"name":"main","image":"reg/x:1"}]}}`, "reg/x:1"}, + {"nothing", `{}`, ""}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + var p selfPod + if err := json.Unmarshal([]byte(tc.pod), &p); err != nil { + t.Fatal(err) + } + if got := p.hiveImage(); got != tc.want { + t.Errorf("hiveImage = %q, want %q", got, tc.want) + } + }) + } +} + +func TestPullableDigest(t *testing.T) { + for in, want := range map[string]string{ + "docker-pullable://reg/x@sha256:ab": "reg/x@sha256:ab", + "reg/x@sha256:ab": "reg/x@sha256:ab", + "sha256:ab": "", + "@sha256:ab": "", + "reg/x@md5:ab": "", + "": "", + } { + if got := pullableDigest(in); got != want { + t.Errorf("pullableDigest(%q) = %q, want %q", in, got, want) + } + } +} + +func TestPodNameFallbacks(t *testing.T) { + t.Setenv("POD_NAME", "") + t.Setenv("HOSTNAME", "") + if got, err := (&Launcher{PodName: "explicit"}).podName(); err != nil || got != "explicit" { + t.Errorf("podName = %q, %v", got, err) + } + t.Setenv("HOSTNAME", "from-hostname") + if got, _ := (&Launcher{}).podName(); got != "from-hostname" { + t.Errorf("podName = %q, want $HOSTNAME", got) + } + t.Setenv("POD_NAME", "from-downward-api") + if got, _ := (&Launcher{}).podName(); got != "from-downward-api" { + t.Errorf("podName = %q, want $POD_NAME first", got) + } + t.Setenv("POD_NAME", "") + t.Setenv("HOSTNAME", "") + if got, err := (&Launcher{}).podName(); err != nil || got == "" { + t.Errorf("podName = %q, %v, want os.Hostname", got, err) + } +} + +func TestRelayJobPodReadUsesNamespaceAndName(t *testing.T) { + var seen string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + seen = r.URL.Path + _, _ = w.Write([]byte(vanillaSelfPod)) + })) + defer srv.Close() + l := &Launcher{APIServer: srv.URL, HTTPClient: srv.Client(), PodName: "hive-0", TokenPath: "/nonexistent"} + plan, err := l.resolveRelays(context.Background(), "tenant") + if err != nil || plan == nil { + t.Fatalf("resolveRelays = %v, %v", plan, err) + } + if seen != "/api/v1/namespaces/tenant/pods/hive-0" { + t.Errorf("read %s, want hive's own pod in its namespace", seen) + } +} diff --git a/src/crux/sandbox/relays.go b/src/crux/sandbox/relays.go new file mode 100644 index 0000000000..d0f120348b --- /dev/null +++ b/src/crux/sandbox/relays.go @@ -0,0 +1,142 @@ +package sandbox + +import ( + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strings" +) + +// Sandbox relays (#18512). +// +// A sandboxed kick runs in the project's own image, which knows nothing about +// hive. The relays an agent needs to hand work back to hive (today only +// hive-review, which writes a signed request into the kick's review spool, +// #14986) are therefore supplied by hive itself on every launch rather than +// baked into the project image: the launcher puts hive's own copy in a +// read-only directory at RelayMountPath and prepends that directory to PATH. +// Hive's copy always wins over one the image carries, because the relay and +// the spool protocol it speaks ship together in the hive image. +// +// Only relays that work from a sandbox belong in RelayNames. A relay that +// needs the pod-local drop boxes (/var/run/hive-metrics/...) cannot reach them +// from a sandbox and stays out until it has a spool of its own. + +const ( + // RelayMountPath is where the relays appear inside the sandbox. + RelayMountPath = "/opt/hive/relays" + // RelaySourceDirEnv, when set, names the directory hive reads its relays + // from on the host (Podman). Unset, each relay is looked up on hive's own + // PATH, which in the hive image resolves to /usr/local/bin. + RelaySourceDirEnv = "HIVE_SANDBOX_RELAY_DIR" + // RelayPreflightExitCode is the sandbox exit code when the image lacks a + // tool the relays need (EX_CONFIG): the kick fails with the preflight's + // message in its transcript instead of an agent that cannot post. + RelayPreflightExitCode = 78 + // relayPreflightName is $0 of the preflight shell, so a failure in the + // transcript says where it came from. + relayPreflightName = "hive-relay-preflight" + relayShell = "/bin/sh" + relayDirMode = 0o755 + relayFileMode = 0o555 +) + +// RelayNames are the sandbox-safe relays hive supplies to every sandbox. +var RelayNames = []string{"hive-review"} + +// RelayRequirements are the interpreters the relays need inside the image. +var RelayRequirements = []string{"bash", "python3"} + +// relayPreflightScript checks the image for every RelayRequirements tool, +// prepends the relay directory ($1) to PATH and execs the kick command. +var relayPreflightScript = buildRelayPreflightScript() + +func buildRelayPreflightScript() string { + return fmt.Sprintf(`relay_dir="$1"; shift +for tool in %s; do + if ! command -v "$tool" >/dev/null 2>&1; then + printf 'hive: this sandbox image has no %%s on PATH. Hive supplies its sandbox relays (%s) to every sandboxed kick, and they need %s; add %%s to the image.\n' "$tool" "$tool" >&2 + exit %d + fi +done +PATH="$relay_dir:$PATH"; export PATH +exec "$@"`, strings.Join(RelayRequirements, " "), strings.Join(RelayNames, ", "), strings.Join(RelayRequirements, " and "), RelayPreflightExitCode) +} + +// RelayCommand wraps a kick command so it runs with the relays first on PATH +// after a preflight that fails the kick, with a clear message, when the image +// lacks bash or python3. An empty command (the image's own entrypoint or CMD) +// cannot be wrapped and is returned unchanged; the relays are still mounted. +func RelayCommand(command []string) []string { + if len(command) == 0 { + return command + } + out := make([]string, 0, len(command)+4) + out = append(out, relayShell, "-c", relayPreflightScript, relayPreflightName, RelayMountPath) + return append(out, command...) +} + +// relayLookPath resolves a relay on hive's own PATH; a variable for tests. +var relayLookPath = exec.LookPath + +// relaySource returns the host path of one relay. +func relaySource(name string) (string, error) { + if dir := strings.TrimSpace(os.Getenv(RelaySourceDirEnv)); dir != "" { + return filepath.Join(dir, name), nil + } + return relayLookPath(name) +} + +// StageRelays copies hive's relays into a fresh private directory for a +// Podman bind mount and returns it with a cleanup function. Copying, rather +// than mounting hive's own /usr/local/bin entries, lets Podman relabel the +// mount (:Z) without touching the files hive itself runs. +func StageRelays() (string, func(), error) { + dir, err := os.MkdirTemp("", "hive-relays-") + if err != nil { + return "", func() {}, fmt.Errorf("sandbox: staging relays: %w", err) + } + cleanup := func() { _ = os.RemoveAll(dir) } + if err := os.Chmod(dir, relayDirMode); err != nil { + cleanup() + return "", func() {}, fmt.Errorf("sandbox: staging relays: %w", err) + } + for _, name := range RelayNames { + src, err := relaySource(name) + if err == nil { + err = copyRelay(src, filepath.Join(dir, name)) + } + if err != nil { + cleanup() + return "", func() {}, fmt.Errorf("sandbox: hive's %s relay is not available to mount into the sandbox (set %s to the directory holding it): %w", name, RelaySourceDirEnv, err) + } + } + return dir, cleanup, nil +} + +func copyRelay(src, dst string) error { + in, err := os.Open(src) + if err != nil { + return err + } + defer func() { _ = in.Close() }() + info, err := in.Stat() + if err != nil { + return err + } + if !info.Mode().IsRegular() { + return errors.New(src + " is not a regular file") + } + out, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_EXCL, relayFileMode) + if err != nil { + return err + } + if _, err := io.Copy(out, in); err != nil { + _ = out.Close() + return err + } + return out.Close() +} diff --git a/src/crux/sandbox/relays_test.go b/src/crux/sandbox/relays_test.go new file mode 100644 index 0000000000..bdf7954e35 --- /dev/null +++ b/src/crux/sandbox/relays_test.go @@ -0,0 +1,198 @@ +package sandbox + +import ( + "context" + "errors" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" +) + +func writeRelaySource(t *testing.T) string { + t.Helper() + dir := t.TempDir() + for _, name := range RelayNames { + if err := os.WriteFile(filepath.Join(dir, name), []byte("#!/bin/bash\necho relay\n"), 0o755); err != nil { + t.Fatal(err) + } + } + return dir +} + +func TestRelayNamesAreSandboxSafeOnly(t *testing.T) { + // hive-review is the only relay with a sandbox spool today (#14986); a + // relay that needs the pod-local drop boxes must not be added here. + if strings.Join(RelayNames, ",") != "hive-review" { + t.Errorf("RelayNames = %v, want only hive-review", RelayNames) + } +} + +func TestRelayCommandWrapsWithPreflightAndPath(t *testing.T) { + cmd := []string{"claude", "-p", "@.hive/kick-prompt.txt"} + got := RelayCommand(cmd) + if got[0] != "/bin/sh" || got[1] != "-c" || got[3] != relayPreflightName || got[4] != RelayMountPath { + t.Fatalf("RelayCommand = %q", got) + } + if strings.Join(got[5:], " ") != strings.Join(cmd, " ") { + t.Errorf("the kick command must follow the wrapper unchanged: %q", got[5:]) + } + script := got[2] + for _, want := range []string{"python3", "bash", `PATH="$relay_dir:$PATH"`, `exec "$@"`, "exit 78", "hive-review"} { + if !strings.Contains(script, want) { + t.Errorf("preflight script missing %q:\n%s", want, script) + } + } + if RelayCommand(nil) != nil { + t.Error("an empty command (image entrypoint) cannot be wrapped and must stay empty") + } +} + +// TestRelayPreflightRunsAndFails runs the real wrapper under /bin/sh. +func TestRelayPreflightRunsAndFails(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("POSIX shell only") + } + relayDir := writeRelaySource(t) + bin := t.TempDir() + // Only the tools the test needs: sh's builtins plus fake bash/python3. + for _, tool := range RelayRequirements { + if err := os.WriteFile(filepath.Join(bin, tool), []byte("#!/bin/sh\nexit 0\n"), 0o755); err != nil { + t.Fatal(err) + } + } + run := func(path string, command ...string) (string, int) { + args := RelayCommand(command) + args[4] = relayDir + c := exec.Command(args[0], args[1:]...) + c.Env = []string{"PATH=" + path} + out, err := c.CombinedOutput() + code := 0 + var exitErr *exec.ExitError + if errors.As(err, &exitErr) { + code = exitErr.ExitCode() + } else if err != nil { + t.Fatal(err) + } + return string(out), code + } + out, code := run(bin, "/bin/sh", "-c", `printf '%s' "$PATH"`) + if code != 0 || !strings.HasPrefix(out, relayDir+":") { + t.Errorf("with bash and python3 present: exit %d, PATH %q, want the relay dir first", code, out) + } + if err := os.Remove(filepath.Join(bin, "python3")); err != nil { + t.Fatal(err) + } + out, code = run(bin, "/bin/sh", "-c", "echo should-not-run") + if code != RelayPreflightExitCode || !strings.Contains(out, "no python3 on PATH") || strings.Contains(out, "should-not-run") { + t.Errorf("without python3: exit %d, output %q, want exit %d and a clear message", code, out, RelayPreflightExitCode) + } +} + +func TestPodmanArgsMountsRelaysReadOnly(t *testing.T) { + args, err := PodmanArgs(LaunchSpec{Image: "img", Workspace: "/w", Command: []string{"agent"}, RelayHostDir: "/tmp/hive-relays-1/"}) + if err != nil { + t.Fatal(err) + } + joined := strings.Join(args, "\x00") + if !strings.Contains(joined, "-v\x00/tmp/hive-relays-1:"+RelayMountPath+":ro,Z") { + t.Errorf("args %q missing the read-only relay mount", args) + } + tail := args[len(args)-7:] + if tail[0] != "img" || tail[1] != "/bin/sh" || tail[len(tail)-1] != "agent" { + t.Errorf("image and wrapped command = %q", tail) + } + plain, _ := PodmanArgs(LaunchSpec{Image: "img", Workspace: "/w", Command: []string{"agent"}}) + if strings.Contains(strings.Join(plain, " "), RelayMountPath) || plain[len(plain)-1] != "agent" || plain[len(plain)-2] != "img" { + t.Errorf("without a relay dir nothing changes: %q", plain) + } +} + +func TestStageRelaysCopiesFromEnvDir(t *testing.T) { + t.Setenv(RelaySourceDirEnv, writeRelaySource(t)) + dir, cleanup, err := StageRelays() + if err != nil { + t.Fatal(err) + } + info, err := os.Stat(filepath.Join(dir, "hive-review")) + if err != nil || info.Mode().Perm() != relayFileMode { + t.Fatalf("staged relay = %v, %v, want mode %o", info, err, relayFileMode) + } + cleanup() + if _, err := os.Stat(dir); !os.IsNotExist(err) { + t.Errorf("cleanup left %s behind: %v", dir, err) + } +} + +func TestStageRelaysUsesPathLookup(t *testing.T) { + t.Setenv(RelaySourceDirEnv, "") + src := writeRelaySource(t) + orig := relayLookPath + t.Cleanup(func() { relayLookPath = orig }) + relayLookPath = func(name string) (string, error) { return filepath.Join(src, name), nil } + _, cleanup, err := StageRelays() + if err != nil { + t.Fatal(err) + } + cleanup() + + relayLookPath = func(string) (string, error) { return "", exec.ErrNotFound } + if _, _, err := StageRelays(); err == nil || !strings.Contains(err.Error(), "hive-review") || !strings.Contains(err.Error(), RelaySourceDirEnv) { + t.Errorf("StageRelays = %v, want a clear missing-relay error", err) + } +} + +func TestStageRelaysRejectsNonRegularSource(t *testing.T) { + src := t.TempDir() + if err := os.Mkdir(filepath.Join(src, "hive-review"), 0o755); err != nil { + t.Fatal(err) + } + t.Setenv(RelaySourceDirEnv, src) + if _, _, err := StageRelays(); err == nil || !strings.Contains(err.Error(), "not a regular file") { + t.Errorf("StageRelays = %v, want a not-a-regular-file error", err) + } + t.Setenv(RelaySourceDirEnv, filepath.Join(src, "missing")) + if _, _, err := StageRelays(); err == nil { + t.Error("a missing relay dir must fail staging") + } +} + +func TestPodmanRunStagesAndMountsRelays(t *testing.T) { + t.Setenv(RelaySourceDirEnv, writeRelaySource(t)) + argsFile := filepath.Join(t.TempDir(), "args") + bin := fakePodman(t, `for a in "$@"; do printf '%s\n' "$a"; done > `+argsFile) + if _, err := (PodmanLauncher{Binary: bin}).Run(context.Background(), LaunchSpec{ + Image: "agent:latest", Workspace: t.TempDir(), Command: []string{"true"}, Relays: true, + }); err != nil { + t.Fatalf("Run: %v", err) + } + raw, err := os.ReadFile(argsFile) + if err != nil { + t.Fatal(err) + } + var mount string + for _, line := range strings.Split(string(raw), "\n") { + if strings.HasSuffix(line, ":"+RelayMountPath+":ro,Z") { + mount = strings.TrimSuffix(line, ":"+RelayMountPath+":ro,Z") + } + } + if mount == "" { + t.Fatalf("podman args carry no relay mount:\n%s", raw) + } + if _, err := os.Stat(mount); !os.IsNotExist(err) { + t.Errorf("the staged relay dir %s must be removed after the run: %v", mount, err) + } +} + +func TestPodmanRunFailsWhenRelaysCannotBeStaged(t *testing.T) { + t.Setenv(RelaySourceDirEnv, filepath.Join(t.TempDir(), "missing")) + bin := fakePodman(t, `exit 0`) + _, err := (PodmanLauncher{Binary: bin}).Run(context.Background(), LaunchSpec{ + Image: "agent:latest", Workspace: t.TempDir(), Command: []string{"true"}, Relays: true, + }) + if err == nil || !strings.Contains(err.Error(), "hive-review") { + t.Fatalf("Run = %v, want the missing-relay error", err) + } +} diff --git a/src/crux/sandbox/sandbox.go b/src/crux/sandbox/sandbox.go index 8d720d530c..68a2330361 100644 --- a/src/crux/sandbox/sandbox.go +++ b/src/crux/sandbox/sandbox.go @@ -41,6 +41,13 @@ type LaunchSpec struct { NetworkNone bool ReadOnly bool ClineProvider *cline.ProviderFile + // Relays asks the launcher to supply hive's sandbox relays (RelayNames) + // on PATH inside the sandbox (#18512). Set for agent kicks only. + Relays bool + // RelayHostDir is the host directory the Podman launcher bind-mounts + // read-only at RelayMountPath. PodmanLauncher.Run stages it when Relays + // is set; PodmanArgs mounts it and wraps the command when it is non-empty. + RelayHostDir string } type Result struct { @@ -73,6 +80,14 @@ func (l PodmanLauncher) Run(ctx context.Context, spec LaunchSpec) (res Result, r if _, err := exec.LookPath(bin); err != nil { return Result{}, fmt.Errorf("sandbox: podman is required but was not found in PATH: %w", err) } + if spec.Relays && spec.RelayHostDir == "" { + dir, cleanup, err := StageRelays() + if err != nil { + return Result{}, err + } + defer cleanup() + spec.RelayHostDir = dir + } args, err := PodmanArgs(spec) if err != nil { return Result{}, err @@ -146,6 +161,11 @@ func PodmanArgs(spec LaunchSpec) ([]string, error) { } args = append(args, extra...) } + command := spec.Command + if spec.RelayHostDir != "" { + args = append(args, "-v", filepath.Clean(spec.RelayHostDir)+":"+RelayMountPath+":ro,Z") + command = RelayCommand(command) + } args = append(args, "--workdir", workdir) // Name only: podman takes the value from its own environment, which Run // sets from the same spec.Env. A value on the command line would be @@ -157,7 +177,7 @@ func PodmanArgs(spec LaunchSpec) ([]string, error) { } } args = append(args, spec.Image) - args = append(args, spec.Command...) + args = append(args, command...) return args, nil } diff --git a/src/docs/sandbox-isolation.md b/src/docs/sandbox-isolation.md index 17cdbca242..73e1d96a2b 100644 --- a/src/docs/sandbox-isolation.md +++ b/src/docs/sandbox-isolation.md @@ -217,7 +217,7 @@ agents: sandbox: enabled: true runtime: job - image: registry.example/project-dev:latest # must include hive-review + image: registry.example/project-dev:latest # needs bash and python3; hive supplies hive-review job: node_selector: { accelerator: spyre } resources: @@ -238,6 +238,20 @@ The relay touches the spool only through an `os.Root` opened at the workspace. A Out of scope: the quality and deps lanes are unchanged, and nothing inside a Job writes to GitHub directly. +### Hive's relays inside the sandbox + +The project image does not need to carry `hive-review` or any other hive script ([#18512](https://github.com/hivecommons/hive/issues/18512)). On every sandboxed agent kick, hive supplies its **sandbox-safe relays** itself, read-only at `/opt/hive/relays`, and runs the kick command through a short `/bin/sh` preflight that prepends that directory to `PATH`. Today the only relay is `hive-review`, which works from a sandbox through the review spool above. Relays that need the pod-local drop boxes (`hive-open-pr`, `hive-open-issue`, `hive-merge`, `hive-push-branch`) are left out until they have a spool of their own. + +- **Job runtime.** The Job gets an init container, `hive-relays`, from the **running hive image by digest**. Hive reads its own pod (`$POD_NAME`, else `$HOSTNAME`) through the `pods` get grant the launcher already has, and takes the hive container's `imageID`. When the runtime reports no pullable digest, hive falls back to the image as written. Hive's own `imagePullSecrets` are copied onto the Job so the node can pull it. The init container runs `cp /usr/local/bin/hive-review /opt/hive/relays/` into a 16Mi `emptyDir`, and the kick container mounts that `emptyDir` read-only. The init container passes OpenShift's restricted SCC and the Kubernetes `restricted` Pod Security Standard: `runAsNonRoot`, `readOnlyRootFilesystem`, `allowPrivilegeEscalation: false`, every capability dropped, `RuntimeDefault` seccomp. On OpenShift (hive's pod carries the `openshift.io/scc` annotation) it sets no uid, and the SCC assigns one from the namespace range. Elsewhere the hive image's default user is root, which `runAsNonRoot` refuses, so it runs as uid 65534. Two `job` fields override this: + - `relay_image` pins the init container's image, for example a mirror on an internal registry. `none` turns the relays off. + - `relay_run_as_user` pins its uid. It must be positive. + + The first kick on a node pulls the hive image, which is large. Later kicks reuse it (`imagePullPolicy: IfNotPresent`). If hive cannot read its own pod, the kick fails with a message that names both fields, and no Job is created. +- **Podman.** Hive copies its own `hive-review` (found on its `PATH`, or in `$HIVE_SANDBOX_RELAY_DIR` when that is set) into a private temporary directory. It bind-mounts that directory with `-v :/opt/hive/relays:ro,Z` and removes it after the run. If the relay cannot be found, the kick fails with a message that says so. +- **Preflight.** The relays need `bash` and `python3` in the image. If either is missing, the kick exits with code 78 before the agent starts, and its transcript reads `hive: this sandbox image has no python3 on PATH ... add python3 to the image.` The image also needs `/bin/sh`. A kick that relies on the image's own entrypoint (no command) cannot be wrapped: the relays are mounted but not put on `PATH`. +- **Precedence.** Hive's copy always comes first on `PATH`, and a copy in the image is ignored. The relay and the spool protocol it speaks ship together in the hive image, so an older or newer copy baked into a project image could sign or shape a request that this hive rejects. There is deliberately no "use the image's copy if newer" option. +- **OpenShell and Substrate.** Not yet. On these runtimes the gateway or the actor API builds the sandbox pod, and getting hive's image into it needs runtime-specific work: an init container in the OpenShell template, and Landlock read access to `/opt/hive/relays`. Until that follow-up lands, the image still has to include `hive-review` on these runtimes. + ## Running sandboxed kicks in an OpenShell sandbox `sandbox.runtime: openshell` runs an agent's sandboxed kicks in an [NVIDIA OpenShell](https://github.com/NVIDIA/OpenShell) sandbox instead of a Podman container or a plain Kubernetes Job ([ADR-0026](adr/0026-sandbox-runtime-selection.md), epic [#17642](https://github.com/hivecommons/hive/issues/17642)). OpenShell adds what the other two runtimes lack: the workload runs with no capabilities under Landlock and seccomp mediation, and **egress is default-deny** — every outbound connection goes through the gateway's proxy and is checked against a per-binary policy. Phase 1 is opt-in; nothing changes for a hive that does not select it. diff --git a/src/pkg/dashboard/settings_parity_allowlist.txt b/src/pkg/dashboard/settings_parity_allowlist.txt index 148b747fc2..226d658300 100644 --- a/src/pkg/dashboard/settings_parity_allowlist.txt +++ b/src/pkg/dashboard/settings_parity_allowlist.txt @@ -3,6 +3,8 @@ # TODO #11854 agent_sandbox.job.env_from_secrets[] # TODO #11854 agent_sandbox.job.node_selector. # TODO #11854 +agent_sandbox.job.relay_image # TODO #11854 +agent_sandbox.job.relay_run_as_user # TODO #11854 agent_sandbox.job.resources.limits. # TODO #11854 agent_sandbox.job.resources.requests. # TODO #11854 agent_sandbox.job.tolerations[].effect # TODO #11854 @@ -115,6 +117,8 @@ agents..sandbox.env_allowlist[] # TODO #11854 agents..sandbox.image # TODO #11854 agents..sandbox.job.env_from_secrets[] # TODO #11854 agents..sandbox.job.node_selector. # TODO #11854 +agents..sandbox.job.relay_image # TODO #11854 +agents..sandbox.job.relay_run_as_user # TODO #11854 agents..sandbox.job.resources.limits. # TODO #11854 agents..sandbox.job.resources.requests. # TODO #11854 agents..sandbox.job.scheduler_name # TODO #11854 From 464ec852bc5d983162b9efc2996eac0be05ba394 Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Mon, 16 Nov 2026 16:50:35 -0500 Subject: [PATCH 2/3] =?UTF-8?q?=F0=9F=8C=B1=20Split=20kubejob's=20API=20cl?= =?UTF-8?q?ient=20helpers=20into=20client.go=20(file-size=20ratchet)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Andy Anderson --- src/crux/kubejob/client.go | 108 ++++++++++++++++++++++++++++++++++++ src/crux/kubejob/kubejob.go | 96 -------------------------------- 2 files changed, 108 insertions(+), 96 deletions(-) create mode 100644 src/crux/kubejob/client.go diff --git a/src/crux/kubejob/client.go b/src/crux/kubejob/client.go new file mode 100644 index 0000000000..c629878fc8 --- /dev/null +++ b/src/crux/kubejob/client.go @@ -0,0 +1,108 @@ +package kubejob + +import ( + "bytes" + "context" + "crypto/tls" + "crypto/x509" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "os" + "strings" + + "github.com/hivecommons/hive/crux/egressmark" +) + +// do performs one authenticated API call and returns the response body. +// Non-2xx responses are errors carrying the status and the API's message. +func (l *Launcher) do(ctx context.Context, method, path string, body []byte) ([]byte, error) { + client, err := l.client() + if err != nil { + return nil, err + } + server := l.APIServer + if server == "" { + server = DefaultAPIServer + } + reqCtx, cancel := context.WithTimeout(ctx, requestTimeout) + defer cancel() + var rdr io.Reader + if body != nil { + rdr = bytes.NewReader(body) + } + req, err := http.NewRequestWithContext(reqCtx, method, server+path, rdr) + if err != nil { + return nil, err + } + if tok := l.token(); tok != "" { + req.Header.Set("Authorization", "Bearer "+tok) + } + if body != nil { + req.Header.Set("Content-Type", "application/json") + } + resp, err := client.Do(req) + if err != nil { + return nil, err + } + defer func() { _ = resp.Body.Close() }() + out, err := io.ReadAll(io.LimitReader(resp.Body, maxLogBytes+1)) + if err != nil { + return nil, err + } + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return nil, fmt.Errorf("%s %s: HTTP %d: %s", method, path, resp.StatusCode, apiMessage(out)) + } + return out, nil +} + +// apiMessage pulls the human message out of a Kubernetes Status body, falling +// back to a trimmed raw body. +func apiMessage(raw []byte) string { + var st struct { + Message string `json:"message"` + } + if json.Unmarshal(raw, &st) == nil && st.Message != "" { + return st.Message + } + s := strings.TrimSpace(string(raw)) + if len(s) > 200 { + s = s[:200] + } + return s +} + +func (l *Launcher) token() string { + path := l.TokenPath + if path == "" { + path = DefaultTokenPath + } + raw, err := os.ReadFile(path) + if err != nil { + return "" + } + return strings.TrimSpace(string(raw)) +} + +// client returns the injected HTTP client, or one trusting the in-cluster CA. +func (l *Launcher) client() (*http.Client, error) { + if l.HTTPClient != nil { + return l.HTTPClient, nil + } + path := l.CAPath + if path == "" { + path = DefaultCAPath + } + pem, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("kubejob: reading in-cluster CA: %w", err) + } + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(pem) { + return nil, errors.New("kubejob: in-cluster CA bundle contained no certificates") + } + l.HTTPClient = &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12}, DialContext: egressmark.Dialer(0).DialContext}} + return l.HTTPClient, nil +} diff --git a/src/crux/kubejob/kubejob.go b/src/crux/kubejob/kubejob.go index 7cd2bd2d2e..5f742ce7ed 100644 --- a/src/crux/kubejob/kubejob.go +++ b/src/crux/kubejob/kubejob.go @@ -26,14 +26,10 @@ package kubejob import ( - "bytes" "context" - "crypto/tls" - "crypto/x509" "encoding/json" "errors" "fmt" - "io" "net/http" "os" "path/filepath" @@ -41,7 +37,6 @@ import ( "strings" "time" - "github.com/hivecommons/hive/crux/egressmark" "github.com/hivecommons/hive/crux/sandbox" ) @@ -734,94 +729,3 @@ func (l *Launcher) namespace() (string, error) { } return ns, nil } - -// do performs one authenticated API call and returns the response body. -// Non-2xx responses are errors carrying the status and the API's message. -func (l *Launcher) do(ctx context.Context, method, path string, body []byte) ([]byte, error) { - client, err := l.client() - if err != nil { - return nil, err - } - server := l.APIServer - if server == "" { - server = DefaultAPIServer - } - reqCtx, cancel := context.WithTimeout(ctx, requestTimeout) - defer cancel() - var rdr io.Reader - if body != nil { - rdr = bytes.NewReader(body) - } - req, err := http.NewRequestWithContext(reqCtx, method, server+path, rdr) - if err != nil { - return nil, err - } - if tok := l.token(); tok != "" { - req.Header.Set("Authorization", "Bearer "+tok) - } - if body != nil { - req.Header.Set("Content-Type", "application/json") - } - resp, err := client.Do(req) - if err != nil { - return nil, err - } - defer func() { _ = resp.Body.Close() }() - out, err := io.ReadAll(io.LimitReader(resp.Body, maxLogBytes+1)) - if err != nil { - return nil, err - } - if resp.StatusCode < 200 || resp.StatusCode >= 300 { - return nil, fmt.Errorf("%s %s: HTTP %d: %s", method, path, resp.StatusCode, apiMessage(out)) - } - return out, nil -} - -// apiMessage pulls the human message out of a Kubernetes Status body, falling -// back to a trimmed raw body. -func apiMessage(raw []byte) string { - var st struct { - Message string `json:"message"` - } - if json.Unmarshal(raw, &st) == nil && st.Message != "" { - return st.Message - } - s := strings.TrimSpace(string(raw)) - if len(s) > 200 { - s = s[:200] - } - return s -} - -func (l *Launcher) token() string { - path := l.TokenPath - if path == "" { - path = DefaultTokenPath - } - raw, err := os.ReadFile(path) - if err != nil { - return "" - } - return strings.TrimSpace(string(raw)) -} - -// client returns the injected HTTP client, or one trusting the in-cluster CA. -func (l *Launcher) client() (*http.Client, error) { - if l.HTTPClient != nil { - return l.HTTPClient, nil - } - path := l.CAPath - if path == "" { - path = DefaultCAPath - } - pem, err := os.ReadFile(path) - if err != nil { - return nil, fmt.Errorf("kubejob: reading in-cluster CA: %w", err) - } - pool := x509.NewCertPool() - if !pool.AppendCertsFromPEM(pem) { - return nil, errors.New("kubejob: in-cluster CA bundle contained no certificates") - } - l.HTTPClient = &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12}, DialContext: egressmark.Dialer(0).DialContext}} - return l.HTTPClient, nil -} From 584b9ca79594ca7f106953288d011c273ca3f56f Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Mon, 16 Nov 2026 17:08:15 -0500 Subject: [PATCH 3/3] =?UTF-8?q?=F0=9F=8C=B1=20Simplify=20Podman=20relay=20?= =?UTF-8?q?staging=20and=20cover=20its=20temp-dir=20failure?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Andy Anderson --- src/crux/sandbox/relays.go | 27 ++------------------------- src/crux/sandbox/relays_test.go | 14 ++++++++++++-- 2 files changed, 14 insertions(+), 27 deletions(-) diff --git a/src/crux/sandbox/relays.go b/src/crux/sandbox/relays.go index d0f120348b..ad7064f179 100644 --- a/src/crux/sandbox/relays.go +++ b/src/crux/sandbox/relays.go @@ -1,9 +1,7 @@ package sandbox import ( - "errors" "fmt" - "io" "os" "os/exec" "path/filepath" @@ -40,7 +38,6 @@ const ( // transcript says where it came from. relayPreflightName = "hive-relay-preflight" relayShell = "/bin/sh" - relayDirMode = 0o755 relayFileMode = 0o555 ) @@ -100,10 +97,6 @@ func StageRelays() (string, func(), error) { return "", func() {}, fmt.Errorf("sandbox: staging relays: %w", err) } cleanup := func() { _ = os.RemoveAll(dir) } - if err := os.Chmod(dir, relayDirMode); err != nil { - cleanup() - return "", func() {}, fmt.Errorf("sandbox: staging relays: %w", err) - } for _, name := range RelayNames { src, err := relaySource(name) if err == nil { @@ -118,25 +111,9 @@ func StageRelays() (string, func(), error) { } func copyRelay(src, dst string) error { - in, err := os.Open(src) - if err != nil { - return err - } - defer func() { _ = in.Close() }() - info, err := in.Stat() + data, err := os.ReadFile(src) if err != nil { return err } - if !info.Mode().IsRegular() { - return errors.New(src + " is not a regular file") - } - out, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_EXCL, relayFileMode) - if err != nil { - return err - } - if _, err := io.Copy(out, in); err != nil { - _ = out.Close() - return err - } - return out.Close() + return os.WriteFile(dst, data, relayFileMode) } diff --git a/src/crux/sandbox/relays_test.go b/src/crux/sandbox/relays_test.go index bdf7954e35..1bfba83bfa 100644 --- a/src/crux/sandbox/relays_test.go +++ b/src/crux/sandbox/relays_test.go @@ -144,14 +144,24 @@ func TestStageRelaysUsesPathLookup(t *testing.T) { } } +func TestStageRelaysFailsWithoutATempDir(t *testing.T) { + t.Setenv(RelaySourceDirEnv, writeRelaySource(t)) + t.Setenv("TMPDIR", filepath.Join(t.TempDir(), "missing")) + if _, cleanup, err := StageRelays(); err == nil || !strings.Contains(err.Error(), "staging relays") { + t.Errorf("StageRelays = %v, want a staging error", err) + } else { + cleanup() + } +} + func TestStageRelaysRejectsNonRegularSource(t *testing.T) { src := t.TempDir() if err := os.Mkdir(filepath.Join(src, "hive-review"), 0o755); err != nil { t.Fatal(err) } t.Setenv(RelaySourceDirEnv, src) - if _, _, err := StageRelays(); err == nil || !strings.Contains(err.Error(), "not a regular file") { - t.Errorf("StageRelays = %v, want a not-a-regular-file error", err) + if _, _, err := StageRelays(); err == nil || !strings.Contains(err.Error(), "hive-review") { + t.Errorf("StageRelays = %v, want an error naming the relay", err) } t.Setenv(RelaySourceDirEnv, filepath.Join(src, "missing")) if _, _, err := StageRelays(); err == nil {