From 265bc2d8d4c27e82f3ffdaadbfaec82f583ba579 Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Tue, 17 Nov 2026 14:39:42 -0500 Subject: [PATCH 1/5] =?UTF-8?q?=F0=9F=90=9B=20Sandbox=20Jobs=20keep=20thei?= =?UTF-8?q?r=20service=20account's=20imagePullSecrets,=20add=20sandbox.job?= =?UTF-8?q?.image=5Fpull=5Fsecrets=20(#18819)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Andy Anderson --- ...ed-18819-sandbox-job-image-pull-secrets.md | 1 + src/crux/agent/manager_sandbox.go | 1 + src/crux/config/agent_config.go | 34 +++++ src/crux/kubejob/kubejob.go | 7 + src/crux/kubejob/kubejob_test.go | 15 +++ src/crux/kubejob/pullsecrets.go | 122 ++++++++++++++++++ src/crux/kubejob/pullsecrets_test.go | 71 ++++++++++ src/deploy/k8s/sandbox-job-rbac.yaml | 5 + src/docs/agent-configuration.md | 2 +- src/docs/sandbox-isolation.md | 11 +- src/pkg/config/agent_sandbox_job_test.go | 62 +++++++++ src/pkg/config/config.go | 21 +++ src/pkg/config/crux_aliases.go | 2 + src/pkg/dashboard/api_agent_sandbox_fields.go | 11 ++ .../api_agent_sandbox_fields_test.go | 13 ++ .../dashboard/settings_parity_allowlist.txt | 2 + src/pkg/hub/saas_provision.go | 3 + 17 files changed, 381 insertions(+), 2 deletions(-) create mode 100644 changelog.d/fixed-18819-sandbox-job-image-pull-secrets.md create mode 100644 src/crux/kubejob/pullsecrets.go create mode 100644 src/crux/kubejob/pullsecrets_test.go diff --git a/changelog.d/fixed-18819-sandbox-job-image-pull-secrets.md b/changelog.d/fixed-18819-sandbox-job-image-pull-secrets.md new file mode 100644 index 0000000000..b5b9c085f4 --- /dev/null +++ b/changelog.d/fixed-18819-sandbox-job-image-pull-secrets.md @@ -0,0 +1 @@ +- Sandbox Jobs no longer lose their service account's `imagePullSecrets`: the Job pod now carries the deduplicated union of hive's own pod secrets, the Job service account's secrets (read through the API when the launcher's Role grants `get serviceaccounts`, skipped with a warning otherwise) and the new `sandbox.job.image_pull_secrets` list of Secret names, so private-registry project images pull instead of failing ImagePullBackOff. A boot and reload warning fires when `service_account` is set without `image_pull_secrets` (#18819). diff --git a/src/crux/agent/manager_sandbox.go b/src/crux/agent/manager_sandbox.go index 7f1ed2b3df..09b304c468 100644 --- a/src/crux/agent/manager_sandbox.go +++ b/src/crux/agent/manager_sandbox.go @@ -534,6 +534,7 @@ func JobOptionsFromConfig(job config.SandboxJobConfig) kubejob.Options { ServiceAccount: job.ServiceAccount, SchedulerName: job.SchedulerName, EnvFromSecrets: append([]string(nil), job.EnvFromSecrets...), + ImagePullSecrets: append([]string(nil), job.ImagePullSecrets...), TTLSeconds: job.TTLSeconds, RelayImage: job.RelayImage, RelayRunAsUser: job.RelayRunAsUser, diff --git a/src/crux/config/agent_config.go b/src/crux/config/agent_config.go index f808cfe1a1..3f907bc5ae 100644 --- a/src/crux/config/agent_config.go +++ b/src/crux/config/agent_config.go @@ -1,6 +1,8 @@ package config import ( + "fmt" + "regexp" "strings" "time" ) @@ -53,6 +55,33 @@ type SandboxJobConfig struct { // the cluster on OpenShift (the SCC assigns one from the namespace range) // and uses 65534 elsewhere, since the hive image's default user is root. RelayRunAsUser *int64 `yaml:"relay_run_as_user,omitempty" json:"relay_run_as_user,omitempty"` + // ImagePullSecrets are Secret NAMES (never values) added to the Job pod's + // imagePullSecrets, for a private-registry sandbox image (#18819). The pod + // gets the deduplicated union of hive's own pod secrets, the Job service + // account's imagePullSecrets (when the launcher may read it) and this list. + ImagePullSecrets []string `yaml:"image_pull_secrets,omitempty" json:"image_pull_secrets,omitempty"` +} + +// MaxJobImagePullSecrets bounds image_pull_secrets. +const MaxJobImagePullSecrets = 32 + +// maxPullSecretNameLen is the Kubernetes object-name length limit. +const maxPullSecretNameLen = 253 + +var pullSecretNameRE = regexp.MustCompile(`^[a-z0-9]([-a-z0-9.]*[a-z0-9])?$`) + +// ValidateImagePullSecrets refuses anything but a bounded list of Kubernetes +// Secret names, so a pasted credential never reaches the config file. +func ValidateImagePullSecrets(names []string) error { + if len(names) > MaxJobImagePullSecrets { + return fmt.Errorf("sandbox.job.image_pull_secrets takes at most %d names, got %d", MaxJobImagePullSecrets, len(names)) + } + for _, n := range names { + if len(n) > maxPullSecretNameLen || !pullSecretNameRE.MatchString(n) { + return fmt.Errorf("sandbox.job.image_pull_secrets takes Secret names, not values: %q is not a valid name", n) + } + } + return nil } // SandboxJobToleration mirrors the subset of a Kubernetes toleration an @@ -118,6 +147,11 @@ func (a *AgentConfig) SandboxJob(global AgentSandboxConfig) SandboxJobConfig { if out.TTLSeconds == 0 { out.TTLSeconds = g.TTLSeconds } + // Registry credentials are a cluster fact like the claim: a per-agent + // block that names none inherits the global list (#18819). + if len(out.ImagePullSecrets) == 0 { + out.ImagePullSecrets = g.ImagePullSecrets + } return out } diff --git a/src/crux/kubejob/kubejob.go b/src/crux/kubejob/kubejob.go index ba5252b403..8fe75ff11b 100644 --- a/src/crux/kubejob/kubejob.go +++ b/src/crux/kubejob/kubejob.go @@ -132,6 +132,9 @@ type Options struct { // EnvFromSecrets are Secret names whose keys become environment variables // in the Job. This is the ONLY credential path into the Job. EnvFromSecrets []string + // ImagePullSecrets are Secret names added to the pod's imagePullSecrets + // alongside hive's own and the service account's (#18819). Names only. + ImagePullSecrets []string // ExtraVolumes mounts additional claims (a compile cache, a model store). ExtraVolumes []VolumeMount // TTLSeconds is the Job's ttlSecondsAfterFinished (default one hour). @@ -266,6 +269,9 @@ func (o Options) Validate() error { if o.RelayRunAsUser != nil && *o.RelayRunAsUser <= 0 { return errors.New("kubejob: relay_run_as_user must be a positive (non-root) uid") } + if err := validatePullSecretNames(o.ImagePullSecrets); err != nil { + return err + } switch o.network() { case NetworkDefault, NetworkNone, NetworkHost: default: @@ -308,6 +314,7 @@ func (l *Launcher) Run(ctx context.Context, spec sandbox.LaunchSpec) (sandbox.Re } name := jobName(spec.Name, l.now()) manifest := l.jobManifest(ns, name, subPath, spec, relays) + setPullSecrets(manifest, l.resolvePullSecrets(ctx, ns, relays)) if jobSpec, ok := manifest["spec"].(map[string]any); ok { jobSpec["activeDeadlineSeconds"] = l.activeDeadline(ctx) } diff --git a/src/crux/kubejob/kubejob_test.go b/src/crux/kubejob/kubejob_test.go index 6ad2c70eed..f7ed99289c 100644 --- a/src/crux/kubejob/kubejob_test.go +++ b/src/crux/kubejob/kubejob_test.go @@ -45,6 +45,8 @@ type fakeAPI struct { order []string // "policy", "job" in creation order selfPod string // JSON body of hive's own pod (testSelfPod), "" = 404 selfPodGets int + serviceAccounts map[string]string // SA name -> JSON body; absent = 404 + saForbidden bool // SA reads answer 403 } func newFakeAPI() *fakeAPI { @@ -122,6 +124,19 @@ func (f *fakeAPI) handler(t *testing.T) http.HandlerFunc { f.onPodLogAfterComplete() } _, _ = w.Write([]byte(f.logBody)) + case r.Method == http.MethodGet && strings.Contains(r.URL.Path, "/serviceaccounts/"): + if f.saForbidden { + w.WriteHeader(http.StatusForbidden) + _, _ = w.Write([]byte(`{"kind":"Status","message":"serviceaccounts is forbidden"}`)) + return + } + body, ok := f.serviceAccounts[r.URL.Path[strings.LastIndex(r.URL.Path, "/")+1:]] + if !ok { + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte(`{"kind":"Status","message":"not found"}`)) + return + } + _, _ = w.Write([]byte(body)) case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/pods/"+testSelfPod): f.selfPodGets++ if f.selfPod == "" { diff --git a/src/crux/kubejob/pullsecrets.go b/src/crux/kubejob/pullsecrets.go new file mode 100644 index 0000000000..4b1b24acd2 --- /dev/null +++ b/src/crux/kubejob/pullsecrets.go @@ -0,0 +1,122 @@ +package kubejob + +import ( + "context" + "encoding/json" + "fmt" + "log/slog" + "net/http" + "net/url" + "regexp" + "strings" +) + +// Pull secrets on the Job pod (#18819). +// +// Kubernetes' ServiceAccount admission adds the service account's +// imagePullSecrets only to a pod that has none of its own, so the launcher +// setting the hive pod's list (for the relay init image) hid the service +// account's. The pod now carries the union, in a stable order: hive's own, +// then the service account's, then job.image_pull_secrets. Only Secret NAMES +// are ever read, logged or written. + +const ( + // maxPullSecrets bounds the merged list. + maxPullSecrets = 64 + // maxConfiguredPullSecrets bounds job.image_pull_secrets. + maxConfiguredPullSecrets = 32 + maxPullSecretNameLen = 253 +) + +var pullSecretNameRE = regexp.MustCompile(`^[a-z0-9]([-a-z0-9.]*[a-z0-9])?$`) + +// validatePullSecretNames refuses anything but a bounded list of names. +func validatePullSecretNames(names []string) error { + if len(names) > maxConfiguredPullSecrets { + return fmt.Errorf("kubejob: image_pull_secrets takes at most %d names, got %d", maxConfiguredPullSecrets, len(names)) + } + for _, n := range names { + if len(n) > maxPullSecretNameLen || !pullSecretNameRE.MatchString(n) { + return fmt.Errorf("kubejob: image_pull_secrets takes Secret names, not values: %q is not a valid name", n) + } + } + return nil +} + +// mergePullSecretNames concatenates the lists, dropping blanks and repeats and +// keeping first-seen order. +func mergePullSecretNames(lists ...[]string) []string { + seen := map[string]bool{} + var out []string + for _, list := range lists { + for _, n := range list { + n = strings.TrimSpace(n) + if n == "" || seen[n] || len(out) >= maxPullSecrets { + continue + } + seen[n] = true + out = append(out, n) + } + } + return out +} + +// serviceAccountPullSecrets reads the names of the Job service account's +// imagePullSecrets. A launcher whose RBAC lacks `get serviceaccounts` (or any +// other failure) yields a warning and no names, never an error. +func (l *Launcher) serviceAccountPullSecrets(ctx context.Context, ns string) []string { + sa := strings.TrimSpace(l.Options.ServiceAccount) + if sa == "" { + return nil + } + raw, err := l.do(ctx, http.MethodGet, fmt.Sprintf("/api/v1/namespaces/%s/serviceaccounts/%s", ns, url.PathEscape(sa)), nil) + if err != nil { + slog.Warn("kubejob: cannot read the Job service account's imagePullSecrets; set sandbox.job.image_pull_secrets if the sandbox image is in a private registry", + "serviceAccount", sa, "error", err.Error()) + return nil + } + var body struct { + ImagePullSecrets []struct { + Name string `json:"name"` + } `json:"imagePullSecrets"` + } + if err := json.Unmarshal(raw, &body); err != nil { + slog.Warn("kubejob: the Job service account did not decode; its imagePullSecrets are skipped", "serviceAccount", sa) + return nil + } + names := make([]string, 0, len(body.ImagePullSecrets)) + for _, s := range body.ImagePullSecrets { + names = append(names, s.Name) + } + return names +} + +// resolvePullSecrets is the union for one Job pod. +func (l *Launcher) resolvePullSecrets(ctx context.Context, ns string, relays *relayPlan) []string { + var hive []string + if relays != nil { + for _, s := range relays.pullSecrets { + hive = append(hive, s["name"]) + } + } + return mergePullSecretNames(hive, l.serviceAccountPullSecrets(ctx, ns), l.Options.ImagePullSecrets) +} + +// setPullSecrets writes names into the manifest's pod spec; no names leaves it +// untouched. +func setPullSecrets(manifest map[string]any, names []string) { + if len(names) == 0 { + return + } + spec, _ := manifest["spec"].(map[string]any) + tmpl, _ := spec["template"].(map[string]any) + pod, _ := tmpl["spec"].(map[string]any) + if pod == nil { + return + } + refs := make([]map[string]string, 0, len(names)) + for _, n := range names { + refs = append(refs, map[string]string{"name": n}) + } + pod["imagePullSecrets"] = refs +} diff --git a/src/crux/kubejob/pullsecrets_test.go b/src/crux/kubejob/pullsecrets_test.go new file mode 100644 index 0000000000..e0d4d47e5d --- /dev/null +++ b/src/crux/kubejob/pullsecrets_test.go @@ -0,0 +1,71 @@ +package kubejob + +import ( + "strings" + "testing" +) + +const saWithSecrets = `{"imagePullSecrets":[{"name":"reg-a"},{"name":"ghcr-pull"}]}` + +func TestPullSecretsUnionDedupedAndOrdered(t *testing.T) { + api := newFakeAPI() + api.selfPod = vanillaSelfPod // ghcr-pull + api.serviceAccounts = map[string]string{"job-sa": saWithSecrets} + pod := runRelayJob(t, api, Options{ServiceAccount: "job-sa", ImagePullSecrets: []string{"reg-b", "reg-a"}}) + want := `[{"name":"ghcr-pull"},{"name":"reg-a"},{"name":"reg-b"}]` + if got := asJSON(pod["imagePullSecrets"]); got != want { + t.Errorf("imagePullSecrets = %s, want %s", got, want) + } +} + +func TestPullSecretsConfiguredOnlyWithoutHivePodSecrets(t *testing.T) { + api := newFakeAPI() + api.selfPod = openshiftSelfPod // no pull secrets + pod := runRelayJob(t, api, Options{ImagePullSecrets: []string{"reg-b"}}) + if got := asJSON(pod["imagePullSecrets"]); got != `[{"name":"reg-b"}]` { + t.Errorf("imagePullSecrets = %s, want the configured name", got) + } +} + +func TestPullSecretsServiceAccountReadDeniedFallsBack(t *testing.T) { + api := newFakeAPI() + api.selfPod = vanillaSelfPod + api.saForbidden = true + pod := runRelayJob(t, api, Options{ServiceAccount: "job-sa", ImagePullSecrets: []string{"reg-b"}}) + if got := asJSON(pod["imagePullSecrets"]); got != `[{"name":"ghcr-pull"},{"name":"reg-b"}]` { + t.Errorf("imagePullSecrets = %s, want hive's plus configured, no error", got) + } +} + +func TestPullSecretsNoneLeavesFieldOff(t *testing.T) { + api := newFakeAPI() + api.selfPod = openshiftSelfPod + pod := runRelayJob(t, api, Options{}) + if _, ok := pod["imagePullSecrets"]; ok { + t.Errorf("imagePullSecrets = %s, want none", asJSON(pod["imagePullSecrets"])) + } +} + +func TestValidatePullSecretNames(t *testing.T) { + if err := validatePullSecretNames([]string{"reg-a", "a.b-c"}); err != nil { + t.Errorf("valid names refused: %v", err) + } + for _, bad := range []string{"KEY=value", "Upper", "has space", "-lead", ""} { + if err := validatePullSecretNames([]string{bad}); err == nil { + t.Errorf("%q accepted", bad) + } + } + if err := validatePullSecretNames(make([]string, maxConfiguredPullSecrets+1)); err == nil { + t.Error("over-long list accepted") + } + if err := (Options{WorkspaceClaim: "c", ImagePullSecrets: []string{"a=b"}}).Validate(); err == nil || !strings.Contains(err.Error(), "names") { + t.Errorf("Validate = %v, want a names-only refusal", err) + } +} + +func TestMergePullSecretNames(t *testing.T) { + got := mergePullSecretNames([]string{"a", "b", ""}, []string{"b", "c"}, []string{"a", "d"}) + if strings.Join(got, ",") != "a,b,c,d" { + t.Errorf("merge = %v", got) + } +} diff --git a/src/deploy/k8s/sandbox-job-rbac.yaml b/src/deploy/k8s/sandbox-job-rbac.yaml index 5cbdf84abe..713ee067f7 100644 --- a/src/deploy/k8s/sandbox-job-rbac.yaml +++ b/src/deploy/k8s/sandbox-job-rbac.yaml @@ -18,6 +18,11 @@ rules: - apiGroups: [""] resources: ["pods", "pods/log"] verbs: ["get", "list"] + # Names of the Job service account's imagePullSecrets (#18819). Optional: + # without it hive warns and uses job.image_pull_secrets only. + - apiGroups: [""] + resources: ["serviceaccounts"] + verbs: ["get"] --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding diff --git a/src/docs/agent-configuration.md b/src/docs/agent-configuration.md index acbc995ceb..583b6ae2fd 100644 --- a/src/docs/agent-configuration.md +++ b/src/docs/agent-configuration.md @@ -411,7 +411,7 @@ For prompt file resolution and the complete built-in `${VAR}` reference, see ### Sandbox settings (`agents..sandbox`) -An agent's `sandbox` block (image, runtime, Job fields) is editable from its General tab under "Sandbox settings: image, runtime and Job", and through `PUT /api/config/agent//general` with `sandboxImage`, `sandboxRuntime`, `sandboxJob` and `sandboxCopyFrom` (hivecommons/hive#18797). A blank value inherits the hive-wide `agent_sandbox` default, which is set under Settings, Security. "Copy sandbox settings from..." copies another agent's complete block, Secret names included and never values. Enabling a sandbox that resolves no image is refused with the field named. The full set of controls and keys is in [sandbox-isolation.md](sandbox-isolation.md#setting-up-a-sandbox-from-the-dashboard). +An agent's `sandbox` block (image, runtime, Job fields) is editable from its General tab under "Sandbox settings: image, runtime and Job", and through `PUT /api/config/agent//general` with `sandboxImage`, `sandboxRuntime`, `sandboxJob` and `sandboxCopyFrom` (hivecommons/hive#18797). A blank value inherits the hive-wide `agent_sandbox` default, which is set under Settings, Security. "Copy sandbox settings from..." copies another agent's complete block, Secret names included and never values. Enabling a sandbox that resolves no image is refused with the field named. The full set of controls and keys is in [sandbox-isolation.md](sandbox-isolation.md#setting-up-a-sandbox-from-the-dashboard). `sandbox.job.image_pull_secrets` (Secret names only, at most 32; a per-agent block that sets none inherits `agent_sandbox.job`) adds registry pull secrets to the Job pod beside hive's own and the Job service account's; the API field is `sandboxJob.imagePullSecrets` ([#18819](https://github.com/hivecommons/hive/issues/18819), see [Private-registry images](sandbox-isolation.md#running-sandboxed-kicks-as-kubernetes-jobs)). ### Declarative extensions diff --git a/src/docs/sandbox-isolation.md b/src/docs/sandbox-isolation.md index 3b7dba15c8..b3fd84bf81 100644 --- a/src/docs/sandbox-isolation.md +++ b/src/docs/sandbox-isolation.md @@ -247,7 +247,16 @@ agents: **The terminal pane.** A job-runtime agent has no tmux session in the hive pod, so the dashboard's ▶ terminal does not attach ttyd for it. Instead it opens `/terminal/job/`, a read-only viewer that streams the newest Job pod's log through the same in-cluster client and RBAC the launcher already uses ([#14798](https://github.com/hivecommons/hive/issues/14798)): it follows the pod while the kick runs (waiting while the pod is `Pending`, e.g. for an accelerator node), prints the container exit code when it ends, and between kicks, when the completed Job has already been deleted, shows the last run's persisted transcript (`/.hive/sandbox-transcript.log`, which now ends with an `exit code: N` line) or a clear "no sandbox run yet" state. The pane is observation only; there is no input path into the Job. The same terminal handoff and role check as the tmux pane apply. -**RBAC.** The spoke's ServiceAccount needs a Role for `batch/jobs` (create, get, list, delete) and `pods` plus `pods/log` (get, list); the terminal viewer needs nothing beyond that same `pods/log` grant. The standalone kustomize base ships it as `sandbox-job-rbac.yaml`, and the hub provisioner grants it to every hosted spoke as `hive-sandbox-jobs`. +**Private-registry images (#18819).** The Job pod's `imagePullSecrets` are the deduplicated union, in this order, of: hive's own pod secrets (so the `hive-relays` init container can pull the hive image), the Job service account's `imagePullSecrets`, and `sandbox.job.image_pull_secrets`. Kubernetes adds a service account's pull secrets only to a pod that has none of its own, so hive reads the service account through the API (`get serviceaccounts`) and merges the names itself. When that grant is missing or the read fails, hive logs a warning and carries on with the other two lists; it never fails the kick. `image_pull_secrets` holds Secret **names** only (DNS-1123 names, at most 32; a value is refused). A per-agent `job` block that names none inherits the `agent_sandbox.job` list. Hive reads, logs and writes names, never Secret contents. When `service_account` is set and `image_pull_secrets` is not, the boot and reload warning reminds the operator to set it if the grant is absent. The dashboard sandbox controls do not carry this field yet; the per-agent API accepts `sandboxJob.imagePullSecrets`. + +```yaml +agent_sandbox: + job: + service_account: hive-spyre-job + image_pull_secrets: [ registry-pull ] +``` + +**RBAC.** The spoke's ServiceAccount needs a Role for `batch/jobs` (create, get, list, delete), `pods` plus `pods/log` (get, list) and, optionally, `serviceaccounts` (get) so the Job service account's pull secrets are merged; the terminal viewer needs nothing beyond that same `pods/log` grant. The standalone kustomize base ships it as `sandbox-job-rbac.yaml`, and the hub provisioner grants it to every hosted spoke as `hive-sandbox-jobs`. Hive warns at boot and on reload (`config.AgentSandboxGateWarnings`) when a job-runtime agent resolves no `workspace_claim` or names a runtime hive does not have, and refuses the kick with the same message rather than failing it after the agent is marked running. diff --git a/src/pkg/config/agent_sandbox_job_test.go b/src/pkg/config/agent_sandbox_job_test.go index ea7f198ee3..b6cb02029b 100644 --- a/src/pkg/config/agent_sandbox_job_test.go +++ b/src/pkg/config/agent_sandbox_job_test.go @@ -147,3 +147,65 @@ func TestGateWarnsUnknownRuntime(t *testing.T) { t.Errorf("warnings = %q, want one naming deps=lambda and the valid runtimes", got) } } + +func TestSandboxJobImagePullSecretsMergeAndValidation(t *testing.T) { + global := AgentSandboxConfig{Job: &SandboxJobConfig{WorkspaceClaim: "hive-data", ImagePullSecrets: []string{"reg-a"}}} + inherit := &AgentConfig{Sandbox: &AgentSandboxOverride{Job: &SandboxJobConfig{ServiceAccount: "sa"}}} + if got := inherit.SandboxJob(global).ImagePullSecrets; len(got) != 1 || got[0] != "reg-a" { + t.Errorf("inherited = %v, want [reg-a]", got) + } + own := &AgentConfig{Sandbox: &AgentSandboxOverride{Job: &SandboxJobConfig{ImagePullSecrets: []string{"reg-b"}}}} + if got := own.SandboxJob(global).ImagePullSecrets; len(got) != 1 || got[0] != "reg-b" { + t.Errorf("own = %v, want [reg-b]", got) + } + if err := ValidateImagePullSecrets([]string{"reg-a", "x.y"}); err != nil { + t.Errorf("valid names refused: %v", err) + } + for _, bad := range []string{"user:pass", "UPPER", "a b", ""} { + if err := ValidateImagePullSecrets([]string{bad}); err == nil { + t.Errorf("%q accepted", bad) + } + } + if err := ValidateImagePullSecrets(make([]string, MaxJobImagePullSecrets+1)); err == nil { + t.Error("over-long list accepted") + } +} + +func TestSandboxJobImagePullSecretsYAMLRoundTrip(t *testing.T) { + var job SandboxJobConfig + if err := yaml.Unmarshal([]byte("service_account: sa\nimage_pull_secrets: [reg-a, reg-b]\n"), &job); err != nil { + t.Fatal(err) + } + out, err := yaml.Marshal(job) + if err != nil { + t.Fatal(err) + } + var back SandboxJobConfig + if err := yaml.Unmarshal(out, &back); err != nil { + t.Fatal(err) + } + if len(back.ImagePullSecrets) != 2 || back.ImagePullSecrets[1] != "reg-b" { + t.Errorf("round trip = %v", back.ImagePullSecrets) + } +} + +func TestGateWarnsServiceAccountWithoutImagePullSecrets(t *testing.T) { + yes := true + mk := func(pull []string) *Config { + return &Config{AgentRuntimeConfig: AgentRuntimeConfig{ + AgentSandbox: AgentSandboxConfig{Enabled: true, Image: "img", Runtime: SandboxRuntimeJob, + Job: &SandboxJobConfig{WorkspaceClaim: "hive-data", ServiceAccount: "job-sa", ImagePullSecrets: pull}}, + Agents: map[string]AgentConfig{"deps": {Sandbox: &AgentSandboxOverride{Enabled: &yes}}}, + }} + } + got := AgentSandboxGateWarnings(mk(nil)) + if len(got) != 1 || !strings.Contains(got[0], "image_pull_secrets") { + t.Errorf("warnings = %q, want one naming image_pull_secrets", got) + } + if got := AgentSandboxGateWarnings(mk([]string{"reg-a"})); len(got) != 0 { + t.Errorf("configured list must raise no warning, got %q", got) + } + if got := AgentSandboxGateWarnings(mk([]string{"user:pass"})); len(got) != 1 || !strings.Contains(got[0], "invalid") { + t.Errorf("invalid name warnings = %q", got) + } +} diff --git a/src/pkg/config/config.go b/src/pkg/config/config.go index 6cab8008a0..8402dd159e 100644 --- a/src/pkg/config/config.go +++ b/src/pkg/config/config.go @@ -1648,6 +1648,7 @@ func AgentSandboxGateReport(cfg *Config) SandboxGateReport { return SandboxGateReport{} } + var noPullSecrets, badPullSecrets []string var optedIn, noImage, noClaim, noGateway, noDigest, badRuntime []string var openshellBlocks []SandboxOpenShellConfig var substrateBlocks []SandboxSubstrateConfig @@ -1659,6 +1660,14 @@ func AgentSandboxGateReport(cfg *Config) SandboxGateReport { if strings.TrimSpace(a.SandboxImage(cfg.AgentSandbox)) == "" { noImage = append(noImage, name) } + if a.SandboxRuntime(cfg.AgentSandbox) == SandboxRuntimeJob { + job := a.SandboxJob(cfg.AgentSandbox) + if err := ValidateImagePullSecrets(job.ImagePullSecrets); err != nil { + badPullSecrets = append(badPullSecrets, name) + } else if strings.TrimSpace(job.ServiceAccount) != "" && len(job.ImagePullSecrets) == 0 { + noPullSecrets = append(noPullSecrets, name) + } + } switch rt := a.SandboxRuntime(cfg.AgentSandbox); { case !ValidSandboxRuntimes[rt]: badRuntime = append(badRuntime, name+"="+rt) @@ -1691,6 +1700,8 @@ func AgentSandboxGateReport(cfg *Config) SandboxGateReport { sort.Strings(noGateway) sort.Strings(noDigest) sort.Strings(badRuntime) + sort.Strings(noPullSecrets) + sort.Strings(badPullSecrets) var out []string if len(optedIn) == 0 { @@ -1718,6 +1729,16 @@ func AgentSandboxGateReport(cfg *Config) SandboxGateReport { "set agent_sandbox.job.workspace_claim / agent_sandbox.openshell.workspace_claim (or the per-agent one) to the PVC hive's data lives on (#6311)", strings.Join(noClaim, ", "))) } + if len(badPullSecrets) > 0 { + out = append(out, fmt.Sprintf( + "agent(s) %s have an invalid sandbox.job.image_pull_secrets (names only, at most %d); sandboxed kicks will be refused (#18819)", + strings.Join(badPullSecrets, ", "), MaxJobImagePullSecrets)) + } + if len(noPullSecrets) > 0 { + out = append(out, fmt.Sprintf( + "agent(s) %s run Jobs under sandbox.job.service_account with no sandbox.job.image_pull_secrets; hive reads that service account's imagePullSecrets only when its Role grants `get serviceaccounts`, and without it a private-registry sandbox image fails ImagePullBackOff (see the launcher warning) — set sandbox.job.image_pull_secrets to the registry Secret names (#18819)", + strings.Join(noPullSecrets, ", "))) + } if len(noGateway) > 0 { out = append(out, fmt.Sprintf( "agent(s) %s use the openshell sandbox runtime but resolve no openshell.gateway_url; every kick will fail — "+ diff --git a/src/pkg/config/crux_aliases.go b/src/pkg/config/crux_aliases.go index c7561e6c40..2878ecac4b 100644 --- a/src/pkg/config/crux_aliases.go +++ b/src/pkg/config/crux_aliases.go @@ -145,6 +145,7 @@ const ( ReviewModelsFallbackSkip = cruxconfig.ReviewModelsFallbackSkip RotationHeadroomSourceBuiltin = cruxconfig.RotationHeadroomSourceBuiltin RotationHeadroomSourceCCLeft = cruxconfig.RotationHeadroomSourceCCLeft + MaxJobImagePullSecrets = cruxconfig.MaxJobImagePullSecrets SandboxRuntimeJob = cruxconfig.SandboxRuntimeJob SandboxRuntimeOpenShell = cruxconfig.SandboxRuntimeOpenShell SandboxRuntimeSubstrate = cruxconfig.SandboxRuntimeSubstrate @@ -165,6 +166,7 @@ var ( ReasoningEffortsByBackend = cruxconfig.ReasoningEffortsByBackend ToolPresets = cruxconfig.ToolPresets ValidSandboxRuntimes = cruxconfig.ValidSandboxRuntimes + ValidateImagePullSecrets = cruxconfig.ValidateImagePullSecrets ) var ( diff --git a/src/pkg/dashboard/api_agent_sandbox_fields.go b/src/pkg/dashboard/api_agent_sandbox_fields.go index 14bddd712e..e007ae8529 100644 --- a/src/pkg/dashboard/api_agent_sandbox_fields.go +++ b/src/pkg/dashboard/api_agent_sandbox_fields.go @@ -69,6 +69,7 @@ func agentSandboxFieldsView(a config.AgentConfig, global config.AgentSandboxConf "schedulerName": job.SchedulerName, "ttlSeconds": job.TTLSeconds, "envFromSecrets": nonNilStrings(job.EnvFromSecrets), + "imagePullSecrets": nonNilStrings(job.ImagePullSecrets), }, } } @@ -154,6 +155,16 @@ func applySandboxJobFields(job *config.SandboxJobConfig, m map[string]interface{ } job.EnvFromSecrets = names } + if v, ok := m["imagePullSecrets"]; ok { + names, err := secretNamesFromBody(v) + if err != nil { + return err + } + if err := config.ValidateImagePullSecrets(names); err != nil { + return err + } + job.ImagePullSecrets = names + } return nil } diff --git a/src/pkg/dashboard/api_agent_sandbox_fields_test.go b/src/pkg/dashboard/api_agent_sandbox_fields_test.go index 362f4c10bb..1c1e0a284f 100644 --- a/src/pkg/dashboard/api_agent_sandbox_fields_test.go +++ b/src/pkg/dashboard/api_agent_sandbox_fields_test.go @@ -161,3 +161,16 @@ func TestSecuritySectionSetsDefaultSandboxImageAndJobSecrets(t *testing.T) { t.Fatalf("secret value accepted: %d %s", rec.Code, rec.Body.String()) } } + +func TestApplySandboxJobFieldsImagePullSecrets(t *testing.T) { + var job config.SandboxJobConfig + if err := applySandboxJobFields(&job, map[string]interface{}{"imagePullSecrets": []interface{}{"reg-a", " reg-b "}}); err != nil { + t.Fatal(err) + } + if len(job.ImagePullSecrets) != 2 || job.ImagePullSecrets[1] != "reg-b" { + t.Errorf("ImagePullSecrets = %v", job.ImagePullSecrets) + } + if err := applySandboxJobFields(&job, map[string]interface{}{"imagePullSecrets": []interface{}{"user:pass"}}); err == nil { + t.Error("a value pasted as a name was accepted") + } +} diff --git a/src/pkg/dashboard/settings_parity_allowlist.txt b/src/pkg/dashboard/settings_parity_allowlist.txt index 4d422b515d..fe51d89051 100644 --- a/src/pkg/dashboard/settings_parity_allowlist.txt +++ b/src/pkg/dashboard/settings_parity_allowlist.txt @@ -1,6 +1,7 @@ # Current Settings dialog parity gaps. # Every entry must have a Settings control before it is removed. # TODO #11854 +agent_sandbox.job.image_pull_secrets[] # TODO #11854 agent_sandbox.job.node_selector. # TODO #11854 agent_sandbox.job.relay_image # TODO #11854 agent_sandbox.job.relay_run_as_user # TODO #11854 @@ -116,6 +117,7 @@ agents..sandbox.enabled # TODO #11854 agents..sandbox.env_allowlist[] # TODO #11854 agents..sandbox.image # TODO #11854 agents..sandbox.job.env_from_secrets[] # TODO #11854 +agents..sandbox.job.image_pull_secrets[] # TODO #11854 agents..sandbox.job.node_selector. # TODO #11854 agents..sandbox.job.relay_image # TODO #11854 agents..sandbox.job.relay_run_as_user # TODO #11854 diff --git a/src/pkg/hub/saas_provision.go b/src/pkg/hub/saas_provision.go index 75939a069a..ba1ea75d55 100644 --- a/src/pkg/hub/saas_provision.go +++ b/src/pkg/hub/saas_provision.go @@ -3098,6 +3098,9 @@ rules: - apiGroups: [""] resources: ["pods", "pods/log"] verbs: ["get", "list"] +- apiGroups: [""] + resources: ["serviceaccounts"] + verbs: ["get"] --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding From 3fda442a57c376d112423c707050e7eafe157ef4 Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Tue, 17 Nov 2026 14:43:18 -0500 Subject: [PATCH 2/5] =?UTF-8?q?=F0=9F=90=9B=20Move=20the=20image=5Fpull=5F?= =?UTF-8?q?secrets=20validator=20out=20of=20agent=5Fconfig.go=20(file-size?= =?UTF-8?q?=20ratchet)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Andy Anderson --- src/crux/config/agent_config.go | 24 ----------------------- src/crux/config/image_pull_secrets.go | 28 +++++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 24 deletions(-) create mode 100644 src/crux/config/image_pull_secrets.go diff --git a/src/crux/config/agent_config.go b/src/crux/config/agent_config.go index 3f907bc5ae..f02669df1b 100644 --- a/src/crux/config/agent_config.go +++ b/src/crux/config/agent_config.go @@ -1,8 +1,6 @@ package config import ( - "fmt" - "regexp" "strings" "time" ) @@ -62,28 +60,6 @@ type SandboxJobConfig struct { ImagePullSecrets []string `yaml:"image_pull_secrets,omitempty" json:"image_pull_secrets,omitempty"` } -// MaxJobImagePullSecrets bounds image_pull_secrets. -const MaxJobImagePullSecrets = 32 - -// maxPullSecretNameLen is the Kubernetes object-name length limit. -const maxPullSecretNameLen = 253 - -var pullSecretNameRE = regexp.MustCompile(`^[a-z0-9]([-a-z0-9.]*[a-z0-9])?$`) - -// ValidateImagePullSecrets refuses anything but a bounded list of Kubernetes -// Secret names, so a pasted credential never reaches the config file. -func ValidateImagePullSecrets(names []string) error { - if len(names) > MaxJobImagePullSecrets { - return fmt.Errorf("sandbox.job.image_pull_secrets takes at most %d names, got %d", MaxJobImagePullSecrets, len(names)) - } - for _, n := range names { - if len(n) > maxPullSecretNameLen || !pullSecretNameRE.MatchString(n) { - return fmt.Errorf("sandbox.job.image_pull_secrets takes Secret names, not values: %q is not a valid name", n) - } - } - return nil -} - // SandboxJobToleration mirrors the subset of a Kubernetes toleration an // operator writes by hand. type SandboxJobToleration struct { diff --git a/src/crux/config/image_pull_secrets.go b/src/crux/config/image_pull_secrets.go new file mode 100644 index 0000000000..b2c362eadb --- /dev/null +++ b/src/crux/config/image_pull_secrets.go @@ -0,0 +1,28 @@ +package config + +import ( + "fmt" + "regexp" +) + +// MaxJobImagePullSecrets bounds image_pull_secrets. +const MaxJobImagePullSecrets = 32 + +// maxPullSecretNameLen is the Kubernetes object-name length limit. +const maxPullSecretNameLen = 253 + +var pullSecretNameRE = regexp.MustCompile(`^[a-z0-9]([-a-z0-9.]*[a-z0-9])?$`) + +// ValidateImagePullSecrets refuses anything but a bounded list of Kubernetes +// Secret names, so a pasted credential never reaches the config file. +func ValidateImagePullSecrets(names []string) error { + if len(names) > MaxJobImagePullSecrets { + return fmt.Errorf("sandbox.job.image_pull_secrets takes at most %d names, got %d", MaxJobImagePullSecrets, len(names)) + } + for _, n := range names { + if len(n) > maxPullSecretNameLen || !pullSecretNameRE.MatchString(n) { + return fmt.Errorf("sandbox.job.image_pull_secrets takes Secret names, not values: %q is not a valid name", n) + } + } + return nil +} From bfe735236835d0692f8c5647aa23d3fd089f1f70 Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Tue, 17 Nov 2026 14:43:36 -0500 Subject: [PATCH 3/5] =?UTF-8?q?=F0=9F=90=9B=20Keep=20agent=5Fconfig.go=20w?= =?UTF-8?q?ithin=20the=20file-size=20limit?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Andy Anderson --- src/crux/config/agent_config.go | 11 ++--------- src/crux/config/image_pull_secrets.go | 11 +++++++++++ 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/src/crux/config/agent_config.go b/src/crux/config/agent_config.go index f02669df1b..1be04f24ca 100644 --- a/src/crux/config/agent_config.go +++ b/src/crux/config/agent_config.go @@ -53,10 +53,7 @@ type SandboxJobConfig struct { // the cluster on OpenShift (the SCC assigns one from the namespace range) // and uses 65534 elsewhere, since the hive image's default user is root. RelayRunAsUser *int64 `yaml:"relay_run_as_user,omitempty" json:"relay_run_as_user,omitempty"` - // ImagePullSecrets are Secret NAMES (never values) added to the Job pod's - // imagePullSecrets, for a private-registry sandbox image (#18819). The pod - // gets the deduplicated union of hive's own pod secrets, the Job service - // account's imagePullSecrets (when the launcher may read it) and this list. + // ImagePullSecrets: Secret names for the Job pod (#18819); see image_pull_secrets.go. ImagePullSecrets []string `yaml:"image_pull_secrets,omitempty" json:"image_pull_secrets,omitempty"` } @@ -123,11 +120,7 @@ func (a *AgentConfig) SandboxJob(global AgentSandboxConfig) SandboxJobConfig { if out.TTLSeconds == 0 { out.TTLSeconds = g.TTLSeconds } - // Registry credentials are a cluster fact like the claim: a per-agent - // block that names none inherits the global list (#18819). - if len(out.ImagePullSecrets) == 0 { - out.ImagePullSecrets = g.ImagePullSecrets - } + out.ImagePullSecrets = inheritPullSecrets(out.ImagePullSecrets, g.ImagePullSecrets) return out } diff --git a/src/crux/config/image_pull_secrets.go b/src/crux/config/image_pull_secrets.go index b2c362eadb..c38b9d7591 100644 --- a/src/crux/config/image_pull_secrets.go +++ b/src/crux/config/image_pull_secrets.go @@ -26,3 +26,14 @@ func ValidateImagePullSecrets(names []string) error { } return nil } + +// inheritPullSecrets keeps a per-agent list and falls back to the global one: +// registry credentials are a cluster fact like the workspace claim, so a +// per-agent block that names none inherits them. The pod gets the union of +// hive's own pod secrets, the Job service account's and this list (#18819). +func inheritPullSecrets(own, global []string) []string { + if len(own) == 0 { + return global + } + return own +} From 43da2e4dcb448ce9b42d626be681113f3e5e2d75 Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Tue, 17 Nov 2026 14:43:49 -0500 Subject: [PATCH 4/5] =?UTF-8?q?=F0=9F=90=9B=20Keep=20agent=5Fconfig.go=20w?= =?UTF-8?q?ithin=20the=20file-size=20limit?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Andy Anderson --- src/crux/config/agent_config.go | 4 +--- src/crux/config/image_pull_secrets.go | 8 ++++---- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/src/crux/config/agent_config.go b/src/crux/config/agent_config.go index 1be04f24ca..78e5ec2f20 100644 --- a/src/crux/config/agent_config.go +++ b/src/crux/config/agent_config.go @@ -53,7 +53,6 @@ type SandboxJobConfig struct { // the cluster on OpenShift (the SCC assigns one from the namespace range) // and uses 65534 elsewhere, since the hive image's default user is root. RelayRunAsUser *int64 `yaml:"relay_run_as_user,omitempty" json:"relay_run_as_user,omitempty"` - // ImagePullSecrets: Secret names for the Job pod (#18819); see image_pull_secrets.go. ImagePullSecrets []string `yaml:"image_pull_secrets,omitempty" json:"image_pull_secrets,omitempty"` } @@ -120,8 +119,7 @@ func (a *AgentConfig) SandboxJob(global AgentSandboxConfig) SandboxJobConfig { if out.TTLSeconds == 0 { out.TTLSeconds = g.TTLSeconds } - out.ImagePullSecrets = inheritPullSecrets(out.ImagePullSecrets, g.ImagePullSecrets) - return out + return inheritPullSecrets(out, g) } // AgentRuntimeConfig is the top-level hive.yaml surface owned by the agent diff --git a/src/crux/config/image_pull_secrets.go b/src/crux/config/image_pull_secrets.go index c38b9d7591..3c41c849dc 100644 --- a/src/crux/config/image_pull_secrets.go +++ b/src/crux/config/image_pull_secrets.go @@ -31,9 +31,9 @@ func ValidateImagePullSecrets(names []string) error { // registry credentials are a cluster fact like the workspace claim, so a // per-agent block that names none inherits them. The pod gets the union of // hive's own pod secrets, the Job service account's and this list (#18819). -func inheritPullSecrets(own, global []string) []string { - if len(own) == 0 { - return global +func inheritPullSecrets(out, global SandboxJobConfig) SandboxJobConfig { + if len(out.ImagePullSecrets) == 0 { + out.ImagePullSecrets = global.ImagePullSecrets } - return own + return out } From d3c8d45787de74ccd56b166124b2d5b5e4c3df29 Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Tue, 17 Nov 2026 14:43:58 -0500 Subject: [PATCH 5/5] =?UTF-8?q?=F0=9F=90=9B=20gofmt=20agent=5Fconfig.go?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Andy Anderson --- src/crux/config/agent_config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/crux/config/agent_config.go b/src/crux/config/agent_config.go index 78e5ec2f20..bccc897a95 100644 --- a/src/crux/config/agent_config.go +++ b/src/crux/config/agent_config.go @@ -52,7 +52,7 @@ type SandboxJobConfig struct { // RelayRunAsUser pins the relay init container's uid. Unset leaves it to // the cluster on OpenShift (the SCC assigns one from the namespace range) // and uses 65534 elsewhere, since the hive image's default user is root. - RelayRunAsUser *int64 `yaml:"relay_run_as_user,omitempty" json:"relay_run_as_user,omitempty"` + RelayRunAsUser *int64 `yaml:"relay_run_as_user,omitempty" json:"relay_run_as_user,omitempty"` ImagePullSecrets []string `yaml:"image_pull_secrets,omitempty" json:"image_pull_secrets,omitempty"` }