Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/fixed-18819-sandbox-job-image-pull-secrets.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- Sandbox Jobs no longer lose their service account's `imagePullSecrets`: the Job pod now carries the deduplicated union of hive's own pod secrets, the Job service account's secrets (read through the API when the launcher's Role grants `get serviceaccounts`, skipped with a warning otherwise) and the new `sandbox.job.image_pull_secrets` list of Secret names, so private-registry project images pull instead of failing ImagePullBackOff. A boot and reload warning fires when `service_account` is set without `image_pull_secrets` (#18819).
1 change: 1 addition & 0 deletions src/crux/agent/manager_sandbox.go
Original file line number Diff line number Diff line change
Expand Up @@ -534,6 +534,7 @@ func JobOptionsFromConfig(job config.SandboxJobConfig) kubejob.Options {
ServiceAccount: job.ServiceAccount,
SchedulerName: job.SchedulerName,
EnvFromSecrets: append([]string(nil), job.EnvFromSecrets...),
ImagePullSecrets: append([]string(nil), job.ImagePullSecrets...),
TTLSeconds: job.TTLSeconds,
RelayImage: job.RelayImage,
RelayRunAsUser: job.RelayRunAsUser,
Expand Down
5 changes: 3 additions & 2 deletions src/crux/config/agent_config.go
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,8 @@ type SandboxJobConfig struct {
// RelayRunAsUser pins the relay init container's uid. Unset leaves it to
// the cluster on OpenShift (the SCC assigns one from the namespace range)
// and uses 65534 elsewhere, since the hive image's default user is root.
RelayRunAsUser *int64 `yaml:"relay_run_as_user,omitempty" json:"relay_run_as_user,omitempty"`
RelayRunAsUser *int64 `yaml:"relay_run_as_user,omitempty" json:"relay_run_as_user,omitempty"`
ImagePullSecrets []string `yaml:"image_pull_secrets,omitempty" json:"image_pull_secrets,omitempty"`
}

// SandboxJobToleration mirrors the subset of a Kubernetes toleration an
Expand Down Expand Up @@ -118,7 +119,7 @@ func (a *AgentConfig) SandboxJob(global AgentSandboxConfig) SandboxJobConfig {
if out.TTLSeconds == 0 {
out.TTLSeconds = g.TTLSeconds
}
return out
return inheritPullSecrets(out, g)
}

// AgentRuntimeConfig is the top-level hive.yaml surface owned by the agent
Expand Down
39 changes: 39 additions & 0 deletions src/crux/config/image_pull_secrets.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
package config

import (
"fmt"
"regexp"
)

// MaxJobImagePullSecrets bounds image_pull_secrets.
const MaxJobImagePullSecrets = 32

// maxPullSecretNameLen is the Kubernetes object-name length limit.
const maxPullSecretNameLen = 253

var pullSecretNameRE = regexp.MustCompile(`^[a-z0-9]([-a-z0-9.]*[a-z0-9])?$`)

// ValidateImagePullSecrets refuses anything but a bounded list of Kubernetes
// Secret names, so a pasted credential never reaches the config file.
func ValidateImagePullSecrets(names []string) error {
if len(names) > MaxJobImagePullSecrets {
return fmt.Errorf("sandbox.job.image_pull_secrets takes at most %d names, got %d", MaxJobImagePullSecrets, len(names))
}
for _, n := range names {
if len(n) > maxPullSecretNameLen || !pullSecretNameRE.MatchString(n) {
return fmt.Errorf("sandbox.job.image_pull_secrets takes Secret names, not values: %q is not a valid name", n)
}
}
return nil
}

// inheritPullSecrets keeps a per-agent list and falls back to the global one:
// registry credentials are a cluster fact like the workspace claim, so a
// per-agent block that names none inherits them. The pod gets the union of
// hive's own pod secrets, the Job service account's and this list (#18819).
func inheritPullSecrets(out, global SandboxJobConfig) SandboxJobConfig {
if len(out.ImagePullSecrets) == 0 {
out.ImagePullSecrets = global.ImagePullSecrets
}
return out
}
7 changes: 7 additions & 0 deletions src/crux/kubejob/kubejob.go
Original file line number Diff line number Diff line change
Expand Up @@ -132,6 +132,9 @@ type Options struct {
// EnvFromSecrets are Secret names whose keys become environment variables
// in the Job. This is the ONLY credential path into the Job.
EnvFromSecrets []string
// ImagePullSecrets are Secret names added to the pod's imagePullSecrets
// alongside hive's own and the service account's (#18819). Names only.
ImagePullSecrets []string
// ExtraVolumes mounts additional claims (a compile cache, a model store).
ExtraVolumes []VolumeMount
// TTLSeconds is the Job's ttlSecondsAfterFinished (default one hour).
Expand Down Expand Up @@ -266,6 +269,9 @@ func (o Options) Validate() error {
if o.RelayRunAsUser != nil && *o.RelayRunAsUser <= 0 {
return errors.New("kubejob: relay_run_as_user must be a positive (non-root) uid")
}
if err := validatePullSecretNames(o.ImagePullSecrets); err != nil {
return err
}
switch o.network() {
case NetworkDefault, NetworkNone, NetworkHost:
default:
Expand Down Expand Up @@ -308,6 +314,7 @@ func (l *Launcher) Run(ctx context.Context, spec sandbox.LaunchSpec) (sandbox.Re
}
name := jobName(spec.Name, l.now())
manifest := l.jobManifest(ns, name, subPath, spec, relays)
setPullSecrets(manifest, l.resolvePullSecrets(ctx, ns, relays))
if jobSpec, ok := manifest["spec"].(map[string]any); ok {
jobSpec["activeDeadlineSeconds"] = l.activeDeadline(ctx)
}
Expand Down
15 changes: 15 additions & 0 deletions src/crux/kubejob/kubejob_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,8 @@ type fakeAPI struct {
order []string // "policy", "job" in creation order
selfPod string // JSON body of hive's own pod (testSelfPod), "" = 404
selfPodGets int
serviceAccounts map[string]string // SA name -> JSON body; absent = 404
saForbidden bool // SA reads answer 403
}

func newFakeAPI() *fakeAPI {
Expand Down Expand Up @@ -122,6 +124,19 @@ func (f *fakeAPI) handler(t *testing.T) http.HandlerFunc {
f.onPodLogAfterComplete()
}
_, _ = w.Write([]byte(f.logBody))
case r.Method == http.MethodGet && strings.Contains(r.URL.Path, "/serviceaccounts/"):
if f.saForbidden {
w.WriteHeader(http.StatusForbidden)
_, _ = w.Write([]byte(`{"kind":"Status","message":"serviceaccounts is forbidden"}`))
return
}
body, ok := f.serviceAccounts[r.URL.Path[strings.LastIndex(r.URL.Path, "/")+1:]]
if !ok {
w.WriteHeader(http.StatusNotFound)
_, _ = w.Write([]byte(`{"kind":"Status","message":"not found"}`))
return
}
_, _ = w.Write([]byte(body))
case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/pods/"+testSelfPod):
f.selfPodGets++
if f.selfPod == "" {
Expand Down
122 changes: 122 additions & 0 deletions src/crux/kubejob/pullsecrets.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,122 @@
package kubejob

import (
"context"
"encoding/json"
"fmt"
"log/slog"
"net/http"
"net/url"
"regexp"
"strings"
)

// Pull secrets on the Job pod (#18819).
//
// Kubernetes' ServiceAccount admission adds the service account's
// imagePullSecrets only to a pod that has none of its own, so the launcher
// setting the hive pod's list (for the relay init image) hid the service
// account's. The pod now carries the union, in a stable order: hive's own,
// then the service account's, then job.image_pull_secrets. Only Secret NAMES
// are ever read, logged or written.

const (
// maxPullSecrets bounds the merged list.
maxPullSecrets = 64
// maxConfiguredPullSecrets bounds job.image_pull_secrets.
maxConfiguredPullSecrets = 32
maxPullSecretNameLen = 253
)

var pullSecretNameRE = regexp.MustCompile(`^[a-z0-9]([-a-z0-9.]*[a-z0-9])?$`)

// validatePullSecretNames refuses anything but a bounded list of names.
func validatePullSecretNames(names []string) error {
if len(names) > maxConfiguredPullSecrets {
return fmt.Errorf("kubejob: image_pull_secrets takes at most %d names, got %d", maxConfiguredPullSecrets, len(names))
}
for _, n := range names {
if len(n) > maxPullSecretNameLen || !pullSecretNameRE.MatchString(n) {
return fmt.Errorf("kubejob: image_pull_secrets takes Secret names, not values: %q is not a valid name", n)
}
}
return nil
}

// mergePullSecretNames concatenates the lists, dropping blanks and repeats and
// keeping first-seen order.
func mergePullSecretNames(lists ...[]string) []string {
seen := map[string]bool{}
var out []string
for _, list := range lists {
for _, n := range list {
n = strings.TrimSpace(n)
if n == "" || seen[n] || len(out) >= maxPullSecrets {
continue
}
seen[n] = true
out = append(out, n)
}
}
return out
}

// serviceAccountPullSecrets reads the names of the Job service account's
// imagePullSecrets. A launcher whose RBAC lacks `get serviceaccounts` (or any
// other failure) yields a warning and no names, never an error.
func (l *Launcher) serviceAccountPullSecrets(ctx context.Context, ns string) []string {
sa := strings.TrimSpace(l.Options.ServiceAccount)
if sa == "" {
return nil
}
raw, err := l.do(ctx, http.MethodGet, fmt.Sprintf("/api/v1/namespaces/%s/serviceaccounts/%s", ns, url.PathEscape(sa)), nil)
if err != nil {
slog.Warn("kubejob: cannot read the Job service account's imagePullSecrets; set sandbox.job.image_pull_secrets if the sandbox image is in a private registry",
"serviceAccount", sa, "error", err.Error())
return nil
}
var body struct {
ImagePullSecrets []struct {
Name string `json:"name"`
} `json:"imagePullSecrets"`
}
if err := json.Unmarshal(raw, &body); err != nil {
slog.Warn("kubejob: the Job service account did not decode; its imagePullSecrets are skipped", "serviceAccount", sa)
return nil
}
names := make([]string, 0, len(body.ImagePullSecrets))
for _, s := range body.ImagePullSecrets {
names = append(names, s.Name)
}
return names
}

// resolvePullSecrets is the union for one Job pod.
func (l *Launcher) resolvePullSecrets(ctx context.Context, ns string, relays *relayPlan) []string {
var hive []string
if relays != nil {
for _, s := range relays.pullSecrets {
hive = append(hive, s["name"])
}
}
return mergePullSecretNames(hive, l.serviceAccountPullSecrets(ctx, ns), l.Options.ImagePullSecrets)
}

// setPullSecrets writes names into the manifest's pod spec; no names leaves it
// untouched.
func setPullSecrets(manifest map[string]any, names []string) {
if len(names) == 0 {
return
}
spec, _ := manifest["spec"].(map[string]any)
tmpl, _ := spec["template"].(map[string]any)
pod, _ := tmpl["spec"].(map[string]any)
if pod == nil {
return
}
refs := make([]map[string]string, 0, len(names))
for _, n := range names {
refs = append(refs, map[string]string{"name": n})
}
pod["imagePullSecrets"] = refs
}
71 changes: 71 additions & 0 deletions src/crux/kubejob/pullsecrets_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
package kubejob

import (
"strings"
"testing"
)

const saWithSecrets = `{"imagePullSecrets":[{"name":"reg-a"},{"name":"ghcr-pull"}]}`

func TestPullSecretsUnionDedupedAndOrdered(t *testing.T) {
api := newFakeAPI()
api.selfPod = vanillaSelfPod // ghcr-pull
api.serviceAccounts = map[string]string{"job-sa": saWithSecrets}
pod := runRelayJob(t, api, Options{ServiceAccount: "job-sa", ImagePullSecrets: []string{"reg-b", "reg-a"}})
want := `[{"name":"ghcr-pull"},{"name":"reg-a"},{"name":"reg-b"}]`
if got := asJSON(pod["imagePullSecrets"]); got != want {
t.Errorf("imagePullSecrets = %s, want %s", got, want)
}
}

func TestPullSecretsConfiguredOnlyWithoutHivePodSecrets(t *testing.T) {
api := newFakeAPI()
api.selfPod = openshiftSelfPod // no pull secrets
pod := runRelayJob(t, api, Options{ImagePullSecrets: []string{"reg-b"}})
if got := asJSON(pod["imagePullSecrets"]); got != `[{"name":"reg-b"}]` {
t.Errorf("imagePullSecrets = %s, want the configured name", got)
}
}

func TestPullSecretsServiceAccountReadDeniedFallsBack(t *testing.T) {
api := newFakeAPI()
api.selfPod = vanillaSelfPod
api.saForbidden = true
pod := runRelayJob(t, api, Options{ServiceAccount: "job-sa", ImagePullSecrets: []string{"reg-b"}})
if got := asJSON(pod["imagePullSecrets"]); got != `[{"name":"ghcr-pull"},{"name":"reg-b"}]` {
t.Errorf("imagePullSecrets = %s, want hive's plus configured, no error", got)
}
}

func TestPullSecretsNoneLeavesFieldOff(t *testing.T) {
api := newFakeAPI()
api.selfPod = openshiftSelfPod
pod := runRelayJob(t, api, Options{})
if _, ok := pod["imagePullSecrets"]; ok {
t.Errorf("imagePullSecrets = %s, want none", asJSON(pod["imagePullSecrets"]))
}
}

func TestValidatePullSecretNames(t *testing.T) {
if err := validatePullSecretNames([]string{"reg-a", "a.b-c"}); err != nil {
t.Errorf("valid names refused: %v", err)
}
for _, bad := range []string{"KEY=value", "Upper", "has space", "-lead", ""} {
if err := validatePullSecretNames([]string{bad}); err == nil {
t.Errorf("%q accepted", bad)
}
}
if err := validatePullSecretNames(make([]string, maxConfiguredPullSecrets+1)); err == nil {
t.Error("over-long list accepted")
}
if err := (Options{WorkspaceClaim: "c", ImagePullSecrets: []string{"a=b"}}).Validate(); err == nil || !strings.Contains(err.Error(), "names") {
t.Errorf("Validate = %v, want a names-only refusal", err)
}
}

func TestMergePullSecretNames(t *testing.T) {
got := mergePullSecretNames([]string{"a", "b", ""}, []string{"b", "c"}, []string{"a", "d"})
if strings.Join(got, ",") != "a,b,c,d" {
t.Errorf("merge = %v", got)
}
}
5 changes: 5 additions & 0 deletions src/deploy/k8s/sandbox-job-rbac.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,11 @@ rules:
- apiGroups: [""]
resources: ["pods", "pods/log"]
verbs: ["get", "list"]
# Names of the Job service account's imagePullSecrets (#18819). Optional:
# without it hive warns and uses job.image_pull_secrets only.
- apiGroups: [""]
resources: ["serviceaccounts"]
verbs: ["get"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
Expand Down
2 changes: 1 addition & 1 deletion src/docs/agent-configuration.md
Original file line number Diff line number Diff line change
Expand Up @@ -411,7 +411,7 @@ For prompt file resolution and the complete built-in `${VAR}` reference, see

### Sandbox settings (`agents.<name>.sandbox`)

An agent's `sandbox` block (image, runtime, Job fields) is editable from its General tab under "Sandbox settings: image, runtime and Job", and through `PUT /api/config/agent/<name>/general` with `sandboxImage`, `sandboxRuntime`, `sandboxJob` and `sandboxCopyFrom` (hivecommons/hive#18797). A blank value inherits the hive-wide `agent_sandbox` default, which is set under Settings, Security. "Copy sandbox settings from..." copies another agent's complete block, Secret names included and never values. Enabling a sandbox that resolves no image is refused with the field named. The full set of controls and keys is in [sandbox-isolation.md](sandbox-isolation.md#setting-up-a-sandbox-from-the-dashboard).
An agent's `sandbox` block (image, runtime, Job fields) is editable from its General tab under "Sandbox settings: image, runtime and Job", and through `PUT /api/config/agent/<name>/general` with `sandboxImage`, `sandboxRuntime`, `sandboxJob` and `sandboxCopyFrom` (hivecommons/hive#18797). A blank value inherits the hive-wide `agent_sandbox` default, which is set under Settings, Security. "Copy sandbox settings from..." copies another agent's complete block, Secret names included and never values. Enabling a sandbox that resolves no image is refused with the field named. The full set of controls and keys is in [sandbox-isolation.md](sandbox-isolation.md#setting-up-a-sandbox-from-the-dashboard). `sandbox.job.image_pull_secrets` (Secret names only, at most 32; a per-agent block that sets none inherits `agent_sandbox.job`) adds registry pull secrets to the Job pod beside hive's own and the Job service account's; the API field is `sandboxJob.imagePullSecrets` ([#18819](https://github.com/hivecommons/hive/issues/18819), see [Private-registry images](sandbox-isolation.md#running-sandboxed-kicks-as-kubernetes-jobs)).

### Declarative extensions

Expand Down
11 changes: 10 additions & 1 deletion src/docs/sandbox-isolation.md
Original file line number Diff line number Diff line change
Expand Up @@ -247,7 +247,16 @@ agents:

**The terminal pane.** A job-runtime agent has no tmux session in the hive pod, so the dashboard's ▶ terminal does not attach ttyd for it. Instead it opens `/terminal/job/<agent>`, a read-only viewer that streams the newest Job pod's log through the same in-cluster client and RBAC the launcher already uses ([#14798](https://github.com/hivecommons/hive/issues/14798)): it follows the pod while the kick runs (waiting while the pod is `Pending`, e.g. for an accelerator node), prints the container exit code when it ends, and between kicks, when the completed Job has already been deleted, shows the last run's persisted transcript (`<workspace>/.hive/sandbox-transcript.log`, which now ends with an `exit code: N` line) or a clear "no sandbox run yet" state. The pane is observation only; there is no input path into the Job. The same terminal handoff and role check as the tmux pane apply.

**RBAC.** The spoke's ServiceAccount needs a Role for `batch/jobs` (create, get, list, delete) and `pods` plus `pods/log` (get, list); the terminal viewer needs nothing beyond that same `pods/log` grant. The standalone kustomize base ships it as `sandbox-job-rbac.yaml`, and the hub provisioner grants it to every hosted spoke as `hive-sandbox-jobs`.
**Private-registry images (#18819).** The Job pod's `imagePullSecrets` are the deduplicated union, in this order, of: hive's own pod secrets (so the `hive-relays` init container can pull the hive image), the Job service account's `imagePullSecrets`, and `sandbox.job.image_pull_secrets`. Kubernetes adds a service account's pull secrets only to a pod that has none of its own, so hive reads the service account through the API (`get serviceaccounts`) and merges the names itself. When that grant is missing or the read fails, hive logs a warning and carries on with the other two lists; it never fails the kick. `image_pull_secrets` holds Secret **names** only (DNS-1123 names, at most 32; a value is refused). A per-agent `job` block that names none inherits the `agent_sandbox.job` list. Hive reads, logs and writes names, never Secret contents. When `service_account` is set and `image_pull_secrets` is not, the boot and reload warning reminds the operator to set it if the grant is absent. The dashboard sandbox controls do not carry this field yet; the per-agent API accepts `sandboxJob.imagePullSecrets`.

```yaml
agent_sandbox:
job:
service_account: hive-spyre-job
image_pull_secrets: [ registry-pull ]
```

**RBAC.** The spoke's ServiceAccount needs a Role for `batch/jobs` (create, get, list, delete), `pods` plus `pods/log` (get, list) and, optionally, `serviceaccounts` (get) so the Job service account's pull secrets are merged; the terminal viewer needs nothing beyond that same `pods/log` grant. The standalone kustomize base ships it as `sandbox-job-rbac.yaml`, and the hub provisioner grants it to every hosted spoke as `hive-sandbox-jobs`.

Hive warns at boot and on reload (`config.AgentSandboxGateWarnings`) when a job-runtime agent resolves no `workspace_claim` or names a runtime hive does not have, and refuses the kick with the same message rather than failing it after the agent is marked running.

Expand Down
Loading
Loading