Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/added-sandbox-job-scheduler-name.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- `agents.<name>.sandbox.job.scheduler_name` sets the Job pod's `schedulerName`, for clusters whose admission webhook requires a custom scheduler for device-requesting pods (e.g. `spyre-scheduler`)
1 change: 1 addition & 0 deletions src/crux/agent/manager_sandbox.go
Original file line number Diff line number Diff line change
Expand Up @@ -285,6 +285,7 @@ func JobOptionsFromConfig(job config.SandboxJobConfig) kubejob.Options {
WorkspaceClaimMount: job.WorkspaceClaimMount,
NodeSelector: job.NodeSelector,
ServiceAccount: job.ServiceAccount,
SchedulerName: job.SchedulerName,
EnvFromSecrets: append([]string(nil), job.EnvFromSecrets...),
TTLSeconds: job.TTLSeconds,
Resources: kubejob.Resources{
Expand Down
3 changes: 2 additions & 1 deletion src/crux/agent/sandbox_job_runtime_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -140,11 +140,12 @@ func TestJobOptionsFromConfig(t *testing.T) {
Tolerations: []config.SandboxJobToleration{{Key: "spyre", Operator: "Exists", Effect: "NoSchedule"}},
Resources: config.SandboxJobResources{Limits: map[string]string{"ibm.com/spyre_pf": "2"}, Requests: map[string]string{"cpu": "4"}},
ServiceAccount: "hive",
SchedulerName: "spyre-scheduler",
EnvFromSecrets: []string{"artifact-registry"},
Volumes: []config.SandboxJobVolume{{Name: "cache", Claim: "compile-cache", MountPath: "/cache", ReadOnly: true}},
TTLSeconds: 600,
})
if got.WorkspaceClaim != "hive-data" || got.WorkspaceClaimMount != "/data" || got.ServiceAccount != "hive" || got.TTLSeconds != 600 {
if got.WorkspaceClaim != "hive-data" || got.WorkspaceClaimMount != "/data" || got.ServiceAccount != "hive" || got.SchedulerName != "spyre-scheduler" || got.TTLSeconds != 600 {
t.Errorf("scalar fields not carried: %+v", got)
}
if got.NodeSelector["accelerator"] != "spyre" {
Expand Down
5 changes: 5 additions & 0 deletions src/crux/config/agent_config.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,11 @@ type SandboxJobConfig struct {
Tolerations []SandboxJobToleration `yaml:"tolerations,omitempty" json:"tolerations,omitempty"`
Resources SandboxJobResources `yaml:"resources,omitempty" json:"resources,omitempty"`
ServiceAccount string `yaml:"service_account,omitempty" json:"service_account,omitempty"`
// SchedulerName sets the pod's schedulerName. Accelerator clusters often
// gate device-requesting pods behind a custom scheduler via an admission
// webhook (e.g. `A pod must use "schedulerName: spyre-scheduler"`);
// without this the Job create is denied before any pod exists.
SchedulerName string `yaml:"scheduler_name,omitempty" json:"scheduler_name,omitempty"`
EnvFromSecrets []string `yaml:"env_from_secrets,omitempty" json:"env_from_secrets,omitempty"`
Volumes []SandboxJobVolume `yaml:"volumes,omitempty" json:"volumes,omitempty"`
TTLSeconds int `yaml:"ttl_seconds,omitempty" json:"ttl_seconds,omitempty"`
Expand Down
7 changes: 7 additions & 0 deletions src/crux/kubejob/kubejob.go
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,10 @@ type Options struct {
Tolerations []Toleration
Resources Resources
ServiceAccount string
// SchedulerName is passed through as the pod's schedulerName, for clusters
// whose admission webhook requires device-requesting pods to use a custom
// scheduler. Empty leaves the cluster default.
SchedulerName string
// EnvFromSecrets are Secret names whose keys become environment variables
// in the Job. This is the ONLY credential path into the Job.
EnvFromSecrets []string
Expand Down Expand Up @@ -461,6 +465,9 @@ func (l *Launcher) manifest(ns, name, subPath string, spec sandbox.LaunchSpec) m
if l.Options.ServiceAccount != "" {
podSpec["serviceAccountName"] = l.Options.ServiceAccount
}
if l.Options.SchedulerName != "" {
podSpec["schedulerName"] = l.Options.SchedulerName
}
labels := map[string]string{managedByLabel: managedByValue}
if spec.Name != "" {
labels[agentLabel] = sanitizeLabel(spec.Name)
Expand Down
2 changes: 2 additions & 0 deletions src/crux/kubejob/kubejob_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -304,6 +304,7 @@ func TestManifestShape(t *testing.T) {
Tolerations: []Toleration{{Key: "spyre", Operator: "Exists", Effect: "NoSchedule"}},
Resources: Resources{Limits: map[string]string{"ibm.com/spyre_pf": "2"}},
ServiceAccount: "hive",
SchedulerName: "spyre-scheduler",
EnvFromSecrets: []string{"artifact-registry"},
ExtraVolumes: []VolumeMount{{Claim: "compile-cache", MountPath: "/cache", ReadOnly: true}},
})
Expand All @@ -318,6 +319,7 @@ func TestManifestShape(t *testing.T) {
`"ttlSecondsAfterFinished":3600`,
`"restartPolicy":"Never"`,
`"nodeSelector":{"accelerator":"spyre"}`,
`"schedulerName":"spyre-scheduler"`,
// The fake decodes the manifest into a map before we re-marshal it
// here, so keys come back alphabetical regardless of struct order.
`"tolerations":[{"effect":"NoSchedule","key":"spyre","operator":"Exists"}]`,
Expand Down
1 change: 1 addition & 0 deletions src/docs/sandbox-isolation.md
Original file line number Diff line number Diff line change
Expand Up @@ -154,6 +154,7 @@ agents:
limits: { "vendor.com/device": "2" }
tolerations:
- { key: accelerator, operator: Exists, effect: NoSchedule }
scheduler_name: spyre-scheduler # only if the cluster's admission webhook requires a custom scheduler for device pods
env_from_secrets: [ artifact-registry ]
volumes:
- { name: cache, claim: compile-cache, mount_path: /cache }
Expand Down
7 changes: 6 additions & 1 deletion src/pkg/dashboard/api_governor_security.go
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
package dashboard

import (
Expand Down Expand Up @@ -50,6 +50,7 @@
SandboxWorkspaceDir *string `json:"agentSandboxWorkspaceDir"`
SandboxRuntime *string `json:"agentSandboxRuntime"`
SandboxJobServiceAccount *string `json:"agentSandboxJobServiceAccount"`
SandboxJobSchedulerName *string `json:"agentSandboxJobSchedulerName"`
SandboxJobTTLSeconds *int `json:"agentSandboxJobTtlSeconds"`
SandboxJobWorkspaceClaim *string `json:"agentSandboxJobWorkspaceClaim"`
SandboxJobClaimMount *string `json:"agentSandboxJobWorkspaceClaimMount"`
Expand Down Expand Up @@ -241,7 +242,7 @@
if body.SandboxTimeoutS != nil {
cfg.AgentSandbox.TimeoutS = *body.SandboxTimeoutS
}
if body.SandboxJobServiceAccount != nil || body.SandboxJobTTLSeconds != nil ||
if body.SandboxJobServiceAccount != nil || body.SandboxJobSchedulerName != nil || body.SandboxJobTTLSeconds != nil ||
body.SandboxJobWorkspaceClaim != nil || body.SandboxJobClaimMount != nil {
if cfg.AgentSandbox.Job == nil {
cfg.AgentSandbox.Job = &config.SandboxJobConfig{}
Expand All @@ -250,6 +251,9 @@
if body.SandboxJobServiceAccount != nil {
job.ServiceAccount = sanitizeString(strings.TrimSpace(*body.SandboxJobServiceAccount))
}
if body.SandboxJobSchedulerName != nil {
job.SchedulerName = sanitizeString(strings.TrimSpace(*body.SandboxJobSchedulerName))
}
if body.SandboxJobWorkspaceClaim != nil {
job.WorkspaceClaim = sanitizeString(strings.TrimSpace(*body.SandboxJobWorkspaceClaim))
}
Expand Down Expand Up @@ -349,6 +353,7 @@
"agentSandboxWorkspaceDir": cfg.AgentSandbox.WorkspaceDir,
"agentSandboxRuntime": cfg.AgentSandbox.Runtime,
"agentSandboxJobServiceAccount": sandboxJobOrEmpty(cfg).ServiceAccount,
"agentSandboxJobSchedulerName": sandboxJobOrEmpty(cfg).SchedulerName,
"agentSandboxJobTtlSeconds": sandboxJobOrEmpty(cfg).TTLSeconds,
"agentSandboxJobWorkspaceClaim": sandboxJobOrEmpty(cfg).WorkspaceClaim,
"agentSandboxJobWorkspaceClaimMount": sandboxJobOrEmpty(cfg).WorkspaceClaimMount,
Expand Down
1 change: 1 addition & 0 deletions src/pkg/dashboard/settings_parity_allowlist.txt
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,7 @@ agents.<key>.sandbox.job.env_from_secrets[] # TODO #11854
agents.<key>.sandbox.job.node_selector.<key> # TODO #11854
agents.<key>.sandbox.job.resources.limits.<key> # TODO #11854
agents.<key>.sandbox.job.resources.requests.<key> # TODO #11854
agents.<key>.sandbox.job.scheduler_name # TODO #11854
agents.<key>.sandbox.job.service_account # TODO #11854
agents.<key>.sandbox.job.tolerations[].effect # TODO #11854
agents.<key>.sandbox.job.tolerations[].key # TODO #11854
Expand Down
1 change: 1 addition & 0 deletions src/pkg/dashboard/static/index.html
Original file line number Diff line number Diff line change
Expand Up @@ -49210,6 +49210,7 @@ <h4 style="margin:18px 0 8px">Escalation Breaker</h4>
<div class="config-field"><label>Workspace dir ${yamlHint('agent_sandbox.workspace_dir')} ${settingHelpMark('settings', 'security-and-access', 'Workspace dir', `Directory mounted as the sandbox workspace.`)}</label><input type="text" value="${esc(sec.agentSandboxWorkspaceDir || '')}" placeholder="default workspace dir" data-change-action="markDirty" data-arg0="security" data-arg1="agentSandboxWorkspaceDir" data-arg-types="s,s,v"></div>
<div class="config-field"><label>Env allowlist ${yamlHint('agent_sandbox.env_allowlist[]')} ${settingHelpMark('settings', 'security-and-access', 'Sandbox env allowlist', `Environment variable names passed into the sandbox. One per line.`)}</label><textarea rows="3" class="mono" data-change-action="markSecurityLines" data-arg0="agentSandboxEnvAllowlist" data-arg-types="s,v">${esc((sec.agentSandboxEnvAllowlist || []).join('\n'))}</textarea></div>
<div class="config-field"><label>Job service account ${yamlHint('agent_sandbox.job.service_account')} ${settingHelpMark('settings', 'security-and-access', 'Job service account', `Kubernetes service account for sandbox Jobs.`)}</label><input type="text" value="${esc(sec.agentSandboxJobServiceAccount || '')}" placeholder="service account name" data-change-action="markDirty" data-arg0="security" data-arg1="agentSandboxJobServiceAccount" data-arg-types="s,s,v"></div>
<div class="config-field"><label>Job scheduler name ${yamlHint('agent_sandbox.job.scheduler_name')} ${settingHelpMark('settings', 'security-and-access', 'Job scheduler name', `Kubernetes schedulerName for sandbox Job pods. Leave empty for the cluster default; set when an admission webhook requires a custom scheduler for device-requesting pods (e.g. spyre-scheduler).`)}</label><input type="text" value="${esc(sec.agentSandboxJobSchedulerName || '')}" placeholder="cluster default" data-change-action="markDirty" data-arg0="security" data-arg1="agentSandboxJobSchedulerName" data-arg-types="s,s,v"></div>
<div class="config-field"><label>Job TTL (seconds) ${yamlHint('agent_sandbox.job.ttl_seconds')} ${settingHelpMark('settings', 'security-and-access', 'Job TTL (seconds)', `Seconds a finished sandbox Job is kept. 0 uses the default.`)}</label><input type="number" min="0" value="${Number(sec.agentSandboxJobTtlSeconds || 0)}" data-change-action="markSecurityNumber" data-arg0="agentSandboxJobTtlSeconds" data-arg-types="s,v"></div>
<div class="config-field"><label>Job workspace claim ${yamlHint('agent_sandbox.job.workspace_claim')} ${settingHelpMark('settings', 'security-and-access', 'Job workspace claim', `PVC holding the workspace for the job runtime. Required for runtime job.`)}</label><input type="text" value="${esc(sec.agentSandboxJobWorkspaceClaim || '')}" placeholder="PersistentVolumeClaim name" data-change-action="markDirty" data-arg0="security" data-arg1="agentSandboxJobWorkspaceClaim" data-arg-types="s,s,v"></div>
<div class="config-field"><label>Job workspace claim mount ${yamlHint('agent_sandbox.job.workspace_claim_mount')} ${settingHelpMark('settings', 'security-and-access', 'Job workspace claim mount', `Where the workspace claim is mounted in the Job.`)}</label><input type="text" value="${esc(sec.agentSandboxJobWorkspaceClaimMount || '')}" placeholder="mount path" data-change-action="markDirty" data-arg0="security" data-arg1="agentSandboxJobWorkspaceClaimMount" data-arg-types="s,s,v"></div>
Expand Down
Loading