diff --git a/changelog.d/added-sandbox-job-scheduler-name.md b/changelog.d/added-sandbox-job-scheduler-name.md new file mode 100644 index 0000000000..c5a78ff4ac --- /dev/null +++ b/changelog.d/added-sandbox-job-scheduler-name.md @@ -0,0 +1 @@ +- `agents..sandbox.job.scheduler_name` sets the Job pod's `schedulerName`, for clusters whose admission webhook requires a custom scheduler for device-requesting pods (e.g. `spyre-scheduler`) diff --git a/src/crux/agent/manager_sandbox.go b/src/crux/agent/manager_sandbox.go index b645a1adc3..0349763a0d 100644 --- a/src/crux/agent/manager_sandbox.go +++ b/src/crux/agent/manager_sandbox.go @@ -285,6 +285,7 @@ func JobOptionsFromConfig(job config.SandboxJobConfig) kubejob.Options { WorkspaceClaimMount: job.WorkspaceClaimMount, NodeSelector: job.NodeSelector, ServiceAccount: job.ServiceAccount, + SchedulerName: job.SchedulerName, EnvFromSecrets: append([]string(nil), job.EnvFromSecrets...), TTLSeconds: job.TTLSeconds, Resources: kubejob.Resources{ diff --git a/src/crux/agent/sandbox_job_runtime_test.go b/src/crux/agent/sandbox_job_runtime_test.go index 911546990a..02010eddc3 100644 --- a/src/crux/agent/sandbox_job_runtime_test.go +++ b/src/crux/agent/sandbox_job_runtime_test.go @@ -140,11 +140,12 @@ func TestJobOptionsFromConfig(t *testing.T) { Tolerations: []config.SandboxJobToleration{{Key: "spyre", Operator: "Exists", Effect: "NoSchedule"}}, Resources: config.SandboxJobResources{Limits: map[string]string{"ibm.com/spyre_pf": "2"}, Requests: map[string]string{"cpu": "4"}}, ServiceAccount: "hive", + SchedulerName: "spyre-scheduler", EnvFromSecrets: []string{"artifact-registry"}, Volumes: []config.SandboxJobVolume{{Name: "cache", Claim: "compile-cache", MountPath: "/cache", ReadOnly: true}}, TTLSeconds: 600, }) - if got.WorkspaceClaim != "hive-data" || got.WorkspaceClaimMount != "/data" || got.ServiceAccount != "hive" || got.TTLSeconds != 600 { + if got.WorkspaceClaim != "hive-data" || got.WorkspaceClaimMount != "/data" || got.ServiceAccount != "hive" || got.SchedulerName != "spyre-scheduler" || got.TTLSeconds != 600 { t.Errorf("scalar fields not carried: %+v", got) } if got.NodeSelector["accelerator"] != "spyre" { diff --git a/src/crux/config/agent_config.go b/src/crux/config/agent_config.go index 303ba1e83c..7c0edcb551 100644 --- a/src/crux/config/agent_config.go +++ b/src/crux/config/agent_config.go @@ -34,6 +34,11 @@ type SandboxJobConfig struct { Tolerations []SandboxJobToleration `yaml:"tolerations,omitempty" json:"tolerations,omitempty"` Resources SandboxJobResources `yaml:"resources,omitempty" json:"resources,omitempty"` ServiceAccount string `yaml:"service_account,omitempty" json:"service_account,omitempty"` + // SchedulerName sets the pod's schedulerName. Accelerator clusters often + // gate device-requesting pods behind a custom scheduler via an admission + // webhook (e.g. `A pod must use "schedulerName: spyre-scheduler"`); + // without this the Job create is denied before any pod exists. + SchedulerName string `yaml:"scheduler_name,omitempty" json:"scheduler_name,omitempty"` EnvFromSecrets []string `yaml:"env_from_secrets,omitempty" json:"env_from_secrets,omitempty"` Volumes []SandboxJobVolume `yaml:"volumes,omitempty" json:"volumes,omitempty"` TTLSeconds int `yaml:"ttl_seconds,omitempty" json:"ttl_seconds,omitempty"` diff --git a/src/crux/kubejob/kubejob.go b/src/crux/kubejob/kubejob.go index e317eb8a5c..ff797aacc5 100644 --- a/src/crux/kubejob/kubejob.go +++ b/src/crux/kubejob/kubejob.go @@ -113,6 +113,10 @@ type Options struct { Tolerations []Toleration Resources Resources ServiceAccount string + // SchedulerName is passed through as the pod's schedulerName, for clusters + // whose admission webhook requires device-requesting pods to use a custom + // scheduler. Empty leaves the cluster default. + SchedulerName string // EnvFromSecrets are Secret names whose keys become environment variables // in the Job. This is the ONLY credential path into the Job. EnvFromSecrets []string @@ -461,6 +465,9 @@ func (l *Launcher) manifest(ns, name, subPath string, spec sandbox.LaunchSpec) m if l.Options.ServiceAccount != "" { podSpec["serviceAccountName"] = l.Options.ServiceAccount } + if l.Options.SchedulerName != "" { + podSpec["schedulerName"] = l.Options.SchedulerName + } labels := map[string]string{managedByLabel: managedByValue} if spec.Name != "" { labels[agentLabel] = sanitizeLabel(spec.Name) diff --git a/src/crux/kubejob/kubejob_test.go b/src/crux/kubejob/kubejob_test.go index 8f2d92dce8..f1cb022d49 100644 --- a/src/crux/kubejob/kubejob_test.go +++ b/src/crux/kubejob/kubejob_test.go @@ -304,6 +304,7 @@ func TestManifestShape(t *testing.T) { Tolerations: []Toleration{{Key: "spyre", Operator: "Exists", Effect: "NoSchedule"}}, Resources: Resources{Limits: map[string]string{"ibm.com/spyre_pf": "2"}}, ServiceAccount: "hive", + SchedulerName: "spyre-scheduler", EnvFromSecrets: []string{"artifact-registry"}, ExtraVolumes: []VolumeMount{{Claim: "compile-cache", MountPath: "/cache", ReadOnly: true}}, }) @@ -318,6 +319,7 @@ func TestManifestShape(t *testing.T) { `"ttlSecondsAfterFinished":3600`, `"restartPolicy":"Never"`, `"nodeSelector":{"accelerator":"spyre"}`, + `"schedulerName":"spyre-scheduler"`, // The fake decodes the manifest into a map before we re-marshal it // here, so keys come back alphabetical regardless of struct order. `"tolerations":[{"effect":"NoSchedule","key":"spyre","operator":"Exists"}]`, diff --git a/src/docs/sandbox-isolation.md b/src/docs/sandbox-isolation.md index 06886ed2f6..5f6b4975bc 100644 --- a/src/docs/sandbox-isolation.md +++ b/src/docs/sandbox-isolation.md @@ -154,6 +154,7 @@ agents: limits: { "vendor.com/device": "2" } tolerations: - { key: accelerator, operator: Exists, effect: NoSchedule } + scheduler_name: spyre-scheduler # only if the cluster's admission webhook requires a custom scheduler for device pods env_from_secrets: [ artifact-registry ] volumes: - { name: cache, claim: compile-cache, mount_path: /cache } diff --git a/src/pkg/dashboard/api_governor_security.go b/src/pkg/dashboard/api_governor_security.go index 24bb0102ba..e8b0f4509f 100644 --- a/src/pkg/dashboard/api_governor_security.go +++ b/src/pkg/dashboard/api_governor_security.go @@ -50,6 +50,7 @@ func (s *Server) handleGovernorSecurity(w http.ResponseWriter, r *http.Request) SandboxWorkspaceDir *string `json:"agentSandboxWorkspaceDir"` SandboxRuntime *string `json:"agentSandboxRuntime"` SandboxJobServiceAccount *string `json:"agentSandboxJobServiceAccount"` + SandboxJobSchedulerName *string `json:"agentSandboxJobSchedulerName"` SandboxJobTTLSeconds *int `json:"agentSandboxJobTtlSeconds"` SandboxJobWorkspaceClaim *string `json:"agentSandboxJobWorkspaceClaim"` SandboxJobClaimMount *string `json:"agentSandboxJobWorkspaceClaimMount"` @@ -241,7 +242,7 @@ func (s *Server) handleGovernorSecurity(w http.ResponseWriter, r *http.Request) if body.SandboxTimeoutS != nil { cfg.AgentSandbox.TimeoutS = *body.SandboxTimeoutS } - if body.SandboxJobServiceAccount != nil || body.SandboxJobTTLSeconds != nil || + if body.SandboxJobServiceAccount != nil || body.SandboxJobSchedulerName != nil || body.SandboxJobTTLSeconds != nil || body.SandboxJobWorkspaceClaim != nil || body.SandboxJobClaimMount != nil { if cfg.AgentSandbox.Job == nil { cfg.AgentSandbox.Job = &config.SandboxJobConfig{} @@ -250,6 +251,9 @@ func (s *Server) handleGovernorSecurity(w http.ResponseWriter, r *http.Request) if body.SandboxJobServiceAccount != nil { job.ServiceAccount = sanitizeString(strings.TrimSpace(*body.SandboxJobServiceAccount)) } + if body.SandboxJobSchedulerName != nil { + job.SchedulerName = sanitizeString(strings.TrimSpace(*body.SandboxJobSchedulerName)) + } if body.SandboxJobWorkspaceClaim != nil { job.WorkspaceClaim = sanitizeString(strings.TrimSpace(*body.SandboxJobWorkspaceClaim)) } @@ -349,6 +353,7 @@ func securitySectionResponse(cfg *config.Config) map[string]interface{} { "agentSandboxWorkspaceDir": cfg.AgentSandbox.WorkspaceDir, "agentSandboxRuntime": cfg.AgentSandbox.Runtime, "agentSandboxJobServiceAccount": sandboxJobOrEmpty(cfg).ServiceAccount, + "agentSandboxJobSchedulerName": sandboxJobOrEmpty(cfg).SchedulerName, "agentSandboxJobTtlSeconds": sandboxJobOrEmpty(cfg).TTLSeconds, "agentSandboxJobWorkspaceClaim": sandboxJobOrEmpty(cfg).WorkspaceClaim, "agentSandboxJobWorkspaceClaimMount": sandboxJobOrEmpty(cfg).WorkspaceClaimMount, diff --git a/src/pkg/dashboard/settings_parity_allowlist.txt b/src/pkg/dashboard/settings_parity_allowlist.txt index f02c8b8942..4a8883df2a 100644 --- a/src/pkg/dashboard/settings_parity_allowlist.txt +++ b/src/pkg/dashboard/settings_parity_allowlist.txt @@ -99,6 +99,7 @@ agents..sandbox.job.env_from_secrets[] # TODO #11854 agents..sandbox.job.node_selector. # TODO #11854 agents..sandbox.job.resources.limits. # TODO #11854 agents..sandbox.job.resources.requests. # TODO #11854 +agents..sandbox.job.scheduler_name # TODO #11854 agents..sandbox.job.service_account # TODO #11854 agents..sandbox.job.tolerations[].effect # TODO #11854 agents..sandbox.job.tolerations[].key # TODO #11854 diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html index 31b1bb451f..74e8ef6c6d 100644 --- a/src/pkg/dashboard/static/index.html +++ b/src/pkg/dashboard/static/index.html @@ -49210,6 +49210,7 @@

Escalation Breaker

+