From 3c9f3473faba1fa98d843cb13ca966f5c8888714 Mon Sep 17 00:00:00 2001 From: Andrew Anderson Date: Tue, 17 Nov 2026 23:25:18 -0500 Subject: [PATCH 1/2] =?UTF-8?q?=E2=9C=A8=20feature:=20sandbox=20node=20pre?= =?UTF-8?q?flight=20and=20fail-fast=20on=20an=20unschedulable=20Job?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add sandbox.preflight: configurable commands run in the Job's main container before the agent. A failure ends the kick in seconds as node_unhealthy with the node recorded, retries once with a hostname NotIn affinity, and is not an agent failure. Add job.schedule_timeout_s (default 10m): a pod Unschedulable that long fails the kick as waiting_for_node with the scheduler's message, deletes the Job and releases the agent. The dashboard shows both states. Fixes #18811 Signed-off-by: Andrew Anderson --- ...8811-sandbox-preflight-schedule-timeout.md | 1 + src/crux/agent/manager_pane_signals.go | 1 + src/crux/agent/manager_sandbox.go | 9 + src/crux/agent/manager_sandbox_job.go | 122 ++++ src/crux/agent/sandbox_job_preflight_test.go | 225 ++++++++ src/crux/agent/sandbox_run.go | 2 + src/crux/config/agent_config.go | 37 +- src/crux/config/sandbox_job_types.go | 26 + src/crux/config/sandbox_preflight.go | 116 ++++ src/crux/config/sandbox_preflight_test.go | 94 ++++ src/crux/kubejob/attempt.go | 182 ++++++ src/crux/kubejob/kubejob.go | 281 +-------- src/crux/kubejob/kubejob_names.go | 61 ++ src/crux/kubejob/kubejob_test.go | 50 +- src/crux/kubejob/kubejob_types.go | 66 +++ src/crux/kubejob/preflight.go | 238 ++++++++ src/crux/kubejob/preflight_test.go | 532 ++++++++++++++++++ src/crux/kubejob/sched.go | 197 +++++++ src/crux/process.go | 1 + src/crux/process_types.go | 16 + src/crux/sandbox/failreason.go | 33 ++ src/crux/sandbox/failreason_test.go | 24 + src/crux/sandbox/sandbox.go | 7 + src/docs/sandbox-isolation.md | 36 +- src/pkg/config/crux_aliases.go | 1 + src/pkg/config/sandbox_preflight_test.go | 88 +++ src/pkg/config/validate.go | 3 + src/pkg/config/validate_sandbox_preflight.go | 38 ++ src/pkg/dashboard/meetings_hive_agents.go | 2 +- src/pkg/dashboard/sandbox_health.go | 25 + src/pkg/dashboard/sandbox_health_test.go | 27 + .../dashboard/settings_parity_allowlist.txt | 10 + .../dashboard/static-src/css/agent-fields.css | 1 + .../dashboard/static-src/js/agent-cards.js | 9 +- src/pkg/dashboard/static/index.html | 10 +- 35 files changed, 2279 insertions(+), 292 deletions(-) create mode 100644 changelog.d/added-18811-sandbox-preflight-schedule-timeout.md create mode 100644 src/crux/agent/manager_sandbox_job.go create mode 100644 src/crux/agent/sandbox_job_preflight_test.go create mode 100644 src/crux/config/sandbox_job_types.go create mode 100644 src/crux/config/sandbox_preflight.go create mode 100644 src/crux/config/sandbox_preflight_test.go create mode 100644 src/crux/kubejob/attempt.go create mode 100644 src/crux/kubejob/kubejob_names.go create mode 100644 src/crux/kubejob/kubejob_types.go create mode 100644 src/crux/kubejob/preflight.go create mode 100644 src/crux/kubejob/preflight_test.go create mode 100644 src/crux/kubejob/sched.go create mode 100644 src/crux/sandbox/failreason.go create mode 100644 src/crux/sandbox/failreason_test.go create mode 100644 src/pkg/config/sandbox_preflight_test.go create mode 100644 src/pkg/config/validate_sandbox_preflight.go diff --git a/changelog.d/added-18811-sandbox-preflight-schedule-timeout.md b/changelog.d/added-18811-sandbox-preflight-schedule-timeout.md new file mode 100644 index 0000000000..6c9d66b149 --- /dev/null +++ b/changelog.d/added-18811-sandbox-preflight-schedule-timeout.md @@ -0,0 +1 @@ +- Add `sandbox.preflight`, a configurable node health check (for example a one-line device smoke test) that runs in a Job-runtime sandbox kick before the agent starts and, when it fails, ends the kick in seconds as `node_unhealthy` with the node recorded and retries once on a different node via a hostname `NotIn` affinity, without counting as an agent failure; and add `job.schedule_timeout_s` (default 10 minutes), which fails a kick whose Job pod stays Unschedulable as `waiting_for_node` with the scheduler's message, deletes the Job and releases the agent instead of waiting out the 45 minute timeout; the dashboard shows "waiting for a node" and "node unhealthy" as their own states. Preflight is off unless configured (#18811). diff --git a/src/crux/agent/manager_pane_signals.go b/src/crux/agent/manager_pane_signals.go index 4a7b1f57a8..ac5180495d 100644 --- a/src/crux/agent/manager_pane_signals.go +++ b/src/crux/agent/manager_pane_signals.go @@ -110,6 +110,7 @@ func (a *AgentProcess) snapshot() crux.Process { LastError: a.LastError, KickOutcome: a.KickOutcome, SandboxLastRun: cloneSandboxRun(a.SandboxLastRun), + SandboxPhase: sandboxPhaseOf(a), NeedsLogin: needsLogin, LoginURL: loginURL, Starting: a.startupLaunchQueued || a.launching, diff --git a/src/crux/agent/manager_sandbox.go b/src/crux/agent/manager_sandbox.go index 8704ad0c76..6b91268843 100644 --- a/src/crux/agent/manager_sandbox.go +++ b/src/crux/agent/manager_sandbox.go @@ -174,6 +174,7 @@ func (m *Manager) startSandboxKickLocked(agent *AgentProcess, message string) er case config.SandboxRuntimeSubstrate: launcher = m.substrateLauncherLocked(agent.Config.SandboxSubstrate(m.sandboxConfig)) } + m.prepareJobKickLocked(launcher, agent, runtime) repo := m.project.PrimaryRepo() now := time.Now() agent.State = StateRunning @@ -338,6 +339,7 @@ func (m *Manager) runSandboxKick(ctx context.Context, name, runtime string, spec } agent.cancel = nil agent.sandboxKickActive = false + clearSandboxPhase(agent) // The run record is what a sandboxed agent's health and activity derive // from: there is no session to probe between kicks. Recorded on every // outcome but an operator cancel, and persisted so a pod roll does not @@ -375,6 +377,9 @@ func (m *Manager) runSandboxKick(ctx context.Context, name, runtime string, spec m.auditSandbox(name, "sandbox_cancelled", "sandbox kick cancelled while resuming") return } + if m.sandboxInfraFailureLocked(agent, name, res, err) { + return + } if agent.Paused || agent.State == StatePaused { agent.State = StatePaused } else if agent.State != StateStopped { @@ -411,6 +416,9 @@ func (m *Manager) runSandboxKick(ctx context.Context, name, runtime string, spec agent.OutputBuffer.Write(msg) } detail := fmt.Sprintf("workspace=%s commits=%d branch=%s", res.Workspace, res.CommitCount, res.Branch) + if res.Sandbox.Node != "" { + detail += " node=" + res.Sandbox.Node + } if res.PR != nil && res.PR.URL != "" { detail += " pr=" + res.PR.URL } @@ -537,6 +545,7 @@ func JobOptionsFromConfig(job config.SandboxJobConfig) kubejob.Options { EnvFromSecrets: append([]string(nil), job.EnvFromSecrets...), ImagePullSecrets: append([]string(nil), job.ImagePullSecrets...), TTLSeconds: job.TTLSeconds, + ScheduleTimeout: time.Duration(job.ScheduleTimeoutS) * time.Second, RelayImage: job.RelayImage, RelayRunAsUser: job.RelayRunAsUser, Resources: kubejob.Resources{ diff --git a/src/crux/agent/manager_sandbox_job.go b/src/crux/agent/manager_sandbox_job.go new file mode 100644 index 0000000000..ba87473bb7 --- /dev/null +++ b/src/crux/agent/manager_sandbox_job.go @@ -0,0 +1,122 @@ +package agent + +import ( + "fmt" + "sync" + "time" + + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/crux/config" + "github.com/hivecommons/hive/crux/kubejob" + "github.com/hivecommons/hive/crux/sandbox" +) + +// Audit actions for the Job runtime's infrastructure failures +// (hivecommons/hive#18811). Each detail names the node when it is known. +const ( + auditSandboxNodeUnhealthy = "sandbox_node_unhealthy" + auditSandboxNodeRetry = "sandbox_node_unhealthy_retry" + auditSandboxWaitingForNode = "sandbox_waiting_for_node" + auditSandboxWaitingForNodeFail = "sandbox_waiting_for_node_failed" +) + +// sandboxPhases holds the live phase of each in-flight sandboxed kick, keyed by +// the agent. It lives beside AgentProcess rather than on it because manager.go +// is at its crux file-size ratchet; every read and write is under m.mu anyway, +// the sync.Map only keeps the package-level table race-free across Managers. +var sandboxPhases sync.Map + +func setSandboxPhase(a *AgentProcess, p crux.SandboxPhase) { sandboxPhases.Store(a, p) } + +func clearSandboxPhase(a *AgentProcess) { sandboxPhases.Delete(a) } + +// sandboxPhaseOf returns a copy of the agent's live phase, or nil. +func sandboxPhaseOf(a *AgentProcess) *crux.SandboxPhase { + v, ok := sandboxPhases.Load(a) + if !ok { + return nil + } + p, ok := v.(crux.SandboxPhase) + if !ok { + return nil + } + return &p +} + +// prepareJobKickLocked wires sandbox.preflight and the scheduling events into +// the launcher of a job-runtime kick. A test factory's launcher is left alone. +// A preflight configured for another runtime is reported and ignored: it only +// has a meaning on the Job runtime, where it runs in the Job's own container. +func (m *Manager) prepareJobKickLocked(launcher sandbox.Launcher, agent *AgentProcess, runtime string) { + preflight := agent.Config.SandboxPreflight(m.sandboxConfig) + if runtime != config.SandboxRuntimeJob { + if len(preflight.Commands) > 0 { + m.logger.Warn("sandbox.preflight is set but the runtime is not job; the preflight is ignored", "name", agent.Name, "runtime", runtime) + } + return + } + kl, ok := launcher.(*kubejob.Launcher) + if !ok { + return + } + kl.Options.Preflight = preflight.Commands + kl.Options.PreflightTimeout = time.Duration(preflight.TimeoutS) * time.Second + kl.Options.PreflightTotalTimeout = time.Duration(preflight.TotalTimeoutS) * time.Second + kl.Options.PreflightRetryDelay = time.Duration(preflight.RetryDelayS) * time.Second + name := agent.Name + kl.Options.OnEvent = func(e kubejob.Event) { m.onJobEvent(name, e) } +} + +// onJobEvent records what the launcher reports while a Job is placed: the +// pod waiting for a node (shown on the dashboard until it lands) and a +// preflight failure that is being retried on another node. +func (m *Manager) onJobEvent(name string, e kubejob.Event) { + m.mu.Lock() + defer m.mu.Unlock() + agent, ok := m.agents[name] + if !ok { + return + } + switch e.Kind { + case kubejob.EventWaitingForNode: + setSandboxPhase(agent, crux.SandboxPhase{Phase: sandbox.FailReasonWaitingForNode, Detail: e.Detail, Since: time.Now()}) + if agent.OutputBuffer != nil { + agent.OutputBuffer.Write("sandbox Job waiting for a node: " + e.Detail) + } + m.auditSandbox(name, auditSandboxWaitingForNode, "reason="+e.Detail) + case kubejob.EventScheduled: + clearSandboxPhase(agent) + case kubejob.EventNodeRetry: + if agent.OutputBuffer != nil { + agent.OutputBuffer.Write("sandbox node preflight failed on " + e.Node + "; retrying on another node") + } + m.auditSandbox(name, auditSandboxNodeRetry, fmt.Sprintf("node=%s reason=%s", e.Node, e.Detail)) + } +} + +// sandboxInfraFailureLocked handles a kick that failed for the infrastructure's +// reason (node_unhealthy, waiting_for_node): the agent never ran, so it goes +// back to Idle with the reason, not to Failed, and nothing that watches Failed +// agents (restart, escalation, a failure-fix loop) sees it. The next cadence +// tries again. It reports whether it handled the failure. +func (m *Manager) sandboxInfraFailureLocked(agent *AgentProcess, name string, res SandboxKickResult, err error) bool { + reason := res.Sandbox.FailReason + if reason == "" { + return false + } + if agent.Paused || agent.State == StatePaused { + agent.State = StatePaused + } else if agent.State != StateStopped { + agent.State = StateIdle + } + agent.LastError = err.Error() + if agent.OutputBuffer != nil { + agent.OutputBuffer.Write("sandbox kick not run: " + err.Error()) + } + action := auditSandboxNodeUnhealthy + if reason == sandbox.FailReasonWaitingForNode { + action = auditSandboxWaitingForNodeFail + } + m.auditSandbox(name, action, fmt.Sprintf("node=%s reason=%s workspace=%s", res.Sandbox.Node, err.Error(), res.Workspace)) + return true +} diff --git a/src/crux/agent/sandbox_job_preflight_test.go b/src/crux/agent/sandbox_job_preflight_test.go new file mode 100644 index 0000000000..ff6c845246 --- /dev/null +++ b/src/crux/agent/sandbox_job_preflight_test.go @@ -0,0 +1,225 @@ +package agent + +import ( + "context" + "errors" + "strings" + "sync" + "testing" + "time" + + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/crux/config" + "github.com/hivecommons/hive/crux/kubejob" + "github.com/hivecommons/hive/crux/sandbox" +) + +// auditLog collects sandbox audit entries from the kick goroutine. +type auditLog struct { + mu sync.Mutex + entries []string +} + +func (a *auditLog) add(agent, action, detail string) { + a.mu.Lock() + defer a.mu.Unlock() + a.entries = append(a.entries, agent+":"+action+":"+detail) +} + +func (a *auditLog) find(action string) string { + a.mu.Lock() + defer a.mu.Unlock() + for _, e := range a.entries { + if strings.Contains(e, ":"+action+":") { + return e + } + } + return "" +} + +// infraLauncher is a Job launcher whose run fails for the infrastructure's +// reason, as kubejob reports it. +type infraLauncher struct { + res sandbox.Result + err error +} + +func (l infraLauncher) Run(context.Context, sandbox.LaunchSpec) (sandbox.Result, error) { + return l.res, l.err +} + +func infraManager(t *testing.T, launcher sandbox.Launcher) (*Manager, *auditLog) { + t.Helper() + yes := true + cfg := map[string]config.AgentConfig{"deps": {Backend: "claude", Sandbox: &config.AgentSandboxOverride{Enabled: &yes, Runtime: config.SandboxRuntimeJob}}} + m := NewManager(cfg, quietTestLogger(), ProjectContext{Org: "torch-spyre", Repos: []string{"spyre-inference"}}) + m.SetSandboxConfig(config.AgentSandboxConfig{Enabled: true, Image: "dev-image", WorkspaceDir: t.TempDir(), + Job: &config.SandboxJobConfig{WorkspaceClaim: "hive-data"}}) + audits := &auditLog{} + m.SetSandboxAuditCallback(audits.add) + m.setSandboxJobLauncherFactoryForTest(func(config.SandboxJobConfig) sandbox.Launcher { return launcher }) + m.setSandboxRunnerForTest(&sandboxFakeRunner{}) + if err := m.Start(context.Background(), "deps"); err != nil { + t.Fatalf("Start: %v", err) + } + return m, audits +} + +func waitLastRun(t *testing.T, m *Manager) *crux.Process { + t.Helper() + deadline := time.Now().Add(3 * time.Second) + for time.Now().Before(deadline) { + if p := m.AllStatuses()["deps"]; p.SandboxLastRun != nil && p.State != StateRunning { + return p + } + time.Sleep(10 * time.Millisecond) + } + t.Fatalf("the kick never finished: %+v", m.AllStatuses()["deps"]) + return nil +} + +func TestNodeUnhealthyKickIsNotAnAgentFailure(t *testing.T) { + failure := &sandbox.FailError{Reason: sandbox.FailReasonNodeUnhealthy, Node: "p1-worker-93", Message: "hive-node-preflight: FAILED command 1 of 1 exited 1: python smoke"} + m, audits := infraManager(t, infraLauncher{ + res: sandbox.Result{Stdout: "torch: no spyre device\n", ExitCode: kubejob.ExitCodeNodeUnhealthy, Node: "p1-worker-93", FailReason: sandbox.FailReasonNodeUnhealthy}, + err: failure, + }) + if err := m.SendKick("deps", "bump the dependency"); err != nil { + t.Fatalf("SendKick: %v", err) + } + p := waitLastRun(t, m) + if p.State != StateIdle { + t.Errorf("state = %s, want idle: the agent never ran, so it is not failed", p.State) + } + run := p.SandboxLastRun + if run.Succeeded || run.FailReason != sandbox.FailReasonNodeUnhealthy || run.Node != "p1-worker-93" { + t.Errorf("run record = %+v, want node_unhealthy on p1-worker-93", run) + } + if !strings.Contains(p.LastError, "node_unhealthy") || !strings.Contains(p.LastError, "p1-worker-93") { + t.Errorf("LastError = %q, want the reason and the node", p.LastError) + } + if got := audits.find("sandbox_node_unhealthy"); !strings.Contains(got, "node=p1-worker-93") { + t.Errorf("audit = %q, want an entry naming the node", got) + } + if got := audits.find("sandbox_failed"); got != "" { + t.Errorf("a node failure was audited as an agent failure: %q", got) + } +} + +func TestWaitingForNodeKickIsNotAnAgentFailure(t *testing.T) { + m, audits := infraManager(t, infraLauncher{ + res: sandbox.Result{ExitCode: -1, FailReason: sandbox.FailReasonWaitingForNode}, + err: &sandbox.FailError{Reason: sandbox.FailReasonWaitingForNode, Message: "no node took the pod within 10m0s: 0/38 nodes are available"}, + }) + if err := m.SendKick("deps", "bump"); err != nil { + t.Fatalf("SendKick: %v", err) + } + p := waitLastRun(t, m) + if p.State != StateIdle || p.SandboxLastRun.FailReason != sandbox.FailReasonWaitingForNode { + t.Errorf("state=%s run=%+v, want idle with waiting_for_node", p.State, p.SandboxLastRun) + } + if got := audits.find("sandbox_waiting_for_node_failed"); !strings.Contains(got, "0/38 nodes are available") { + t.Errorf("audit = %q, want the scheduler's reason", got) + } +} + +// POSITIVE CONTROL: an ordinary launcher failure is still a failed agent, so +// the two tests above are a real branch and not an always-idle path. +func TestOrdinaryJobFailureStillFailsTheAgent(t *testing.T) { + m, audits := infraManager(t, infraLauncher{res: sandbox.Result{ExitCode: 3}, err: errors.New("kubejob: job x failed: BackoffLimitExceeded")}) + if err := m.SendKick("deps", "bump"); err != nil { + t.Fatalf("SendKick: %v", err) + } + deadline := time.Now().Add(3 * time.Second) + for time.Now().Before(deadline) && m.AllStatuses()["deps"].State != StateFailed { + time.Sleep(10 * time.Millisecond) + } + p := m.AllStatuses()["deps"] + if p.State != StateFailed || p.SandboxLastRun == nil || p.SandboxLastRun.FailReason != "" { + t.Fatalf("state=%s run=%+v, want failed with no infrastructure reason", p.State, p.SandboxLastRun) + } + if audits.find("sandbox_failed") == "" { + t.Error("an ordinary failure must still be audited as sandbox_failed") + } +} + +func TestBuildSandboxRunCarriesTheInfrastructureReason(t *testing.T) { + now := time.Now() + run := buildSandboxRun("job", now, now, SandboxKickResult{Sandbox: sandbox.Result{Node: "n1", FailReason: sandbox.FailReasonNodeUnhealthy}}, errors.New("x")) + if run.FailReason != sandbox.FailReasonNodeUnhealthy || run.Node != "n1" || run.Succeeded { + t.Errorf("run = %+v", run) + } + if ok := buildSandboxRun("job", now, now, SandboxKickResult{Sandbox: sandbox.Result{Node: "n2"}}, nil); ok.FailReason != "" || ok.Node != "n2" || !ok.Succeeded { + t.Errorf("a successful run still names its node: %+v", ok) + } +} + +func TestPrepareJobKickWiresPreflightAndEvents(t *testing.T) { + yes := true + cfg := map[string]config.AgentConfig{"deps": {Backend: "claude", Sandbox: &config.AgentSandboxOverride{Enabled: &yes, Runtime: config.SandboxRuntimeJob}}} + m := NewManager(cfg, quietTestLogger(), ProjectContext{Org: "o", Repos: []string{"r"}}) + m.SetSandboxConfig(config.AgentSandboxConfig{Enabled: true, Image: "img", WorkspaceDir: t.TempDir(), + Job: &config.SandboxJobConfig{WorkspaceClaim: "hive-data", ScheduleTimeoutS: 120}, + Preflight: &config.SandboxPreflightConfig{Commands: []string{"python smoke"}, TimeoutS: 30, TotalTimeoutS: 60, RetryDelayS: 2}}) + audits := &auditLog{} + m.SetSandboxAuditCallback(audits.add) + if err := m.Start(context.Background(), "deps"); err != nil { + t.Fatal(err) + } + + launcher := &kubejob.Launcher{} + m.mu.Lock() + agent := m.agents["deps"] + m.prepareJobKickLocked(launcher, agent, config.SandboxRuntimeJob) + m.mu.Unlock() + o := launcher.Options + if len(o.Preflight) != 1 || o.Preflight[0] != "python smoke" || o.PreflightTimeout != 30*time.Second || o.PreflightTotalTimeout != time.Minute || o.PreflightRetryDelay != 2*time.Second { + t.Fatalf("preflight options = %+v", o) + } + if o.OnEvent == nil { + t.Fatal("the launcher's events are not wired") + } + if got := JobOptionsFromConfig(*m.sandboxConfig.Job).ScheduleTimeout; got != 2*time.Minute { + t.Errorf("schedule timeout = %s, want 2m", got) + } + + o.OnEvent(kubejob.Event{Kind: kubejob.EventWaitingForNode, Detail: "0/3 nodes are available"}) + ph := m.AllStatuses()["deps"].SandboxPhase + if ph == nil || ph.Phase != sandbox.FailReasonWaitingForNode || ph.Detail != "0/3 nodes are available" { + t.Fatalf("phase = %+v, want waiting_for_node", ph) + } + if audits.find("sandbox_waiting_for_node") == "" { + t.Error("waiting for a node is not audited") + } + o.OnEvent(kubejob.Event{Kind: kubejob.EventScheduled, Node: "n1"}) + if ph := m.AllStatuses()["deps"].SandboxPhase; ph != nil { + t.Errorf("phase = %+v, want it cleared once the pod has a node", ph) + } + o.OnEvent(kubejob.Event{Kind: kubejob.EventNodeRetry, Node: "n1", Detail: "smoke failed"}) + if got := audits.find("sandbox_node_unhealthy_retry"); !strings.Contains(got, "node=n1") { + t.Errorf("retry audit = %q, want the node", got) + } +} + +func TestPrepareJobKickLeavesOtherLaunchersAndRuntimesAlone(t *testing.T) { + cfg := map[string]config.AgentConfig{"deps": {Backend: "claude"}} + m := NewManager(cfg, quietTestLogger(), ProjectContext{}) + m.SetSandboxConfig(config.AgentSandboxConfig{Preflight: &config.SandboxPreflightConfig{Commands: []string{"smoke"}}}) + m.mu.Lock() + defer m.mu.Unlock() + agent := m.agents["deps"] + podman := &kubejob.Launcher{} + m.prepareJobKickLocked(podman, agent, config.SandboxRuntimePodman) + if len(podman.Options.Preflight) != 0 || podman.Options.OnEvent != nil { + t.Errorf("a preflight leaked into a non-job runtime: %+v", podman.Options) + } + m.prepareJobKickLocked(infraLauncher{}, agent, config.SandboxRuntimeJob) // a factory-supplied launcher: no panic, no change + unconfigured := NewManager(cfg, quietTestLogger(), ProjectContext{}) + unconfigured.mu.Lock() + defer unconfigured.mu.Unlock() + plain := &kubejob.Launcher{} + unconfigured.prepareJobKickLocked(plain, unconfigured.agents["deps"], config.SandboxRuntimeJob) + if len(plain.Options.Preflight) != 0 { + t.Errorf("preflight must be off unless configured, got %v", plain.Options.Preflight) + } +} diff --git a/src/crux/agent/sandbox_run.go b/src/crux/agent/sandbox_run.go index a3d4db9892..b50617a9ed 100644 --- a/src/crux/agent/sandbox_run.go +++ b/src/crux/agent/sandbox_run.go @@ -57,6 +57,8 @@ func buildSandboxRun(runtime string, started, ended time.Time, res SandboxKickRe CommitCount: res.CommitCount, Branch: res.Branch, Usage: parseSandboxUsage(res.Sandbox.Stdout), + FailReason: res.Sandbox.FailReason, + Node: res.Sandbox.Node, } if runErr != nil { run.Error = truncateRunes(runErr.Error(), sandboxRunSummaryMax) diff --git a/src/crux/config/agent_config.go b/src/crux/config/agent_config.go index a9d25168ba..9524f8db94 100644 --- a/src/crux/config/agent_config.go +++ b/src/crux/config/agent_config.go @@ -45,6 +45,9 @@ type SandboxJobConfig struct { EnvFromSecrets []string `yaml:"env_from_secrets,omitempty" json:"env_from_secrets,omitempty"` Volumes []SandboxJobVolume `yaml:"volumes,omitempty" json:"volumes,omitempty"` TTLSeconds int `yaml:"ttl_seconds,omitempty" json:"ttl_seconds,omitempty"` + // ScheduleTimeoutS fails a kick whose pod stays Unschedulable this long + // (reason waiting_for_node, #18811). Zero uses DefaultSandboxScheduleTimeoutS. + ScheduleTimeoutS int `yaml:"schedule_timeout_s,omitempty" json:"schedule_timeout_s,omitempty"` // RelayImage is the image of the init container that copies hive's // sandbox relays (hive-review) into the Job (#18512). Empty resolves the // running hive image by digest from hive's own pod; "none" skips the @@ -57,31 +60,6 @@ type SandboxJobConfig struct { ImagePullSecrets []string `yaml:"image_pull_secrets,omitempty" json:"image_pull_secrets,omitempty"` } -// SandboxJobToleration mirrors the subset of a Kubernetes toleration an -// operator writes by hand. -type SandboxJobToleration struct { - Key string `yaml:"key,omitempty" json:"key,omitempty"` - Operator string `yaml:"operator,omitempty" json:"operator,omitempty"` - Value string `yaml:"value,omitempty" json:"value,omitempty"` - Effect string `yaml:"effect,omitempty" json:"effect,omitempty"` -} - -// SandboxJobResources mirrors Kubernetes resource requirements with string -// quantities so a device limit (`.com/: "2"`) passes through. -type SandboxJobResources struct { - Limits map[string]string `yaml:"limits,omitempty" json:"limits,omitempty"` - Requests map[string]string `yaml:"requests,omitempty" json:"requests,omitempty"` -} - -// SandboxJobVolume is an additional PVC mounted into the Job (a compile -// cache, a model store). -type SandboxJobVolume struct { - Name string `yaml:"name,omitempty" json:"name,omitempty"` - Claim string `yaml:"claim" json:"claim"` - MountPath string `yaml:"mount_path" json:"mount_path"` - ReadOnly bool `yaml:"read_only,omitempty" json:"read_only,omitempty"` -} - // SandboxRuntime returns the per-agent runtime override, then the global // runtime, then SandboxRuntimePodman. The value is returned as written; the // manager rejects one outside ValidSandboxRuntimes at kick time. @@ -119,6 +97,9 @@ func (a *AgentConfig) SandboxJob(global AgentSandboxConfig) SandboxJobConfig { if out.TTLSeconds == 0 { out.TTLSeconds = g.TTLSeconds } + if out.ScheduleTimeoutS == 0 { + out.ScheduleTimeoutS = g.ScheduleTimeoutS + } return inheritPullSecrets(out, g) } @@ -514,6 +495,9 @@ type AgentSandboxConfig struct { Substrate *SandboxSubstrateConfig `yaml:"substrate,omitempty" json:"substrate,omitempty"` // Publish gates what a sandboxed run may publish; see SandboxPublishConfig. Publish *SandboxPublishConfig `yaml:"publish,omitempty" json:"publish,omitempty"` + // Preflight is the node health check run before the agent starts on the + // job runtime; see SandboxPreflightConfig. + Preflight *SandboxPreflightConfig `yaml:"preflight,omitempty" json:"preflight,omitempty"` } // SandboxPublishConfig is the deterministic preflight hive runs before it @@ -559,6 +543,9 @@ type AgentSandboxOverride struct { // Publish, when set, replaces the global publish preflight for this // agent; an empty verify list here turns the preflight off for it. Publish *SandboxPublishConfig `yaml:"publish,omitempty" json:"publish,omitempty"` + // Preflight, when set, replaces the global preflight for this agent; an + // empty commands list here turns it off for the agent. + Preflight *SandboxPreflightConfig `yaml:"preflight,omitempty" json:"preflight,omitempty"` } // BobConfig configures the IBM bobshell ("bob") CLI backend. Only the diff --git a/src/crux/config/sandbox_job_types.go b/src/crux/config/sandbox_job_types.go new file mode 100644 index 0000000000..d86ef4aeb5 --- /dev/null +++ b/src/crux/config/sandbox_job_types.go @@ -0,0 +1,26 @@ +package config + +// SandboxJobToleration mirrors the subset of a Kubernetes toleration an +// operator writes by hand. +type SandboxJobToleration struct { + Key string `yaml:"key,omitempty" json:"key,omitempty"` + Operator string `yaml:"operator,omitempty" json:"operator,omitempty"` + Value string `yaml:"value,omitempty" json:"value,omitempty"` + Effect string `yaml:"effect,omitempty" json:"effect,omitempty"` +} + +// SandboxJobResources mirrors Kubernetes resource requirements with string +// quantities so a device limit (`.com/: "2"`) passes through. +type SandboxJobResources struct { + Limits map[string]string `yaml:"limits,omitempty" json:"limits,omitempty"` + Requests map[string]string `yaml:"requests,omitempty" json:"requests,omitempty"` +} + +// SandboxJobVolume is an additional PVC mounted into the Job (a compile +// cache, a model store). +type SandboxJobVolume struct { + Name string `yaml:"name,omitempty" json:"name,omitempty"` + Claim string `yaml:"claim" json:"claim"` + MountPath string `yaml:"mount_path" json:"mount_path"` + ReadOnly bool `yaml:"read_only,omitempty" json:"read_only,omitempty"` +} diff --git a/src/crux/config/sandbox_preflight.go b/src/crux/config/sandbox_preflight.go new file mode 100644 index 0000000000..d45eb31625 --- /dev/null +++ b/src/crux/config/sandbox_preflight.go @@ -0,0 +1,116 @@ +package config + +import ( + "fmt" + "strings" +) + +// SandboxPreflightConfig is the node health check a Job-runtime sandbox kick +// runs in its main container before the agent starts (hivecommons/hive#18811). +// A node whose accelerator card is unusable fails the kick in seconds with +// the distinct reason node_unhealthy instead of minutes into the agent's run, +// and hive retries once on a different node. +// +// Each command is run with `sh -c` in the Job's own image, workspace and +// environment, after hive's relay check. The commands come from this config +// only, never from a kick message, exactly like sandbox.publish.verify. They +// are part of the Job manifest and their output lands in the kick transcript, +// so a command must not carry a credential on its command line. +// +// Preflight is off unless Commands is non-empty. +type SandboxPreflightConfig struct { + Commands []string `yaml:"commands,omitempty" json:"commands,omitempty"` + // TimeoutS bounds each command. Zero uses DefaultSandboxPreflightTimeoutS. + TimeoutS int `yaml:"timeout_s,omitempty" json:"timeout_s,omitempty"` + // TotalTimeoutS bounds all commands together. Zero uses + // DefaultSandboxPreflightTotalTimeoutS. + TotalTimeoutS int `yaml:"total_timeout_s,omitempty" json:"total_timeout_s,omitempty"` + // RetryDelayS is how long hive waits before re-creating the Job on + // another node after node_unhealthy. Zero retries immediately. + RetryDelayS int `yaml:"retry_delay_s,omitempty" json:"retry_delay_s,omitempty"` +} + +// Defaults and bounds for sandbox.preflight and job.schedule_timeout_s. +const ( + DefaultSandboxPreflightTimeoutS = 120 + DefaultSandboxPreflightTotalTimeoutS = 300 + // DefaultSandboxScheduleTimeoutS is how long a kick's pod may stay + // Unschedulable before the kick fails with waiting_for_node. + DefaultSandboxScheduleTimeoutS = 600 + + // MaxSandboxPreflightCommands and MaxSandboxPreflightCommandBytes bound + // operator-supplied preflight lists. + MaxSandboxPreflightCommands = 32 + MaxSandboxPreflightCommandBytes = 4096 +) + +// SandboxPreflight returns the effective preflight for the agent: the +// per-agent block when present (an empty commands list disables it for that +// agent), else the global one. Timeouts left at zero fall back to the global +// block, then to the defaults. +func (a *AgentConfig) SandboxPreflight(global AgentSandboxConfig) SandboxPreflightConfig { + var g SandboxPreflightConfig + if global.Preflight != nil { + g = *global.Preflight + } + out := g + if a != nil && a.Sandbox != nil && a.Sandbox.Preflight != nil { + out = *a.Sandbox.Preflight + if out.TimeoutS == 0 { + out.TimeoutS = g.TimeoutS + } + if out.TotalTimeoutS == 0 { + out.TotalTimeoutS = g.TotalTimeoutS + } + if out.RetryDelayS == 0 { + out.RetryDelayS = g.RetryDelayS + } + } + out.Commands = append([]string(nil), out.Commands...) + if out.TimeoutS == 0 { + out.TimeoutS = DefaultSandboxPreflightTimeoutS + } + if out.TotalTimeoutS == 0 { + out.TotalTimeoutS = DefaultSandboxPreflightTotalTimeoutS + } + return out +} + +// ValidateSandboxPreflight rejects a preflight that could not run: an empty +// or oversized command, negative timeouts, an unbounded list. A nil block is +// valid (preflight off). +func ValidateSandboxPreflight(p *SandboxPreflightConfig) error { + if p == nil { + return nil + } + if len(p.Commands) > MaxSandboxPreflightCommands { + return fmt.Errorf("preflight.commands takes at most %d commands, got %d", MaxSandboxPreflightCommands, len(p.Commands)) + } + for i, c := range p.Commands { + switch { + case strings.TrimSpace(c) == "": + return fmt.Errorf("preflight.commands[%d] must not be empty", i) + case len(c) > MaxSandboxPreflightCommandBytes: + return fmt.Errorf("preflight.commands[%d] is longer than %d bytes", i, MaxSandboxPreflightCommandBytes) + case strings.ContainsRune(c, 0): + return fmt.Errorf("preflight.commands[%d] contains a NUL byte", i) + } + } + for _, f := range []struct { + name string + v int + }{{"timeout_s", p.TimeoutS}, {"total_timeout_s", p.TotalTimeoutS}, {"retry_delay_s", p.RetryDelayS}} { + if f.v < 0 { + return fmt.Errorf("preflight.%s must not be negative, got %d", f.name, f.v) + } + } + return nil +} + +// ValidateSandboxJobScheduleTimeout rejects a negative job.schedule_timeout_s. +func ValidateSandboxJobScheduleTimeout(job *SandboxJobConfig) error { + if job != nil && job.ScheduleTimeoutS < 0 { + return fmt.Errorf("job.schedule_timeout_s must not be negative, got %d", job.ScheduleTimeoutS) + } + return nil +} diff --git a/src/crux/config/sandbox_preflight_test.go b/src/crux/config/sandbox_preflight_test.go new file mode 100644 index 0000000000..524d1b148d --- /dev/null +++ b/src/crux/config/sandbox_preflight_test.go @@ -0,0 +1,94 @@ +package config + +import ( + "reflect" + "strings" + "testing" +) + +func TestSandboxPreflightOffByDefault(t *testing.T) { + var agent *AgentConfig + got := agent.SandboxPreflight(AgentSandboxConfig{}) + if len(got.Commands) != 0 { + t.Fatalf("preflight must be off unless configured, got %v", got.Commands) + } + if got.TimeoutS != DefaultSandboxPreflightTimeoutS || got.TotalTimeoutS != DefaultSandboxPreflightTotalTimeoutS || got.RetryDelayS != 0 { + t.Fatalf("defaults = %+v", got) + } + if DefaultSandboxScheduleTimeoutS != 600 { + t.Fatalf("default schedule timeout = %d, want 600", DefaultSandboxScheduleTimeoutS) + } +} + +func TestSandboxPreflightMergePrecedence(t *testing.T) { + global := AgentSandboxConfig{Preflight: &SandboxPreflightConfig{Commands: []string{"global-check"}, TimeoutS: 30, TotalTimeoutS: 90, RetryDelayS: 4}} + + var none *AgentConfig + if got := none.SandboxPreflight(global); !reflect.DeepEqual(got.Commands, []string{"global-check"}) || got.TimeoutS != 30 || got.TotalTimeoutS != 90 || got.RetryDelayS != 4 { + t.Errorf("global not inherited: %+v", got) + } + inherit := &AgentConfig{Sandbox: &AgentSandboxOverride{}} + if got := inherit.SandboxPreflight(global); !reflect.DeepEqual(got.Commands, []string{"global-check"}) { + t.Errorf("an agent without a preflight block must inherit: %+v", got) + } + own := &AgentConfig{Sandbox: &AgentSandboxOverride{Preflight: &SandboxPreflightConfig{Commands: []string{"own-check"}, TimeoutS: 5}}} + got := own.SandboxPreflight(global) + if !reflect.DeepEqual(got.Commands, []string{"own-check"}) || got.TimeoutS != 5 { + t.Errorf("the agent's block must win: %+v", got) + } + if got.TotalTimeoutS != 90 || got.RetryDelayS != 4 { + t.Errorf("timeouts left at zero must fall back to the global block: %+v", got) + } + off := &AgentConfig{Sandbox: &AgentSandboxOverride{Preflight: &SandboxPreflightConfig{}}} + if got := off.SandboxPreflight(global); len(got.Commands) != 0 { + t.Errorf("an empty commands list must turn the preflight off for the agent: %+v", got) + } + got.Commands[0] = "mutated" + if global.Preflight.Commands[0] != "global-check" || own.Sandbox.Preflight.Commands[0] != "own-check" { + t.Error("the returned commands alias the config") + } +} + +func TestSandboxJobScheduleTimeoutFallsBackToGlobal(t *testing.T) { + global := AgentSandboxConfig{Job: &SandboxJobConfig{WorkspaceClaim: "c", ScheduleTimeoutS: 900}} + agent := &AgentConfig{Sandbox: &AgentSandboxOverride{Job: &SandboxJobConfig{}}} + if got := agent.SandboxJob(global).ScheduleTimeoutS; got != 900 { + t.Errorf("schedule timeout = %d, want the global 900", got) + } + agent.Sandbox.Job.ScheduleTimeoutS = 60 + if got := agent.SandboxJob(global).ScheduleTimeoutS; got != 60 { + t.Errorf("schedule timeout = %d, want the agent's 60", got) + } +} + +func TestValidateSandboxPreflight(t *testing.T) { + if err := ValidateSandboxPreflight(nil); err != nil { + t.Fatalf("nil must be valid (off): %v", err) + } + if err := ValidateSandboxPreflight(&SandboxPreflightConfig{Commands: []string{`python -c 'import torch'`}, TimeoutS: 1, TotalTimeoutS: 2, RetryDelayS: 3}); err != nil { + t.Fatalf("valid preflight refused: %v", err) + } + for _, tc := range []struct { + name string + p SandboxPreflightConfig + want string + }{ + {"empty command", SandboxPreflightConfig{Commands: []string{"true", " "}}, "commands[1] must not be empty"}, + {"nul byte", SandboxPreflightConfig{Commands: []string{"a\x00b"}}, "NUL"}, + {"too long", SandboxPreflightConfig{Commands: []string{strings.Repeat("x", MaxSandboxPreflightCommandBytes+1)}}, "longer than"}, + {"too many", SandboxPreflightConfig{Commands: make([]string, MaxSandboxPreflightCommands+1)}, "at most"}, + {"negative timeout", SandboxPreflightConfig{TimeoutS: -1}, "timeout_s must not be negative"}, + {"negative total", SandboxPreflightConfig{TotalTimeoutS: -1}, "total_timeout_s must not be negative"}, + {"negative delay", SandboxPreflightConfig{RetryDelayS: -1}, "retry_delay_s must not be negative"}, + } { + if err := ValidateSandboxPreflight(&tc.p); err == nil || !strings.Contains(err.Error(), tc.want) { + t.Errorf("%s: error = %v, want it to contain %q", tc.name, err, tc.want) + } + } + if err := ValidateSandboxJobScheduleTimeout(&SandboxJobConfig{ScheduleTimeoutS: -5}); err == nil { + t.Error("a negative schedule_timeout_s must be refused") + } + if err := ValidateSandboxJobScheduleTimeout(nil); err != nil { + t.Errorf("nil job: %v", err) + } +} diff --git a/src/crux/kubejob/attempt.go b/src/crux/kubejob/attempt.go new file mode 100644 index 0000000000..a569878905 --- /dev/null +++ b/src/crux/kubejob/attempt.go @@ -0,0 +1,182 @@ +package kubejob + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "strings" + "time" + + "github.com/hivecommons/hive/crux/sandbox" +) + +// retryJobSuffix names the re-created Job after a node_unhealthy kick. +const retryJobSuffix = "-r" + +// runWithRetry runs the kick as a Job and, when its node preflight fails, runs +// it once more on a different node (#18811). Only an agent kick (Relays set) +// gets a preflight: the publish.verify launches share this launcher and must +// not be gated on the agent's node health. +func (l *Launcher) runWithRetry(ctx context.Context, spec sandbox.LaunchSpec, ns, subPath string, relays *relayPlan) (sandbox.Result, error) { + if !spec.Relays && len(l.Options.Preflight) > 0 { + cp := *l + cp.Options.Preflight = nil + l = &cp + } + first := jobName(spec.Name, l.now()) + var avoid []string + var earlier strings.Builder + for attempt := 0; ; attempt++ { + name := first + if attempt > 0 { + name = retryJobName(first, attempt) + } + res, err := l.runAttempt(ctx, spec, ns, subPath, name, relays, avoid) + if res.FailReason != sandbox.FailReasonNodeUnhealthy || attempt >= maxPreflightRetries || res.Node == "" || ctx.Err() != nil { + if earlier.Len() > 0 { + res.Stdout = earlier.String() + res.Stdout + } + return res, err + } + detail := "" + var fe *sandbox.FailError + if errors.As(err, &fe) { + detail = fe.Message + } + fmt.Fprintf(&earlier, "hive: attempt %d on node %s failed the node preflight; retrying on another node\n%s\n\n", attempt+1, res.Node, res.Stdout) + l.emit(Event{Kind: EventNodeRetry, Node: res.Node, Detail: detail}) + avoid = append(avoid, res.Node) + if d := l.Options.PreflightRetryDelay; d > 0 { + select { + case <-ctx.Done(): + res.Stdout = earlier.String() + return res, fmt.Errorf("kubejob: %w", ctx.Err()) + case <-time.After(d): + } + } + } +} + +// retryJobName is the Job name for the n-th re-creation, kept within jobNameMax. +func retryJobName(first string, n int) string { + suffix := fmt.Sprintf("%s%d", retryJobSuffix, n) + if room := jobNameMax - len(suffix); len(first) > room { + first = strings.TrimRight(first[:room], "-") + } + return first + suffix +} + +// runAttempt creates one Job, waits for it, collects the pod's log and exit +// code, and deletes the Job on success. A Job that failed is left in place +// until its TTL so the operator can inspect it; one that never got a node is +// deleted. avoid lists nodes the pod must not run on (a failed preflight's). +func (l *Launcher) runAttempt(ctx context.Context, spec sandbox.LaunchSpec, ns, subPath, name string, relays *relayPlan, avoid []string) (sandbox.Result, error) { + var res sandbox.Result + manifest := l.jobManifest(ns, name, subPath, spec, relays) + setPullSecrets(manifest, l.resolvePullSecrets(ctx, ns, relays)) + if jobSpec, ok := manifest["spec"].(map[string]any); ok { + jobSpec["activeDeadlineSeconds"] = l.activeDeadline(ctx) + } + avoidNodes(manifest, l.Options.Affinity, avoid) + body, err := json.Marshal(manifest) + if err != nil { + return res, err + } + jobsPath := fmt.Sprintf("/apis/batch/v1/namespaces/%s/jobs", ns) + policyPath := "" + if l.Options.network() == NetworkNone { + // The policy exists before any pod can: a Job created first could + // start its pod in the window before the policy lands. + policiesPath := fmt.Sprintf("/apis/networking.k8s.io/v1/namespaces/%s/networkpolicies", ns) + policy, err := json.Marshal(denyAllPolicy(ns, name)) + if err != nil { + return res, err + } + if _, err := l.do(ctx, http.MethodPost, policiesPath, policy); err != nil { + return res, fmt.Errorf("kubejob: network %q: creating the deny-all NetworkPolicy for %s: %w", NetworkNone, name, err) + } + policyPath = policiesPath + "/" + name + } + if _, err := l.do(ctx, http.MethodPost, jobsPath, body); err != nil { + if policyPath != "" { + _ = l.delete(ctx, policyPath) + } + return res, fmt.Errorf("kubejob: creating job %s: %w", name, err) + } + jobPath := jobsPath + "/" + name + + done, waitErr := l.waitJob(ctx, ns, name, jobPath) + // Collect whatever the pod produced BEFORE deciding anything: a timed-out + // or failed kick with no transcript is the worst outcome for an operator. + // The caller's ctx may be dead already (kick timeout) or die between any + // two of these requests (cancel during log collection), so every step + // re-checks it and falls back to a fresh, request-bounded context. + // Otherwise a readable, Complete pod would report exitCodeUnknown merely + // because ctx was cancelled after the log was read. + var cancels []context.CancelFunc + defer func() { + for _, c := range cancels { + c() + } + }() + live := func() context.Context { + if ctx.Err() == nil { + return ctx + } + fresh, cancel := context.WithTimeout(context.Background(), requestTimeout) + cancels = append(cancels, cancel) + return fresh + } + if waitErr == nil && done.failReason == sandbox.FailReasonWaitingForNode { + // No node took the pod in time: there is no log to read and nothing + // to inspect, so the Job goes and the agent's slot is released. + _ = l.delete(live(), jobPath) + l.deletePolicy(live(), policyPath) + res.ExitCode = exitCodeUnknown + res.FailReason = done.failReason + res.Stderr = done.reason + msg := fmt.Sprintf("no node took the pod within %s: %s", l.Options.scheduleTimeout(), done.reason) + return res, &sandbox.FailError{Reason: done.failReason, Message: msg} + } + pod, podErr := l.findPod(live(), ns, name) + if podErr == nil && pod != "" { + res.Stdout = l.podLog(live(), ns, pod) + info := l.podDetails(live(), ns, pod) + res.ExitCode, res.Node = info.exitCode, info.node + } else { + res.ExitCode = exitCodeUnknown + } + if res.Node == "" { + res.Node = done.node + } + if waitErr != nil { + // Context ended (kick timeout or cancel). The Job would otherwise + // keep running the workload on the accelerator node; stop it. + _ = l.delete(live(), jobPath) + l.deletePolicy(live(), policyPath) + return res, waitErr + } + if !done.succeeded { + // The Job stays for inspection; its pods are finished, so the + // policy has nothing left to isolate and goes. + l.deletePolicy(live(), policyPath) + res.Stderr = done.reason + if msg, bad := l.Options.preflightFailure(res); bad { + res.FailReason = sandbox.FailReasonNodeUnhealthy + return res, &sandbox.FailError{Reason: res.FailReason, Node: res.Node, Message: msg} + } + return res, fmt.Errorf("kubejob: job %s failed: %s", name, done.reason) + } + // When the Job reports Complete but the pod could not be read back + // (already GC'd, or RBAC on pods missing), res.ExitCode stays + // exitCodeUnknown rather than a guessed 0; callers treat a nil error with + // an unknown code as completed-without-code. + // + // Collection may have expired the context even on success; the fresh + // context ensures the finished Job is still removed immediately. + _ = l.delete(live(), jobPath) + l.deletePolicy(live(), policyPath) + return res, nil +} diff --git a/src/crux/kubejob/kubejob.go b/src/crux/kubejob/kubejob.go index d0b241b17b..42136d0195 100644 --- a/src/crux/kubejob/kubejob.go +++ b/src/crux/kubejob/kubejob.go @@ -32,8 +32,6 @@ import ( "fmt" "net/http" "os" - "path/filepath" - "regexp" "strings" "time" @@ -161,6 +159,19 @@ type Options struct { // whichever of this and the context-derived value is later, so the // deadline never cuts a run short of the timeout hive gave it. ActiveDeadlineSeconds int64 + // Preflight, PreflightTimeout, PreflightTotalTimeout and + // PreflightRetryDelay configure the node preflight (#18811); see + // preflight.go. Preflight empty leaves the Job exactly as before. + Preflight []string + PreflightTimeout time.Duration + PreflightTotalTimeout time.Duration + PreflightRetryDelay time.Duration + // ScheduleTimeout is how long the pod may stay Unschedulable before the + // kick fails with waiting_for_node; zero uses DefaultScheduleTimeout. + ScheduleTimeout time.Duration + // OnEvent, when set, is told about scheduling and preflight-retry events + // so the manager can surface them (see Event). It must not block. + OnEvent func(Event) } // ActiveDeadlineSeconds returns the activeDeadlineSeconds for a Job running @@ -196,44 +207,6 @@ func (o Options) network() string { return o.Network } -// Toleration mirrors the subset of corev1.Toleration an operator writes in YAML. -type Toleration struct { - Key string `json:"key,omitempty"` - Operator string `json:"operator,omitempty"` - Value string `json:"value,omitempty"` - Effect string `json:"effect,omitempty"` -} - -// Resources mirrors corev1.ResourceRequirements with string quantities, so a -// device limit like `.com/: "2"` passes through untouched. -type Resources struct { - Limits map[string]string `json:"limits,omitempty"` - Requests map[string]string `json:"requests,omitempty"` -} - -// VolumeMount is an additional PVC to mount into the Job. -type VolumeMount struct { - Name string `json:"name"` - Claim string `json:"claim"` - MountPath string `json:"mount_path"` - ReadOnly bool `json:"read_only,omitempty"` -} - -// Network modes, Options.Network. -const ( - // NetworkDefault leaves the pod on the cluster network with whatever - // policies the namespace already has. The empty string means this. - NetworkDefault = "default" - // NetworkNone creates a deny-all NetworkPolicy selecting the Job's pods - // before the Job and removes it with the Job; the run is refused when - // the policy cannot be created. Enforcement is the CNI's: a cluster - // whose network plugin ignores NetworkPolicy isolates nothing, and the - // launcher cannot tell. - NetworkNone = "none" - // NetworkHost runs the pod with hostNetwork. - NetworkHost = "host" -) - // Launcher implements sandbox.Launcher by running the kick as a Job. type Launcher struct { Options Options @@ -274,6 +247,9 @@ func (o Options) Validate() error { if err := validatePullSecretNames(o.ImagePullSecrets); err != nil { return err } + if err := o.validatePreflight(); err != nil { + return err + } switch o.network() { case NetworkDefault, NetworkNone, NetworkHost: default: @@ -314,148 +290,19 @@ func (l *Launcher) Run(ctx context.Context, spec sandbox.LaunchSpec) (sandbox.Re return res, err } } - name := jobName(spec.Name, l.now()) - manifest := l.jobManifest(ns, name, subPath, spec, relays) - setPullSecrets(manifest, l.resolvePullSecrets(ctx, ns, relays)) - if jobSpec, ok := manifest["spec"].(map[string]any); ok { - jobSpec["activeDeadlineSeconds"] = l.activeDeadline(ctx) - } - body, err := json.Marshal(manifest) - if err != nil { - return res, err - } - jobsPath := fmt.Sprintf("/apis/batch/v1/namespaces/%s/jobs", ns) - policyPath := "" - if l.Options.network() == NetworkNone { - // The policy exists before any pod can: a Job created first could - // start its pod in the window before the policy lands. - policiesPath := fmt.Sprintf("/apis/networking.k8s.io/v1/namespaces/%s/networkpolicies", ns) - policy, err := json.Marshal(denyAllPolicy(ns, name)) - if err != nil { - return res, err - } - if _, err := l.do(ctx, http.MethodPost, policiesPath, policy); err != nil { - return res, fmt.Errorf("kubejob: network %q: creating the deny-all NetworkPolicy for %s: %w", NetworkNone, name, err) - } - policyPath = policiesPath + "/" + name - } - if _, err := l.do(ctx, http.MethodPost, jobsPath, body); err != nil { - if policyPath != "" { - _ = l.delete(ctx, policyPath) - } - return res, fmt.Errorf("kubejob: creating job %s: %w", name, err) - } - jobPath := jobsPath + "/" + name - - done, waitErr := l.wait(ctx, jobPath) - // Collect whatever the pod produced BEFORE deciding anything: a timed-out - // or failed kick with no transcript is the worst outcome for an operator. - // The caller's ctx may be dead already (kick timeout) or die between any - // two of these requests (cancel during log collection), so every step - // re-checks it and falls back to a fresh, request-bounded context. - // Otherwise a readable, Complete pod would report exitCodeUnknown merely - // because ctx was cancelled after the log was read. - var cancels []context.CancelFunc - defer func() { - for _, c := range cancels { - c() - } - }() - live := func() context.Context { - if ctx.Err() == nil { - return ctx - } - fresh, cancel := context.WithTimeout(context.Background(), requestTimeout) - cancels = append(cancels, cancel) - return fresh - } - pod, podErr := l.findPod(live(), ns, name) - if podErr == nil && pod != "" { - res.Stdout = l.podLog(live(), ns, pod) - res.ExitCode = l.podExitCode(live(), ns, pod) - } else { - res.ExitCode = exitCodeUnknown - } - if waitErr != nil { - // Context ended (kick timeout or cancel). The Job would otherwise - // keep running the workload on the accelerator node; stop it. - _ = l.delete(live(), jobPath) - l.deletePolicy(live(), policyPath) - return res, waitErr - } - if !done.succeeded { - // The Job stays for inspection; its pods are finished, so the - // policy has nothing left to isolate and goes. - l.deletePolicy(live(), policyPath) - res.Stderr = done.reason - return res, fmt.Errorf("kubejob: job %s failed: %s", name, done.reason) - } - // When the Job reports Complete but the pod could not be read back - // (already GC'd, or RBAC on pods missing), res.ExitCode stays - // exitCodeUnknown rather than a guessed 0; callers treat a nil error with - // an unknown code as completed-without-code. - // - // Collection may have expired the context even on success; the fresh - // context ensures the finished Job is still removed immediately. - _ = l.delete(live(), jobPath) - l.deletePolicy(live(), policyPath) - return res, nil -} - -// deletePolicy removes the per-Job NetworkPolicy when one was created. -func (l *Launcher) deletePolicy(ctx context.Context, policyPath string) { - if policyPath != "" { - _ = l.delete(ctx, policyPath) - } -} - -// denyAllPolicy is a networking.k8s.io/v1 NetworkPolicy named after the -// Job that selects its pods and allows no ingress and no egress. -func denyAllPolicy(ns, job string) map[string]any { - return map[string]any{ - "apiVersion": "networking.k8s.io/v1", - "kind": "NetworkPolicy", - "metadata": map[string]any{ - "name": job, - "namespace": ns, - "labels": map[string]string{managedByLabel: managedByValue, jobLabel: job}, - }, - "spec": map[string]any{ - "podSelector": map[string]any{"matchLabels": map[string]string{jobLabel: job}}, - "policyTypes": []string{"Ingress", "Egress"}, - }, + if spec.Relays && len(l.Options.Preflight) > 0 && len(spec.Command) == 0 { + return res, errors.New("kubejob: sandbox.preflight needs an explicit command to run the agent after it; this launch has none") } + return l.runWithRetry(ctx, spec, ns, subPath, relays) } type jobOutcome struct { succeeded bool reason string -} - -// wait polls the Job until a Complete or Failed condition, or ctx ends. -func (l *Launcher) wait(ctx context.Context, jobPath string) (jobOutcome, error) { - interval := l.Options.PollInterval - if interval <= 0 { - interval = DefaultPollInterval - } - ticker := time.NewTicker(interval) - defer ticker.Stop() - for { - raw, err := l.do(ctx, http.MethodGet, jobPath, nil) - if err != nil { - if ctx.Err() != nil { - return jobOutcome{}, fmt.Errorf("kubejob: %w", ctx.Err()) - } - // Transient API error: keep polling; the context still bounds us. - } else if out, finished := parseJobStatus(raw); finished { - return out, nil - } - select { - case <-ctx.Done(): - return jobOutcome{}, fmt.Errorf("kubejob: %w", ctx.Err()) - case <-ticker.C: - } - } + // failReason is sandbox.FailReasonWaitingForNode when the pod stayed + // Unschedulable past the schedule timeout; node is where the pod landed. + failReason string + node string } // parseJobStatus reads the Complete/Failed conditions off a Job object. @@ -526,33 +373,7 @@ func (l *Launcher) podLog(ctx context.Context, ns, pod string) string { // podExitCode reads the kick container's terminated exit code, or // exitCodeUnknown when the pod is not terminated or cannot be read. func (l *Launcher) podExitCode(ctx context.Context, ns, pod string) int { - path := fmt.Sprintf("/api/v1/namespaces/%s/pods/%s", ns, pod) - raw, err := l.do(ctx, http.MethodGet, path, nil) - if err != nil { - return exitCodeUnknown - } - var p struct { - Status struct { - ContainerStatuses []struct { - Name string `json:"name"` - State map[string]struct { - ExitCode int `json:"exitCode"` - } `json:"state"` - } `json:"containerStatuses"` - } `json:"status"` - } - if err := json.Unmarshal(raw, &p); err != nil { - return exitCodeUnknown - } - for _, cs := range p.Status.ContainerStatuses { - if cs.Name != containerName { - continue - } - if term, ok := cs.State[stateTerminated]; ok { - return term.ExitCode - } - } - return exitCodeUnknown + return l.podDetails(ctx, ns, pod).exitCode } func (l *Launcher) delete(ctx context.Context, jobPath string) error { @@ -638,7 +459,7 @@ func (l *Launcher) jobManifest(ns, name, subPath string, spec sandbox.LaunchSpec "envFrom": envFrom, "volumeMounts": mounts, } - command := spec.Command + command := preflightCommand(spec.Command, l.Options) if relays != nil { command = sandbox.RelayCommand(command) mounts = append(mounts, map[string]any{"name": relayVolumeName, "mountPath": sandbox.RelayMountPath, "readOnly": true}) @@ -719,58 +540,6 @@ func (l *Launcher) jobManifest(ns, name, subPath string, spec sandbox.LaunchSpec } } -// workspaceSubPath turns hive's absolute workspace path into the path relative -// to the claim mount, which is what the Job's volumeMount.subPath needs. -func workspaceSubPath(workspace, claimMount string) (string, error) { - if claimMount == "" { - claimMount = DefaultWorkspaceClaimMount - } - ws := filepath.Clean(workspace) - cm := filepath.Clean(claimMount) - rel, err := filepath.Rel(cm, ws) - if err != nil || rel == "." || strings.HasPrefix(rel, "..") { - return "", fmt.Errorf("kubejob: workspace %q is not under the workspace claim mount %q — set agent_sandbox.workspace_dir under it, or job.workspace_claim_mount to where the claim is mounted", workspace, claimMount) - } - return filepath.ToSlash(rel), nil -} - -var labelUnsafe = regexp.MustCompile(`[^a-z0-9-]+`) - -// jobName builds a unique, DNS-label-safe Job name from the launch name and a -// timestamp: hive-kick--. Names must survive as pod-name prefixes. -func jobName(launchName string, now time.Time) string { - base := sanitizeLabel(strings.TrimSuffix(launchName, "-sandbox")) - if base == "" { - base = "agent" - } - suffix := fmt.Sprintf("-%d", now.Unix()) - room := jobNameMax - len(namePrefix) - len(suffix) - if len(base) > room { - base = strings.TrimRight(base[:room], "-") - } - return namePrefix + base + suffix -} - -func sanitizeLabel(s string) string { - s = labelUnsafe.ReplaceAllString(strings.ToLower(s), "-") - return strings.Trim(s, "-") -} - -func sortedKeys(m map[string]string) []string { - keys := make([]string, 0, len(m)) - for k := range m { - keys = append(keys, k) - } - // insertion sort: env maps are tiny and this avoids importing sort for - // one call site. - for i := 1; i < len(keys); i++ { - for j := i; j > 0 && keys[j] < keys[j-1]; j-- { - keys[j], keys[j-1] = keys[j-1], keys[j] - } - } - return keys -} - func (l *Launcher) now() time.Time { if l.Now != nil { return l.Now() diff --git a/src/crux/kubejob/kubejob_names.go b/src/crux/kubejob/kubejob_names.go new file mode 100644 index 0000000000..446b684d71 --- /dev/null +++ b/src/crux/kubejob/kubejob_names.go @@ -0,0 +1,61 @@ +package kubejob + +import ( + "fmt" + "path/filepath" + "regexp" + "strings" + "time" +) + +// workspaceSubPath turns hive's absolute workspace path into the path relative +// to the claim mount, which is what the Job's volumeMount.subPath needs. +func workspaceSubPath(workspace, claimMount string) (string, error) { + if claimMount == "" { + claimMount = DefaultWorkspaceClaimMount + } + ws := filepath.Clean(workspace) + cm := filepath.Clean(claimMount) + rel, err := filepath.Rel(cm, ws) + if err != nil || rel == "." || strings.HasPrefix(rel, "..") { + return "", fmt.Errorf("kubejob: workspace %q is not under the workspace claim mount %q — set agent_sandbox.workspace_dir under it, or job.workspace_claim_mount to where the claim is mounted", workspace, claimMount) + } + return filepath.ToSlash(rel), nil +} + +var labelUnsafe = regexp.MustCompile(`[^a-z0-9-]+`) + +// jobName builds a unique, DNS-label-safe Job name from the launch name and a +// timestamp: hive-kick--. Names must survive as pod-name prefixes. +func jobName(launchName string, now time.Time) string { + base := sanitizeLabel(strings.TrimSuffix(launchName, "-sandbox")) + if base == "" { + base = "agent" + } + suffix := fmt.Sprintf("-%d", now.Unix()) + room := jobNameMax - len(namePrefix) - len(suffix) + if len(base) > room { + base = strings.TrimRight(base[:room], "-") + } + return namePrefix + base + suffix +} + +func sanitizeLabel(s string) string { + s = labelUnsafe.ReplaceAllString(strings.ToLower(s), "-") + return strings.Trim(s, "-") +} + +func sortedKeys(m map[string]string) []string { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + // insertion sort: env maps are tiny and this avoids importing sort for + // one call site. + for i := 1; i < len(keys); i++ { + for j := i; j > 0 && keys[j] < keys[j-1]; j-- { + keys[j], keys[j-1] = keys[j-1], keys[j] + } + } + return keys +} diff --git a/src/crux/kubejob/kubejob_test.go b/src/crux/kubejob/kubejob_test.go index d2f1343f67..b247519bc3 100644 --- a/src/crux/kubejob/kubejob_test.go +++ b/src/crux/kubejob/kubejob_test.go @@ -49,6 +49,30 @@ type fakeAPI struct { selfPodGets int serviceAccounts map[string]string // SA name -> JSON body; absent = 404 saForbidden bool // SA reads answer 403 + + // Scripted per-call behavior for the node preflight and scheduling tests + // (#18811). Each slice is indexed by how many such calls came before and + // its last entry repeats; empty keeps the single-value fields above. + createdAll []map[string]any // every Job manifest POSTed, in order + exitCodes []int // terminated exit code per pod GET + podNodes []string // spec.nodeName per pod GET ("" = none) + logs []string // pod log per log GET + podLists []string // body per pods-list GET (the schedule watch) + podGets int + logGets int + podListGets int + jobCreations int +} + +// pick returns the i-th entry of a script, repeating the last. +func pick[T any](script []T, i int, def T) T { + if len(script) == 0 { + return def + } + if i >= len(script) { + i = len(script) - 1 + } + return script[i] } func newFakeAPI() *fakeAPI { @@ -95,6 +119,8 @@ func (f *fakeAPI) handler(t *testing.T) http.HandlerFunc { t.Errorf("manifest is not JSON: %v", err) } f.created = m + f.createdAll = append(f.createdAll, m) + f.jobCreations++ f.order = append(f.order, "job") w.WriteHeader(http.StatusCreated) _, _ = w.Write(body) @@ -120,12 +146,23 @@ func (f *fakeAPI) handler(t *testing.T) http.HandlerFunc { _, _ = w.Write([]byte(`{"items":[]}`)) return } + if len(f.podLists) > 0 { + body := pick(f.podLists, f.podListGets, "") + f.podListGets++ + _, _ = w.Write([]byte(body)) + return + } _, _ = w.Write([]byte(`{"items":[{"metadata":{"name":"` + f.podName + `"}}]}`)) case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/log"): if f.jobCompleted && f.onPodLogAfterComplete != nil { f.onPodLogAfterComplete() } - _, _ = w.Write([]byte(f.logBody)) + body := f.logBody + if len(f.logs) > 0 { + body = pick(f.logs, f.logGets, "") + f.logGets++ + } + _, _ = w.Write([]byte(body)) case r.Method == http.MethodGet && strings.Contains(r.URL.Path, "/serviceaccounts/"): if f.saForbidden { w.WriteHeader(http.StatusForbidden) @@ -148,11 +185,18 @@ func (f *fakeAPI) handler(t *testing.T) http.HandlerFunc { } _, _ = w.Write([]byte(f.selfPod)) case r.Method == http.MethodGet && strings.Contains(r.URL.Path, "/pods/"): - state := `{"terminated":{"exitCode":` + itoa(f.exitCode) + `}}` + code := pick(f.exitCodes, f.podGets, f.exitCode) + node := pick(f.podNodes, f.podGets, "") + f.podGets++ + state := `{"terminated":{"exitCode":` + itoa(code) + `}}` if f.podStatus != "terminated" { state = `{"running":{}}` } - _, _ = w.Write([]byte(`{"status":{"containerStatuses":[{"name":"kick","state":` + state + `}]}}`)) + spec := "" + if node != "" { + spec = `"spec":{"nodeName":"` + node + `"},` + } + _, _ = w.Write([]byte(`{` + spec + `"status":{"containerStatuses":[{"name":"kick","state":` + state + `}]}}`)) default: t.Errorf("unexpected request %s %s", r.Method, r.URL) w.WriteHeader(http.StatusNotFound) diff --git a/src/crux/kubejob/kubejob_types.go b/src/crux/kubejob/kubejob_types.go new file mode 100644 index 0000000000..03f5436d50 --- /dev/null +++ b/src/crux/kubejob/kubejob_types.go @@ -0,0 +1,66 @@ +package kubejob + +import "context" + +// Toleration mirrors the subset of corev1.Toleration an operator writes in YAML. +type Toleration struct { + Key string `json:"key,omitempty"` + Operator string `json:"operator,omitempty"` + Value string `json:"value,omitempty"` + Effect string `json:"effect,omitempty"` +} + +// Resources mirrors corev1.ResourceRequirements with string quantities, so a +// device limit like `.com/: "2"` passes through untouched. +type Resources struct { + Limits map[string]string `json:"limits,omitempty"` + Requests map[string]string `json:"requests,omitempty"` +} + +// VolumeMount is an additional PVC to mount into the Job. +type VolumeMount struct { + Name string `json:"name"` + Claim string `json:"claim"` + MountPath string `json:"mount_path"` + ReadOnly bool `json:"read_only,omitempty"` +} + +// Network modes, Options.Network. +const ( + // NetworkDefault leaves the pod on the cluster network with whatever + // policies the namespace already has. The empty string means this. + NetworkDefault = "default" + // NetworkNone creates a deny-all NetworkPolicy selecting the Job's pods + // before the Job and removes it with the Job; the run is refused when + // the policy cannot be created. Enforcement is the CNI's: a cluster + // whose network plugin ignores NetworkPolicy isolates nothing, and the + // launcher cannot tell. + NetworkNone = "none" + // NetworkHost runs the pod with hostNetwork. + NetworkHost = "host" +) + +// deletePolicy removes the per-Job NetworkPolicy when one was created. +func (l *Launcher) deletePolicy(ctx context.Context, policyPath string) { + if policyPath != "" { + _ = l.delete(ctx, policyPath) + } +} + +// denyAllPolicy is a networking.k8s.io/v1 NetworkPolicy named after the +// Job that selects its pods and allows no ingress and no egress. +func denyAllPolicy(ns, job string) map[string]any { + return map[string]any{ + "apiVersion": "networking.k8s.io/v1", + "kind": "NetworkPolicy", + "metadata": map[string]any{ + "name": job, + "namespace": ns, + "labels": map[string]string{managedByLabel: managedByValue, jobLabel: job}, + }, + "spec": map[string]any{ + "podSelector": map[string]any{"matchLabels": map[string]string{jobLabel: job}}, + "policyTypes": []string{"Ingress", "Egress"}, + }, + } +} diff --git a/src/crux/kubejob/preflight.go b/src/crux/kubejob/preflight.go new file mode 100644 index 0000000000..5a9487b8f5 --- /dev/null +++ b/src/crux/kubejob/preflight.go @@ -0,0 +1,238 @@ +package kubejob + +import ( + "errors" + "fmt" + "strconv" + "strings" + "time" + + "github.com/hivecommons/hive/crux/config" + "github.com/hivecommons/hive/crux/sandbox" +) + +// Node preflight and fail-fast scheduling (hivecommons/hive#18811). +// +// A sandbox.preflight command list runs in the Job's own container, after the +// relay check and before the agent. The first failure exits the container with +// sandbox.ExitCodeNodeUnhealthy after printing preflightMarker, so the Job +// fails in seconds and Run can tell it from an agent that happened to exit 75. +// Run then re-creates the Job once with a node-affinity NotIn for the node the +// failed pod ran on (maxPreflightRetries), instead of waiting for the agent's +// next cadence. Nodes are never pinned: the exclusion is a NotIn on the +// kubernetes.io/hostname label added to the operator's own affinity. +// +// The commands come from configuration only. They are in the Job manifest and +// their output is in the pod log, so none of them may carry a credential on its +// command line; the script itself prints nothing from the environment. + +const ( + // DefaultPreflightTimeout bounds one preflight command; the config + // default is config.DefaultSandboxPreflightTimeoutS. + DefaultPreflightTimeout = time.Duration(config.DefaultSandboxPreflightTimeoutS) * time.Second + // DefaultPreflightTotalTimeout bounds all preflight commands together. + DefaultPreflightTotalTimeout = time.Duration(config.DefaultSandboxPreflightTotalTimeoutS) * time.Second + // DefaultScheduleTimeout is how long a pod may stay Unschedulable. + DefaultScheduleTimeout = time.Duration(config.DefaultSandboxScheduleTimeoutS) * time.Second + + // ExitCodeNodeUnhealthy is the container exit code of a failed preflight. + ExitCodeNodeUnhealthy = sandbox.ExitCodeNodeUnhealthy + + // maxPreflightRetries is how many times a node_unhealthy kick is + // re-created on another node: once. + maxPreflightRetries = 1 + // preflightMarker starts the line the preflight prints when it fails. + preflightMarker = "hive-node-preflight: FAILED" + // hostnameLabel is the node label the retry's NotIn matches. + hostnameLabel = "kubernetes.io/hostname" + // preflightMessageMax bounds the failure line quoted in the error. + preflightMessageMax = 500 + preflightShell = "/bin/sh" + // preflightKillGraceS is the pause between SIGTERM and SIGKILL for a + // preflight command that outlives its timeout. + preflightKillGraceS = 2 +) + +// preflightScript runs the commands passed after the three leading arguments +// (per-command seconds, total seconds, command count) one by one with `sh -lc`, +// as publish.verify does, each under a watchdog, then execs the agent command. +var preflightScript = strings.NewReplacer( + "@MARKER@", preflightMarker, + "@EXIT@", strconv.Itoa(ExitCodeNodeUnhealthy), + "@GRACE@", strconv.Itoa(preflightKillGraceS), +).Replace(`per="$1"; total="$2"; n="$3"; shift 3 +start=$(date +%s 2>/dev/null || echo 0) +i=0 +while [ "$i" -lt "$n" ]; do + i=$((i+1)) + cmd="$1"; shift + now=$(date +%s 2>/dev/null || echo 0) + left=$((total-(now-start))) + if [ "$left" -le 0 ]; then + echo "@MARKER@ the ${total}s total timeout ran out before command $i of $n: $cmd" >&2 + exit @EXIT@ + fi + t=$per + [ "$left" -lt "$t" ] && t=$left + sh -lc "$cmd" & + pid=$! + ( sleep "$t"; kill -TERM "$pid"; sleep @GRACE@; kill -KILL "$pid" ) >/dev/null 2>&1 /dev/null 2>&1 + if [ "$rc" -ne 0 ]; then + if [ "$rc" -eq 143 ] || [ "$rc" -eq 137 ]; then why="timed out after ${t}s"; else why="exited $rc"; fi + echo "@MARKER@ command $i of $n $why: $cmd" >&2 + exit @EXIT@ + fi +done +exec "$@"`) + +// preflightCommands returns the configured commands, blanks dropped. +func (o Options) preflightCommands() []string { + var out []string + for _, c := range o.Preflight { + if c = strings.TrimSpace(c); c != "" { + out = append(out, c) + } + } + return out +} + +// preflightCommand wraps the agent command so the preflight runs first. With no +// preflight configured, or no command to wrap, it returns command unchanged, so +// the Job manifest of a hive that never sets sandbox.preflight is byte-identical +// to what it was. +func preflightCommand(command []string, o Options) []string { + cmds := o.preflightCommands() + if len(cmds) == 0 || len(command) == 0 { + return command + } + per := durationSeconds(o.PreflightTimeout, DefaultPreflightTimeout) + total := durationSeconds(o.PreflightTotalTimeout, DefaultPreflightTotalTimeout) + out := []string{preflightShell, "-c", preflightScript, "hive-node-preflight", strconv.Itoa(per), strconv.Itoa(total), strconv.Itoa(len(cmds))} + out = append(out, cmds...) + return append(out, command...) +} + +// durationSeconds is d in whole seconds (at least one), or def's when d is unset. +func durationSeconds(d, def time.Duration) int { + if d <= 0 { + d = def + } + if s := int(d / time.Second); s > 0 { + return s + } + return 1 +} + +// scheduleTimeout is Options.ScheduleTimeout with the default applied. +func (o Options) scheduleTimeout() time.Duration { + if o.ScheduleTimeout > 0 { + return o.ScheduleTimeout + } + return DefaultScheduleTimeout +} + +// validatePreflight reports the preflight and schedule-timeout settings that +// could not run. +func (o Options) validatePreflight() error { + if len(o.Preflight) > config.MaxSandboxPreflightCommands { + return fmt.Errorf("kubejob: preflight takes at most %d commands, got %d", config.MaxSandboxPreflightCommands, len(o.Preflight)) + } + for i, c := range o.Preflight { + if strings.TrimSpace(c) == "" { + return fmt.Errorf("kubejob: preflight[%d] must not be empty", i) + } + } + for _, f := range []struct { + name string + d time.Duration + }{{"preflight timeout", o.PreflightTimeout}, {"preflight total timeout", o.PreflightTotalTimeout}, {"preflight retry delay", o.PreflightRetryDelay}, {"schedule timeout", o.ScheduleTimeout}} { + if f.d < 0 { + return errors.New("kubejob: " + f.name + " must not be negative") + } + } + return nil +} + +// preflightFailure reports whether a failed run is a failed node preflight, and +// the failure line it printed. The exit code alone is not enough: the agent may +// exit 75 for its own reasons, so the marker the script prints must be there too. +func (o Options) preflightFailure(res sandbox.Result) (string, bool) { + if len(o.preflightCommands()) == 0 || res.ExitCode != ExitCodeNodeUnhealthy { + return "", false + } + i := strings.LastIndex(res.Stdout, preflightMarker) + if i < 0 { + return "", false + } + line, _, _ := strings.Cut(res.Stdout[i:], "\n") + line = strings.TrimSpace(line) + if r := []rune(line); len(r) > preflightMessageMax { + line = string(r[:preflightMessageMax]) + "..." + } + return line, true +} + +// avoidNodes makes the Job's pod template refuse the given nodes: a NotIn on the +// hostname label, ANDed into every required node-selector term (terms are ORed, +// so adding it to one would let the pod through another). The operator's own +// affinity is copied, never mutated, because it is shared across kicks. +func avoidNodes(manifest map[string]any, affinity *config.SandboxJobAffinity, nodes []string) { + if len(nodes) == 0 { + return + } + jobSpec, _ := manifest["spec"].(map[string]any) + template, _ := jobSpec["template"].(map[string]any) + podSpec, _ := template["spec"].(map[string]any) + if podSpec == nil { + return + } + podSpec["affinity"] = affinityAvoiding(affinity, nodes) +} + +func affinityAvoiding(in *config.SandboxJobAffinity, nodes []string) *config.SandboxJobAffinity { + var node config.SandboxJobNodeAffinity + if in != nil && in.NodeAffinity != nil { + node = *in.NodeAffinity + } + avoid := config.SandboxJobNodeSelectorRequirement{Key: hostnameLabel, Operator: "NotIn", Values: append([]string(nil), nodes...)} + var terms []config.SandboxJobNodeSelectorTerm + if req := node.RequiredDuringSchedulingIgnoredDuringExecution; req != nil { + for _, t := range req.NodeSelectorTerms { + exprs := append(append([]config.SandboxJobNodeSelectorRequirement(nil), t.MatchExpressions...), avoid) + terms = append(terms, config.SandboxJobNodeSelectorTerm{MatchExpressions: exprs}) + } + } + if len(terms) == 0 { + terms = []config.SandboxJobNodeSelectorTerm{{MatchExpressions: []config.SandboxJobNodeSelectorRequirement{avoid}}} + } + node.RequiredDuringSchedulingIgnoredDuringExecution = &config.SandboxJobNodeSelector{NodeSelectorTerms: terms} + return &config.SandboxJobAffinity{NodeAffinity: &node} +} + +// Event kinds Options.OnEvent receives. +const ( + // EventWaitingForNode: the pod is Unschedulable; Detail is the scheduler's message. + EventWaitingForNode = "waiting_for_node" + // EventScheduled: the pod no longer waits for a node; Node is where it landed. + EventScheduled = "scheduled" + // EventNodeRetry: a preflight failed on Node and the Job is being + // re-created on another; Detail is the preflight's failure line. + EventNodeRetry = "node_retry" +) + +// Event is one thing worth surfacing while a Job runs. +type Event struct { + Kind string + Node string + Detail string +} + +func (l *Launcher) emit(e Event) { + if l.Options.OnEvent != nil { + l.Options.OnEvent(e) + } +} diff --git a/src/crux/kubejob/preflight_test.go b/src/crux/kubejob/preflight_test.go new file mode 100644 index 0000000000..9ab9ae6e0f --- /dev/null +++ b/src/crux/kubejob/preflight_test.go @@ -0,0 +1,532 @@ +package kubejob + +import ( + "context" + "encoding/json" + "errors" + "net/http/httptest" + "os" + "os/exec" + "reflect" + "strings" + "sync" + "testing" + "time" + + "github.com/hivecommons/hive/crux/config" + "github.com/hivecommons/hive/crux/sandbox" +) + +const preflightWorkspace = "/data/agents/sandbox/deps-123" + +// preflightLog is the pod log of a container whose preflight failed. +const preflightLog = "torch: no spyre device\n" + preflightMarker + " command 1 of 1 exited 1: python -c smoke\n" + +// agentSpec is an agent kick: Relays is what marks it as one. +func agentSpec() sandbox.LaunchSpec { + spec := specFor(preflightWorkspace) + spec.Relays = true + return spec +} + +func preflightOptions() Options { + return Options{Preflight: []string{"python -c smoke"}, RelayImage: RelayImageNone} +} + +// runScript executes the real preflight wrapper with /bin/sh, as the Job's +// container would, and returns its combined output and exit code. +func runScript(t *testing.T, o Options, agent []string, env ...string) (string, int) { + t.Helper() + argv := preflightCommand(agent, o) + cmd := exec.Command(argv[0], argv[1:]...) + cmd.Env = append(os.Environ(), env...) + out, err := cmd.CombinedOutput() + code := 0 + var ee *exec.ExitError + if errors.As(err, &ee) { + code = ee.ExitCode() + } else if err != nil { + t.Fatalf("running the preflight script: %v", err) + } + return string(out), code +} + +func TestPreflightScriptPassingStartsTheAgent(t *testing.T) { + out, code := runScript(t, Options{Preflight: []string{"true", "echo smoke-ok"}}, []string{"echo", "agent-started"}) + if code != 0 || !strings.Contains(out, "agent-started") || !strings.Contains(out, "smoke-ok") { + t.Fatalf("code=%d out=%q, want the preflight to pass and the agent to start", code, out) + } + if strings.Contains(out, preflightMarker) { + t.Fatalf("a passing preflight printed the failure marker: %q", out) + } +} + +func TestPreflightScriptFailureStopsBeforeTheAgent(t *testing.T) { + out, code := runScript(t, Options{Preflight: []string{"true", "echo device-gone >&2; exit 3"}}, []string{"echo", "agent-started"}) + if code != ExitCodeNodeUnhealthy { + t.Fatalf("exit code = %d, want %d", code, ExitCodeNodeUnhealthy) + } + if strings.Contains(out, "agent-started") { + t.Fatalf("the agent ran after a failed preflight: %q", out) + } + for _, want := range []string{"device-gone", preflightMarker + " command 2 of 2 exited 3: echo device-gone"} { + if !strings.Contains(out, want) { + t.Errorf("output %q lacks %q", out, want) + } + } +} + +func TestPreflightScriptTimesOutAHungCommand(t *testing.T) { + o := Options{Preflight: []string{"exec sleep 30"}, PreflightTimeout: time.Second} + start := time.Now() + out, code := runScript(t, o, []string{"echo", "agent-started"}) + if code != ExitCodeNodeUnhealthy || !strings.Contains(out, "timed out after 1s") { + t.Fatalf("code=%d out=%q, want a timeout failure", code, out) + } + if time.Since(start) > 20*time.Second { + t.Fatalf("the hung command was not cut off: took %s", time.Since(start)) + } +} + +func TestPreflightScriptLogsNoCredential(t *testing.T) { + const secret = "ghp_preflightsecretvalue0123456789" + out, code := runScript(t, Options{Preflight: []string{"exit 1"}}, []string{"true"}, "GITHUB_TOKEN="+secret, "MY_SECRET="+secret) + if code != ExitCodeNodeUnhealthy { + t.Fatalf("exit code = %d, want %d", code, ExitCodeNodeUnhealthy) + } + if strings.Contains(out, secret) { + t.Fatalf("the preflight output carries a credential from the environment: %q", out) + } +} + +func TestPreflightCommandUnchangedWithoutConfig(t *testing.T) { + cmd := []string{"claude", "-p"} + if got := preflightCommand(cmd, Options{}); !reflect.DeepEqual(got, cmd) { + t.Fatalf("no preflight must leave the command alone, got %q", got) + } + if got := preflightCommand(cmd, Options{Preflight: []string{" ", ""}}); !reflect.DeepEqual(got, cmd) { + t.Fatalf("blank preflight entries must leave the command alone, got %q", got) + } + if got := preflightCommand(nil, Options{Preflight: []string{"true"}}); got != nil { + t.Fatalf("an empty command cannot be wrapped, got %q", got) + } +} + +func TestPreflightDefaultsAreSafe(t *testing.T) { + if got := (Options{}).scheduleTimeout(); got != 10*time.Minute { + t.Errorf("default schedule timeout = %s, want 10m", got) + } + if DefaultPreflightTimeout != 2*time.Minute || DefaultPreflightTotalTimeout != 5*time.Minute { + t.Errorf("preflight defaults = %s / %s", DefaultPreflightTimeout, DefaultPreflightTotalTimeout) + } + argv := preflightCommand([]string{"agent"}, Options{Preflight: []string{"true"}}) + if argv[4] != "120" || argv[5] != "300" || argv[6] != "1" { + t.Errorf("default per/total/count args = %v", argv[4:7]) + } +} + +func TestManifestWithoutPreflightIsUnchanged(t *testing.T) { + l := &Launcher{Options: Options{WorkspaceClaim: "hive-data"}} + spec := agentSpec() + m := manifestJSON(t, l.jobManifest("hive", "hive-kick-x", "deps", spec, nil)) + if !strings.Contains(m, `"command":["claude","-p","@.hive/kick-prompt.txt"]`) { + t.Fatalf("the command changed without a preflight: %s", m) + } + if strings.Contains(m, "hive-node-preflight") { + t.Fatalf("preflight script in a manifest that configures none: %s", m) + } +} + +func TestManifestRunsPreflightAfterTheRelayCheck(t *testing.T) { + l := &Launcher{Options: Options{WorkspaceClaim: "hive-data", Preflight: []string{"python -c smoke"}}} + m := l.jobManifest("hive", "hive-kick-x", "deps", agentSpec(), &relayPlan{image: "hive:test"}) + cmd := firstContainerCommand(t, m) + if len(cmd) < 8 || cmd[3] != "hive-relay-preflight" { + t.Fatalf("command does not start with the relay check: %q", cmd) + } + rest := cmd[5:] + if rest[0] != preflightShell || rest[3] != "hive-node-preflight" || rest[7] != "python -c smoke" { + t.Fatalf("the node preflight does not follow the relay check: %q", rest) + } + if got := rest[8:]; !reflect.DeepEqual(got, []string{"claude", "-p", "@.hive/kick-prompt.txt"}) { + t.Fatalf("the agent command must come last, got %q", got) + } +} + +func firstContainerCommand(t *testing.T, manifest map[string]any) []string { + t.Helper() + var m struct { + Spec struct { + Template struct { + Spec struct { + Containers []struct { + Command []string `json:"command"` + } `json:"containers"` + } `json:"spec"` + } `json:"template"` + } `json:"spec"` + } + if err := json.Unmarshal([]byte(manifestJSON(t, manifest)), &m); err != nil || len(m.Spec.Template.Spec.Containers) == 0 { + t.Fatalf("decoding the manifest: %v", err) + } + return m.Spec.Template.Spec.Containers[0].Command +} + +func manifestJSON(t *testing.T, manifest map[string]any) string { + t.Helper() + raw, err := json.Marshal(manifest) + if err != nil { + t.Fatal(err) + } + return string(raw) +} + +func TestPreflightOptionsValidation(t *testing.T) { + good := Options{WorkspaceClaim: "c", Preflight: []string{"true"}} + if err := good.Validate(); err != nil { + t.Fatalf("valid preflight refused: %v", err) + } + for name, mutate := range map[string]func(*Options){ + "empty command": func(o *Options) { o.Preflight = []string{"true", " "} }, + "negative timeout": func(o *Options) { o.PreflightTimeout = -time.Second }, + "negative total": func(o *Options) { o.PreflightTotalTimeout = -time.Second }, + "negative delay": func(o *Options) { o.PreflightRetryDelay = -time.Second }, + "negative schedule": func(o *Options) { o.ScheduleTimeout = -time.Second }, + "too many": func(o *Options) { o.Preflight = make([]string, config.MaxSandboxPreflightCommands+1) }, + } { + o := good + mutate(&o) + if err := o.Validate(); err == nil { + t.Errorf("%s: want a validation error", name) + } + } +} + +func TestAffinityAvoidingNodes(t *testing.T) { + avoid := config.SandboxJobNodeSelectorRequirement{Key: "kubernetes.io/hostname", Operator: "NotIn", Values: []string{"node-a"}} + got := affinityAvoiding(nil, []string{"node-a"}) + terms := got.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms + if len(terms) != 1 || !reflect.DeepEqual(terms[0].MatchExpressions, []config.SandboxJobNodeSelectorRequirement{avoid}) { + t.Fatalf("nil affinity: %+v", got) + } + in := &config.SandboxJobAffinity{NodeAffinity: &config.SandboxJobNodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &config.SandboxJobNodeSelector{NodeSelectorTerms: []config.SandboxJobNodeSelectorTerm{ + {MatchExpressions: []config.SandboxJobNodeSelectorRequirement{{Key: "pool", Operator: "In", Values: []string{"spyre"}}}}, + {MatchExpressions: []config.SandboxJobNodeSelectorRequirement{{Key: "pool", Operator: "In", Values: []string{"spyre-b"}}}}, + }}, + PreferredDuringSchedulingIgnoredDuringExecution: []config.SandboxJobPreferredSchedulingTerm{{Weight: 5, Preference: config.SandboxJobNodeSelectorTerm{MatchExpressions: []config.SandboxJobNodeSelectorRequirement{{Key: "x", Operator: "Exists"}}}}}, + }} + before, _ := json.Marshal(in) + got = affinityAvoiding(in, []string{"node-a"}) + // Terms are ORed, so the exclusion has to be ANDed into every one. + for i, term := range got.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms { + if n := len(term.MatchExpressions); n != 2 || !reflect.DeepEqual(term.MatchExpressions[1], avoid) { + t.Errorf("term %d = %+v, want the operator's expression plus the exclusion", i, term) + } + } + if len(got.NodeAffinity.PreferredDuringSchedulingIgnoredDuringExecution) != 1 { + t.Error("the preferred terms were dropped") + } + if after, _ := json.Marshal(in); string(after) != string(before) { + t.Fatalf("the operator's shared affinity was mutated:\n%s\n%s", before, after) + } +} + +// events collects launcher events from the polling goroutine. +type events struct { + mu sync.Mutex + all []Event +} + +func (e *events) add(ev Event) { e.mu.Lock(); e.all = append(e.all, ev); e.mu.Unlock() } +func (e *events) kinds() []string { + e.mu.Lock() + defer e.mu.Unlock() + var k []string + for _, ev := range e.all { + k = append(k, ev.Kind) + } + return k +} + +func TestRunPassingPreflightStartsTheAgentOnce(t *testing.T) { + api := newFakeAPI() + api.statuses = []string{statusComplete} + api.podNodes = []string{"node-a"} + srv := httptest.NewServer(api.handler(t)) + defer srv.Close() + l := newLauncher(srv, preflightOptions()) + + res, err := l.Run(context.Background(), agentSpec()) + if err != nil { + t.Fatalf("Run: %v", err) + } + if api.jobCreations != 1 || res.FailReason != "" || res.Node != "node-a" { + t.Fatalf("creations=%d result=%+v, want one Job on node-a and no failure", api.jobCreations, res) + } + cmd := firstContainerCommand(t, api.created) + if len(cmd) < 8 || cmd[3] != "hive-node-preflight" || cmd[len(cmd)-1] != "@.hive/kick-prompt.txt" { + t.Fatalf("the Job does not run the preflight then the agent: %q", cmd) + } +} + +func TestRunPreflightFailureFailsFastAndRetriesOnAnotherNode(t *testing.T) { + api := newFakeAPI() + api.statuses = []string{statusFailed, statusComplete} + api.exitCodes = []int{ExitCodeNodeUnhealthy, 0} + api.podNodes = []string{"node-a", "node-b"} + api.logs = []string{preflightLog, "agent ok\n"} + srv := httptest.NewServer(api.handler(t)) + defer srv.Close() + opts := preflightOptions() + var ev events + opts.OnEvent = ev.add + opts.Affinity = &config.SandboxJobAffinity{NodeAffinity: &config.SandboxJobNodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &config.SandboxJobNodeSelector{NodeSelectorTerms: []config.SandboxJobNodeSelectorTerm{ + {MatchExpressions: []config.SandboxJobNodeSelectorRequirement{{Key: "pool", Operator: "In", Values: []string{"spyre"}}}}, + }}, + }} + l := newLauncher(srv, opts) + + res, err := l.Run(context.Background(), agentSpec()) + if err != nil { + t.Fatalf("Run: %v (the retry on node-b passed)", err) + } + if api.jobCreations != 2 { + t.Fatalf("Jobs created = %d, want the original and one retry", api.jobCreations) + } + first, second := api.createdAll[0], api.createdAll[1] + if manifestName(first) == manifestName(second) || !strings.HasSuffix(manifestName(second), "-r1") { + t.Errorf("job names %q / %q: the retry needs its own name", manifestName(first), manifestName(second)) + } + if strings.Contains(manifestJSON(t, first), "NotIn") { + t.Errorf("the first attempt already excluded a node: %s", manifestJSON(t, first)) + } + retry := manifestJSON(t, second) + for _, want := range []string{`"key":"kubernetes.io/hostname"`, `"operator":"NotIn"`, `"values":["node-a"]`, `"key":"pool"`} { + if !strings.Contains(retry, want) { + t.Errorf("retry manifest lacks %s: %s", want, retry) + } + } + if res.Node != "node-b" || res.FailReason != "" { + t.Errorf("result = %+v, want a clean run on node-b", res) + } + for _, want := range []string{"attempt 1 on node node-a failed the node preflight", preflightMarker, "agent ok"} { + if !strings.Contains(res.Stdout, want) { + t.Errorf("transcript lacks %q: %q", want, res.Stdout) + } + } + if got := ev.kinds(); !reflect.DeepEqual(got, []string{EventNodeRetry}) { + t.Errorf("events = %v, want one node_retry", got) + } +} + +func manifestName(manifest map[string]any) string { + md, _ := manifest["metadata"].(map[string]any) + n, _ := md["name"].(string) + return n +} + +func TestRunPreflightFailsOnTheRetryNodeToo(t *testing.T) { + api := newFakeAPI() + api.statuses = []string{statusFailed} + api.exitCodes = []int{ExitCodeNodeUnhealthy} + api.podNodes = []string{"node-a", "node-b"} + api.logs = []string{preflightLog} + srv := httptest.NewServer(api.handler(t)) + defer srv.Close() + l := newLauncher(srv, preflightOptions()) + + res, err := l.Run(context.Background(), agentSpec()) + var fe *sandbox.FailError + if !errors.As(err, &fe) || fe.Reason != sandbox.FailReasonNodeUnhealthy || fe.Node != "node-b" { + t.Fatalf("Run error = %v, want node_unhealthy on node-b", err) + } + if !strings.Contains(fe.Message, preflightMarker) || !strings.Contains(fe.Message, "python -c smoke") { + t.Errorf("message = %q, want the failing command", fe.Message) + } + if api.jobCreations != 2 { + t.Errorf("Jobs created = %d, want exactly one retry", api.jobCreations) + } + if res.FailReason != sandbox.FailReasonNodeUnhealthy || res.Node != "node-b" || res.ExitCode != ExitCodeNodeUnhealthy { + t.Errorf("result = %+v", res) + } + if !strings.Contains(res.Stdout, "no spyre device") { + t.Errorf("the command's output is missing from the transcript: %q", res.Stdout) + } + if api.deleted { + t.Error("failed Jobs are left for inspection until their TTL") + } +} + +func TestRunAgentExitingWithTheSameCodeIsNotANodeFailure(t *testing.T) { + api := newFakeAPI() + api.statuses = []string{statusFailed} + api.exitCode = ExitCodeNodeUnhealthy + api.podNodes = []string{"node-a"} + api.logBody = "agent gave up\n" + srv := httptest.NewServer(api.handler(t)) + defer srv.Close() + l := newLauncher(srv, preflightOptions()) + + res, err := l.Run(context.Background(), agentSpec()) + var fe *sandbox.FailError + if err == nil || errors.As(err, &fe) || res.FailReason != "" { + t.Fatalf("err=%v result=%+v: an agent's own exit 75 without the marker is an ordinary failure", err, res) + } + if api.jobCreations != 1 { + t.Errorf("Jobs created = %d, want no retry", api.jobCreations) + } +} + +func TestRunWithoutAKnownNodeDoesNotRetry(t *testing.T) { + api := newFakeAPI() + api.statuses = []string{statusFailed} + api.exitCode = ExitCodeNodeUnhealthy + api.logBody = preflightLog + srv := httptest.NewServer(api.handler(t)) + defer srv.Close() + l := newLauncher(srv, preflightOptions()) + + _, err := l.Run(context.Background(), agentSpec()) + var fe *sandbox.FailError + if !errors.As(err, &fe) || api.jobCreations != 1 { + t.Fatalf("err=%v creations=%d: with no node to exclude there is nothing to retry on", err, api.jobCreations) + } +} + +func TestRunOnlyAgentKicksGetThePreflight(t *testing.T) { + api := newFakeAPI() + api.statuses = []string{statusComplete} + srv := httptest.NewServer(api.handler(t)) + defer srv.Close() + l := newLauncher(srv, preflightOptions()) + spec := specFor(preflightWorkspace) // Relays unset: a publish.verify launch + if _, err := l.Run(context.Background(), spec); err != nil { + t.Fatalf("Run: %v", err) + } + if cmd := firstContainerCommand(t, api.created); !reflect.DeepEqual(cmd, spec.Command) { + t.Fatalf("a non-agent launch was gated on the node preflight: %q", cmd) + } + if len(l.Options.Preflight) != 1 { + t.Error("the shared launcher's options were modified") + } +} + +func TestRunPreflightWithoutACommandIsRefused(t *testing.T) { + spec := agentSpec() + spec.Command = nil + srv := httptest.NewServer(newFakeAPI().handler(t)) + defer srv.Close() + l := newLauncher(srv, preflightOptions()) + if _, err := l.Run(context.Background(), spec); err == nil || !strings.Contains(err.Error(), "explicit command") { + t.Fatalf("Run error = %v, want a refusal: there is no agent command to run after the preflight", err) + } +} + +func TestRetryDelayIsHonoredAndCancellable(t *testing.T) { + api := newFakeAPI() + api.statuses = []string{statusFailed} + api.exitCode = ExitCodeNodeUnhealthy + api.podNodes = []string{"node-a"} + api.logBody = preflightLog + srv := httptest.NewServer(api.handler(t)) + defer srv.Close() + opts := preflightOptions() + opts.PreflightRetryDelay = time.Hour + l := newLauncher(srv, opts) + + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + _, err := l.Run(ctx, agentSpec()) + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("Run error = %v, want the context to end the retry delay", err) + } + if api.jobCreations != 1 { + t.Errorf("Jobs created = %d, want no retry before the delay", api.jobCreations) + } +} + +// steppingClock advances a minute per reading, so a poll loop sees time pass +// without the test sleeping. +func steppingClock() func() time.Time { + var mu sync.Mutex + t := time.Unix(1700000000, 0) + return func() time.Time { + mu.Lock() + defer mu.Unlock() + t = t.Add(time.Minute) + return t + } +} + +const ( + unschedulablePod = `{"items":[{"metadata":{"name":"hive-kick-deps-1-abcde"},"spec":{},"status":{"phase":"Pending","conditions":[{"type":"PodScheduled","status":"False","reason":"Unschedulable","message":"0/38 nodes are available: 38 node(s) didn't match Pod's node affinity/selector."}]}}]}` + scheduledPod = `{"items":[{"metadata":{"name":"hive-kick-deps-1-abcde"},"spec":{"nodeName":"node-a"},"status":{"phase":"Running"}}]}` + creatingPod = `{"items":[{"metadata":{"name":"hive-kick-deps-1-abcde"},"spec":{},"status":{"phase":"Pending"}}]}` +) + +func TestRunUnschedulablePastTheTimeoutFailsFastAndDeletesTheJob(t *testing.T) { + api := newFakeAPI() + api.statuses = []string{statusPending} + api.podLists = []string{unschedulablePod} + srv := httptest.NewServer(api.handler(t)) + defer srv.Close() + var ev events + l := newLauncher(srv, Options{ScheduleTimeout: 3 * time.Minute, OnEvent: ev.add}) + l.Now = steppingClock() + + res, err := l.Run(context.Background(), specFor(preflightWorkspace)) + var fe *sandbox.FailError + if !errors.As(err, &fe) || fe.Reason != sandbox.FailReasonWaitingForNode { + t.Fatalf("Run error = %v, want waiting_for_node", err) + } + if !strings.Contains(fe.Message, "0/38 nodes are available") { + t.Errorf("message = %q, want the scheduler's reason", fe.Message) + } + if !api.deleted { + t.Error("the unschedulable Job must be deleted so the agent's slot is released") + } + if res.FailReason != sandbox.FailReasonWaitingForNode || res.ExitCode != exitCodeUnknown || !strings.Contains(res.Stderr, "0/38 nodes") { + t.Errorf("result = %+v", res) + } + if got := ev.kinds(); !reflect.DeepEqual(got, []string{EventWaitingForNode}) { + t.Errorf("events = %v, want one waiting_for_node", got) + } +} + +func TestRunUnschedulableBelowTheTimeoutKeepsWaiting(t *testing.T) { + api := newFakeAPI() + api.statuses = []string{statusPending, statusPending, statusPending, statusComplete} + api.podLists = []string{unschedulablePod, scheduledPod} + api.podNodes = []string{"node-a"} + srv := httptest.NewServer(api.handler(t)) + defer srv.Close() + var ev events + l := newLauncher(srv, Options{ScheduleTimeout: time.Hour, OnEvent: ev.add}) + l.Now = steppingClock() + + res, err := l.Run(context.Background(), specFor(preflightWorkspace)) + if err != nil { + t.Fatalf("Run: %v (the pod was scheduled before the timeout)", err) + } + if res.Node != "node-a" || res.FailReason != "" { + t.Errorf("result = %+v", res) + } + if got := ev.kinds(); !reflect.DeepEqual(got, []string{EventWaitingForNode, EventScheduled}) { + t.Errorf("events = %v, want waiting then scheduled", got) + } +} + +func TestRunPendingWithoutUnschedulableConditionNeverTrips(t *testing.T) { + api := newFakeAPI() + api.statuses = []string{statusPending, statusPending, statusPending, statusComplete} + api.podLists = []string{creatingPod} + srv := httptest.NewServer(api.handler(t)) + defer srv.Close() + l := newLauncher(srv, Options{ScheduleTimeout: time.Minute}) + l.Now = steppingClock() + if _, err := l.Run(context.Background(), specFor(preflightWorkspace)); err != nil { + t.Fatalf("Run: %v (a pulling image is not unschedulable)", err) + } +} diff --git a/src/crux/kubejob/sched.go b/src/crux/kubejob/sched.go new file mode 100644 index 0000000000..95b265f55f --- /dev/null +++ b/src/crux/kubejob/sched.go @@ -0,0 +1,197 @@ +package kubejob + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "strings" + "time" + + "github.com/hivecommons/hive/crux/sandbox" +) + +// Fail-fast on an unschedulable Job (#18811). A Job whose pod can never be +// placed (no node matches, a device is exhausted) used to hold the agent's slot +// until the kick timeout. wait now also reads the pod while it has no node, and +// once PodScheduled has been False/Unschedulable for the schedule timeout it +// gives up with sandbox.FailReasonWaitingForNode and the scheduler's message. + +const ( + podScheduledCond = "PodScheduled" + unschedulable = "Unschedulable" + // scheduleMessageMax bounds the scheduler's message quoted in the error. + scheduleMessageMax = 500 +) + +// podState is what the schedule watch reads off the Job's pod. +type podState struct { + name string + node string + unschedulable bool + message string +} + +// scheduleWatch is the watch's state across polls. +type scheduleWatch struct { + scheduled bool + node string + since time.Time +} + +func (w *scheduleWatch) waiting() bool { return !w.since.IsZero() } + +// waitJob polls the Job until a Complete or Failed condition, the schedule +// timeout, or ctx ending. +func (l *Launcher) waitJob(ctx context.Context, ns, name, jobPath string) (jobOutcome, error) { + interval := l.Options.PollInterval + if interval <= 0 { + interval = DefaultPollInterval + } + ticker := time.NewTicker(interval) + defer ticker.Stop() + var watch scheduleWatch + for { + raw, err := l.do(ctx, http.MethodGet, jobPath, nil) + if err != nil { + if ctx.Err() != nil { + return jobOutcome{node: watch.node}, fmt.Errorf("kubejob: %w", ctx.Err()) + } + // Transient API error: keep polling; the context still bounds us. + } else if finished, ok := parseJobStatus(raw); ok { + finished.node = watch.node + return finished, nil + } else if tripped, ok := l.watchSchedule(ctx, ns, name, &watch); ok { + return tripped, nil + } + select { + case <-ctx.Done(): + return jobOutcome{node: watch.node}, fmt.Errorf("kubejob: %w", ctx.Err()) + case <-ticker.C: + } + } +} + +// watchSchedule reads the pod while it has no node. It reports the outcome and +// true once the pod has been Unschedulable for the schedule timeout. +func (l *Launcher) watchSchedule(ctx context.Context, ns, job string, w *scheduleWatch) (jobOutcome, bool) { + if w.scheduled { + return jobOutcome{}, false + } + st, err := l.podState(ctx, ns, job) + if err != nil || st.name == "" { + return jobOutcome{}, false + } + if st.node != "" { + w.scheduled, w.node = true, st.node + if w.waiting() { + w.since = time.Time{} + l.emit(Event{Kind: EventScheduled, Node: st.node}) + } + return jobOutcome{}, false + } + if !st.unschedulable { + if w.waiting() { + w.since = time.Time{} + l.emit(Event{Kind: EventScheduled}) + } + return jobOutcome{}, false + } + now := l.now() + if !w.waiting() { + w.since = now + l.emit(Event{Kind: EventWaitingForNode, Detail: st.message}) + } + if now.Sub(w.since) >= l.Options.scheduleTimeout() { + return jobOutcome{failReason: sandbox.FailReasonWaitingForNode, reason: st.message}, true + } + return jobOutcome{}, false +} + +// podState reads the Job's pod: where it landed, or why it cannot. +func (l *Launcher) podState(ctx context.Context, ns, job string) (podState, error) { + path := fmt.Sprintf("/api/v1/namespaces/%s/pods?labelSelector=%s", ns, fmt.Sprintf(labelSelectorFmt, jobLabelKey, job)) + raw, err := l.do(ctx, http.MethodGet, path, nil) + if err != nil { + return podState{}, err + } + var list struct { + Items []struct { + Metadata struct { + Name string `json:"name"` + } `json:"metadata"` + Spec struct { + NodeName string `json:"nodeName"` + } `json:"spec"` + Status struct { + Conditions []struct { + Type string `json:"type"` + Status string `json:"status"` + Reason string `json:"reason"` + Message string `json:"message"` + } `json:"conditions"` + } `json:"status"` + } `json:"items"` + } + if err := json.Unmarshal(raw, &list); err != nil { + return podState{}, err + } + if len(list.Items) == 0 { + return podState{}, nil + } + // backoffLimit is 0, so there is at most one pod; take the last. + item := list.Items[len(list.Items)-1] + st := podState{name: item.Metadata.Name, node: item.Spec.NodeName} + for _, c := range item.Status.Conditions { + if c.Type == podScheduledCond && c.Status == "False" && c.Reason == unschedulable { + st.unschedulable = true + st.message = strings.TrimSpace(c.Message) + if r := []rune(st.message); len(r) > scheduleMessageMax { + st.message = string(r[:scheduleMessageMax]) + "..." + } + } + } + return st, nil +} + +// podDetails is the kick container's exit code and the node the pod ran on. +type podDetails struct { + exitCode int + node string +} + +// podDetails reads both off the pod, with exitCodeUnknown and "" when it cannot. +func (l *Launcher) podDetails(ctx context.Context, ns, pod string) podDetails { + out := podDetails{exitCode: exitCodeUnknown} + path := fmt.Sprintf("/api/v1/namespaces/%s/pods/%s", ns, pod) + raw, err := l.do(ctx, http.MethodGet, path, nil) + if err != nil { + return out + } + var p struct { + Spec struct { + NodeName string `json:"nodeName"` + } `json:"spec"` + Status struct { + ContainerStatuses []struct { + Name string `json:"name"` + State map[string]struct { + ExitCode int `json:"exitCode"` + } `json:"state"` + } `json:"containerStatuses"` + } `json:"status"` + } + if err := json.Unmarshal(raw, &p); err != nil { + return out + } + out.node = p.Spec.NodeName + for _, cs := range p.Status.ContainerStatuses { + if cs.Name != containerName { + continue + } + if term, ok := cs.State[stateTerminated]; ok { + out.exitCode = term.ExitCode + } + } + return out +} diff --git a/src/crux/process.go b/src/crux/process.go index 12064374e1..1f7249fbe4 100644 --- a/src/crux/process.go +++ b/src/crux/process.go @@ -77,6 +77,7 @@ type Process struct { ActionNudges int KickOutcome KickOutcome SandboxLastRun *SandboxRun + SandboxPhase *SandboxPhase StartFailureClass string StartFailureReason string StartFailureCount int diff --git a/src/crux/process_types.go b/src/crux/process_types.go index db4d8229e2..3d19486a8a 100644 --- a/src/crux/process_types.go +++ b/src/crux/process_types.go @@ -161,6 +161,22 @@ type SandboxRun struct { // scrubbed tail of the transcript. Bounded; see sandboxRunSummaryMax. Summary string `json:"summary,omitempty"` Usage SandboxRunUsage `json:"usage,omitempty"` + // FailReason is "node_unhealthy" (the node preflight failed) or + // "waiting_for_node" (no node took the pod in time) when the run failed + // for the infrastructure's reason rather than the agent's; Node is where + // the pod ran, when known (hivecommons/hive#18811). + FailReason string `json:"failReason,omitempty"` + Node string `json:"node,omitempty"` +} + +// SandboxPhase is what an in-flight sandboxed kick is doing when that is not +// plain "running": Phase "waiting_for_node" means its Job's pod is Pending +// and Unschedulable, with the scheduler's message in Detail. +type SandboxPhase struct { + Phase string `json:"phase"` + Node string `json:"node,omitempty"` + Detail string `json:"detail,omitempty"` + Since time.Time `json:"since"` } // Duration is how long the run took, zero when the record is incomplete. diff --git a/src/crux/sandbox/failreason.go b/src/crux/sandbox/failreason.go new file mode 100644 index 0000000000..c2a5dea82d --- /dev/null +++ b/src/crux/sandbox/failreason.go @@ -0,0 +1,33 @@ +package sandbox + +import "fmt" + +// Failure reasons a launcher reports in Result.FailReason (hivecommons/hive#18811). +// Both mean the run never reached the agent through no fault of the agent, so +// the executor and the manager do not count them as agent failures. +const ( + // FailReasonNodeUnhealthy: the node preflight (sandbox.preflight) failed + // on the node the Job landed on. + FailReasonNodeUnhealthy = "node_unhealthy" + // FailReasonWaitingForNode: the Job's pod stayed Unschedulable past + // job.schedule_timeout_s, so hive deleted the Job. + FailReasonWaitingForNode = "waiting_for_node" + // ExitCodeNodeUnhealthy is the exit code of a Job container whose node + // preflight failed (EX_TEMPFAIL). + ExitCodeNodeUnhealthy = 75 +) + +// FailError is the error a launcher returns with a Result whose FailReason is +// set. Message is the preflight failure line or the scheduler's message. +type FailError struct { + Reason string + Node string + Message string +} + +func (e *FailError) Error() string { + if e.Node != "" { + return fmt.Sprintf("sandbox: %s on node %s: %s", e.Reason, e.Node, e.Message) + } + return fmt.Sprintf("sandbox: %s: %s", e.Reason, e.Message) +} diff --git a/src/crux/sandbox/failreason_test.go b/src/crux/sandbox/failreason_test.go new file mode 100644 index 0000000000..fefe827423 --- /dev/null +++ b/src/crux/sandbox/failreason_test.go @@ -0,0 +1,24 @@ +package sandbox + +import ( + "errors" + "strings" + "testing" +) + +func TestFailErrorNamesReasonAndNode(t *testing.T) { + err := error(&FailError{Reason: FailReasonNodeUnhealthy, Node: "node-a", Message: "device gone"}) + var fe *FailError + if !errors.As(err, &fe) || fe.Reason != "node_unhealthy" { + t.Fatalf("errors.As lost the reason: %v", err) + } + if got := err.Error(); !strings.Contains(got, "node_unhealthy") || !strings.Contains(got, "node-a") || !strings.Contains(got, "device gone") { + t.Errorf("Error() = %q", got) + } + if got := (&FailError{Reason: FailReasonWaitingForNode, Message: "0/3 nodes"}).Error(); strings.Contains(got, "on node") || !strings.Contains(got, "waiting_for_node") { + t.Errorf("a node-less error = %q", got) + } + if ExitCodeNodeUnhealthy != 75 { + t.Errorf("ExitCodeNodeUnhealthy = %d, want 75", ExitCodeNodeUnhealthy) + } +} diff --git a/src/crux/sandbox/sandbox.go b/src/crux/sandbox/sandbox.go index 68a2330361..88a383d123 100644 --- a/src/crux/sandbox/sandbox.go +++ b/src/crux/sandbox/sandbox.go @@ -54,6 +54,13 @@ type Result struct { Stdout string Stderr string ExitCode int + // Node is the node the run's pod was scheduled on, when the launcher can + // tell (the Job runtime); empty otherwise. + Node string + // FailReason is set when the run failed for a reason that is the + // infrastructure's, not the agent's: FailReasonNodeUnhealthy or + // FailReasonWaitingForNode. Empty for every other outcome. + FailReason string } type Launcher interface { diff --git a/src/docs/sandbox-isolation.md b/src/docs/sandbox-isolation.md index 1251b2ec41..26c83a7997 100644 --- a/src/docs/sandbox-isolation.md +++ b/src/docs/sandbox-isolation.md @@ -258,6 +258,38 @@ conflict with the exclusion and leave the Job pending. If using a custom `scheduler_name`, ensure that scheduler honors Kubernetes `nodeAffinity`; otherwise the exclusion is not guaranteed. +### Node preflight and failing fast on an unschedulable Job + +Two checks keep a bad or missing node from costing a whole kick ([#18811](https://github.com/hivecommons/hive/issues/18811)). Both are for `runtime: job`. + +**`sandbox.preflight`: a node health check before the agent starts.** An accelerator card can be unusable on one node (a VFIO mmap failure, a crashed device) while the pod starts fine; without a check hive learns that only when the agent's own tests fail, minutes in. `preflight.commands` is a list of shell commands run in the Job's main container after hive's relay check and before the agent, in the same image, workspace and environment. Each runs as `sh -lc `, like `publish.verify`, under `timeout_s` (default 120 s) and a `total_timeout_s` for the whole list (default 300 s). It is off unless `commands` is set, per agent (`agents..sandbox.preflight`) or hive-wide (`agent_sandbox.preflight`); an agent's block replaces the global one, and an empty `commands` there turns it off for that agent. Timeouts left at zero fall back to the global block, then the defaults. + +```yaml +agent_sandbox: + preflight: + commands: + # A device smoke test: fails on a node whose Spyre card is unusable. + - python -c 'import torch; torch.ones(1).to("spyre")' + timeout_s: 120 # per command + total_timeout_s: 300 # all commands + retry_delay_s: 0 # wait before the retry on another node; 0 = immediately +``` + +A failing command ends the container in seconds with exit code 75 after printing `hive-node-preflight: FAILED command 1 of 1 exited 1: `, and hive reports the kick as `node_unhealthy`: + +- The command's output is in the kick transcript and the kick log, the run record carries `failReason: node_unhealthy` and the `node` the pod ran on (`spec.nodeName`), and the audit log names the node (`sandbox_node_unhealthy`). +- It is **not an agent failure.** The agent never ran, so it returns to `idle` rather than `failed`, nothing that watches failed agents (restart, escalation, a failure-fix loop) sees it, and the next cadence tries again. The dashboard shows `node unhealthy` with the node, distinct from `last run failed`. +- **One retry on another node.** Hive re-creates the Job once, after `retry_delay_s`, with a node-affinity `NotIn` on `kubernetes.io/hostname` for the failed node, added to every required term of the `affinity` above and never pinning a node (`sandbox_node_unhealthy_retry` in the audit log; the first attempt's output is kept at the top of the transcript). If the retry fails its preflight too, the kick fails as `node_unhealthy`. A cluster whose node names differ from the `kubernetes.io/hostname` label, or a custom `scheduler_name` that ignores node affinity, cannot honor the exclusion. + +The commands come from configuration only, never from a kick message, like `publish.verify`. They are part of the Job manifest and their output is in the pod log, so never put a credential on a command line; the script prints nothing from the environment. A preflight set for a runtime other than `job` is logged and ignored, and only agent kicks run it (the `publish.verify` Jobs do not). + +**`job.schedule_timeout_s`: fail fast when no node takes the pod.** While waiting on the Job, hive also reads its pod. If the pod stays `PodScheduled=False` with reason `Unschedulable` for `schedule_timeout_s` (default 600 s, in `job`, with the global value as the fallback for an agent's own `job` block), hive fails the kick as `waiting_for_node` with the scheduler's message (for example `0/38 nodes are available: ...`), deletes the Job and releases the agent, instead of holding its slot for the whole kick timeout plus the cluster-side deadline margin. There is no retry; the next cadence tries again. While it waits the dashboard shows `Job waiting for a node` with the scheduler's message, distinct from `Job running`, and the audit log records `sandbox_waiting_for_node`, then `sandbox_waiting_for_node_failed` if it gives up. If your cluster autoscaler needs longer than ten minutes to add an accelerator node, raise `schedule_timeout_s`; the kick timeout still bounds the wait. + +| Reason | Meaning | What to check | +|--------|---------|---------------| +| `node_unhealthy` | `preflight` failed on the node, and again on the retry node | The command's output in the kick log; the node named in the run record; cordon or fix the node | +| `waiting_for_node` | the pod stayed Unschedulable past `schedule_timeout_s` | The scheduler's message in the kick log: `node_selector`, `affinity`, `tolerations`, device `resources` and `scheduler_name` against the cluster's nodes; a retry excluding the only fitting node ends here | + **Workspace transport is a shared claim, not a copy.** The Job mounts `workspace_claim` with a `subPath` pointing at the per-kick workspace, at the same `/workspace` mount the Podman launcher uses. The sandbox `workspace_dir` must therefore live under `workspace_claim_mount` (it does by default: `/data/agents/sandbox`). When the Job is scheduled onto a different node than hive, which is the whole point of a node selector, the claim must be **ReadWriteMany**. Hive cannot check the access mode from inside its pod; a Job that fails to start with a multi-attach error is the symptom. **The credential boundary.** The Job receives exactly the Secrets named in `env_from_secrets`, resolved by Kubernetes from the pod template; hive never reads them. Environment variables from the allowlist are copied in with the same credential-name filter Podman applies. An allowlisted `ANTHROPIC_BASE_URL` that points at a pod-local proxy (such as `127.0.0.1:18444`) is not reachable from a Job pod, so operators must supply a routable endpoint or model credentials via `env_from_secrets`. Hive's brokered GitHub token **never enters the Job**, so a Job cannot push or open a PR itself; hive does both from the returned workspace, with the same scan and scoped-token minting as the Podman path. If a workload needs its own registry or model credentials, they go in a Secret named here, and they are that workload's to protect. @@ -467,7 +499,9 @@ A sandboxed agent (Podman or Job runtime) has no tmux session, no in-pod process | `health` | Meaning | Dot | |-----------|---------|-----| | `ok` | last run succeeded (or no run yet) and the next cadence occurrence is still in the future | green | -| `running` | a kick is in flight (container or Job running/pending) | amber | +| `running` | a kick is in flight (container or Job running) | amber | +| `waiting_for_node` | a kick is in flight but its Job's pod is Pending and Unschedulable; `waitDetail` has the scheduler's message | amber | +| `node_unhealthy` | the last kick never reached the agent: the node `preflight` failed, also on the retry; `failReason` and `node` say why and where | red | | `failed` | the last run failed or timed out — see the agent's log | red | | `overdue` | the cadence had a due occurrence that never started, older than the grace (half the period, clamped to 15 min – 2 h); `dueAt` names it | red | diff --git a/src/pkg/config/crux_aliases.go b/src/pkg/config/crux_aliases.go index e86b3f3c0f..5b6cd3254a 100644 --- a/src/pkg/config/crux_aliases.go +++ b/src/pkg/config/crux_aliases.go @@ -58,6 +58,7 @@ type ( type ( SandboxJobAffinity = cruxconfig.SandboxJobAffinity + SandboxPreflightConfig = cruxconfig.SandboxPreflightConfig SandboxJobNodeAffinity = cruxconfig.SandboxJobNodeAffinity SandboxJobNodeSelector = cruxconfig.SandboxJobNodeSelector SandboxJobNodeSelectorTerm = cruxconfig.SandboxJobNodeSelectorTerm diff --git a/src/pkg/config/sandbox_preflight_test.go b/src/pkg/config/sandbox_preflight_test.go new file mode 100644 index 0000000000..24e950bc22 --- /dev/null +++ b/src/pkg/config/sandbox_preflight_test.go @@ -0,0 +1,88 @@ +package config + +import ( + "os" + "path/filepath" + "reflect" + "strings" + "testing" +) + +func testPreflight() *SandboxPreflightConfig { + return &SandboxPreflightConfig{Commands: []string{`python -c 'import torch; torch.ones(1).to("spyre")'`}, TimeoutS: 60, TotalTimeoutS: 120, RetryDelayS: 3} +} + +func TestSandboxPreflightRejectedAtValidate(t *testing.T) { + cfg := &Config{AgentRuntimeConfig: AgentRuntimeConfig{AgentSandbox: AgentSandboxConfig{Preflight: &SandboxPreflightConfig{Commands: []string{" "}}}}} + if err := cfg.ValidateWithOptions(ValidateOptions{}); err == nil || !strings.Contains(err.Error(), "agent_sandbox.preflight.commands[0]") { + t.Fatalf("global empty command error = %v", err) + } + cfg.AgentSandbox = AgentSandboxConfig{Job: &SandboxJobConfig{ScheduleTimeoutS: -1}} + if err := cfg.ValidateWithOptions(ValidateOptions{}); err == nil || !strings.Contains(err.Error(), "agent_sandbox.job.schedule_timeout_s") { + t.Fatalf("global negative schedule timeout error = %v", err) + } + cfg.AgentSandbox = AgentSandboxConfig{} + cfg.Agents = map[string]AgentConfig{"quality": {Sandbox: &AgentSandboxOverride{Preflight: &SandboxPreflightConfig{Commands: []string{"true"}, TimeoutS: -1}}}} + if err := cfg.ValidateWithOptions(ValidateOptions{}); err == nil || !strings.Contains(err.Error(), "agents.quality.sandbox.preflight.timeout_s") { + t.Fatalf("agent negative timeout error = %v", err) + } + cfg.Agents = map[string]AgentConfig{"quality": {Sandbox: &AgentSandboxOverride{Job: &SandboxJobConfig{ScheduleTimeoutS: -1}}}} + if err := cfg.ValidateWithOptions(ValidateOptions{}); err == nil || !strings.Contains(err.Error(), "agents.quality.sandbox.job.schedule_timeout_s") { + t.Fatalf("agent negative schedule timeout error = %v", err) + } +} + +func TestSandboxPreflightRejectedOnLoad(t *testing.T) { + dir := t.TempDir() + source := filepath.Join(dir, "hive.yaml") + seed := "project:\n org: testorg\ngithub:\n token: ghp_test123456789\nagents:\n quality:\n backend: copilot\n enabled: true\n kick_template: quality-advisory.md\n mode: ADVISORY\n" + invalid := "agent_sandbox:\n preflight:\n commands: [\"\"]\n" + if err := os.WriteFile(source, []byte(seed+invalid), 0o600); err != nil { + t.Fatal(err) + } + if _, err := LoadWithOverrides(source, "-"); err == nil || !strings.Contains(err.Error(), "agent_sandbox.preflight.commands[0]") { + t.Fatalf("load error = %v", err) + } +} + +func TestSandboxPreflightSaveReload(t *testing.T) { + dir := t.TempDir() + source := filepath.Join(dir, "hive.yaml") + oldOverlay, oldRuntime := DashboardOverlayFile, RuntimeConfigFile + DashboardOverlayFile, RuntimeConfigFile = filepath.Join(dir, "dashboard"), filepath.Join(dir, "runtime") + t.Cleanup(func() { DashboardOverlayFile, RuntimeConfigFile = oldOverlay, oldRuntime }) + t.Setenv("KUBERNETES_SERVICE_HOST", "10.0.0.1") + seed := "project:\n org: testorg\ngithub:\n token: ghp_test123456789\nagents:\n quality:\n backend: copilot\n enabled: true\n kick_template: quality-advisory.md\n mode: ADVISORY\n" + if err := os.WriteFile(source, []byte(seed), 0o600); err != nil { + t.Fatal(err) + } + cfg, err := Load(source) + if err != nil { + t.Fatal(err) + } + cfg.AgentSandbox.Preflight = testPreflight() + cfg.AgentSandbox.Job = &SandboxJobConfig{ScheduleTimeoutS: 900} + a := cfg.Agents["quality"] + a.Sandbox = &AgentSandboxOverride{Preflight: &SandboxPreflightConfig{Commands: []string{"true"}, RetryDelayS: 1}, Job: &SandboxJobConfig{ScheduleTimeoutS: 30}} + cfg.Agents["quality"] = a + if err := cfg.Save(); err != nil { + t.Fatal(err) + } + got, err := LoadWithDashboardOverlay(source) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got.AgentSandbox.Preflight, testPreflight()) { + t.Fatalf("global preflight lost: %+v", got.AgentSandbox.Preflight) + } + if got.AgentSandbox.Job == nil || got.AgentSandbox.Job.ScheduleTimeoutS != 900 { + t.Fatalf("global schedule timeout lost: %+v", got.AgentSandbox.Job) + } + own := got.Agents["quality"].Sandbox + if own == nil || own.Preflight == nil || !reflect.DeepEqual(own.Preflight.Commands, []string{"true"}) || own.Preflight.RetryDelayS != 1 { + t.Fatalf("agent preflight lost: %+v", own) + } + if own.Job == nil || own.Job.ScheduleTimeoutS != 30 { + t.Fatalf("agent schedule timeout lost: %+v", own) + } +} diff --git a/src/pkg/config/validate.go b/src/pkg/config/validate.go index 10e1718e9f..d141eb37be 100644 --- a/src/pkg/config/validate.go +++ b/src/pkg/config/validate.go @@ -24,6 +24,9 @@ func (c *Config) ValidateWithOptions(opts ValidateOptions) error { if err := c.validateJobAffinities(); err != nil { return err } + if err := c.validateSandboxPreflights(); err != nil { + return err + } if err := c.IntegrationOrchestration.Validate(); err != nil { return err } diff --git a/src/pkg/config/validate_sandbox_preflight.go b/src/pkg/config/validate_sandbox_preflight.go new file mode 100644 index 0000000000..eb5ce79a16 --- /dev/null +++ b/src/pkg/config/validate_sandbox_preflight.go @@ -0,0 +1,38 @@ +package config + +import ( + "fmt" + "sort" + + cruxconfig "github.com/hivecommons/hive/crux/config" +) + +// validateSandboxPreflights checks sandbox.preflight and job.schedule_timeout_s +// in the global agent_sandbox block and in every agent's sandbox override +// (hivecommons/hive#18811), so a bad value fails config load rather than a kick. +func (c *Config) validateSandboxPreflights() error { + if err := cruxconfig.ValidateSandboxPreflight(c.AgentSandbox.Preflight); err != nil { + return fmt.Errorf("agent_sandbox.%w", err) + } + if err := cruxconfig.ValidateSandboxJobScheduleTimeout(c.AgentSandbox.Job); err != nil { + return fmt.Errorf("agent_sandbox.%w", err) + } + names := make([]string, 0, len(c.Agents)) + for name := range c.Agents { + names = append(names, name) + } + sort.Strings(names) + for _, name := range names { + sb := c.Agents[name].Sandbox + if sb == nil { + continue + } + if err := cruxconfig.ValidateSandboxPreflight(sb.Preflight); err != nil { + return fmt.Errorf("agents.%s.sandbox.%w", name, err) + } + if err := cruxconfig.ValidateSandboxJobScheduleTimeout(sb.Job); err != nil { + return fmt.Errorf("agents.%s.sandbox.%w", name, err) + } + } + return nil +} diff --git a/src/pkg/dashboard/meetings_hive_agents.go b/src/pkg/dashboard/meetings_hive_agents.go index 0dfc61037c..7e65af0d17 100644 --- a/src/pkg/dashboard/meetings_hive_agents.go +++ b/src/pkg/dashboard/meetings_hive_agents.go @@ -54,7 +54,7 @@ func hiveAgentClassOf(a FrontendAgent) hiveAgentClass { starting := a.Starting && a.State != hiveAgentStateRunning && !paused && !off sessionDown := a.State != hiveAgentStateRunning if a.Sandboxed && a.SandboxRun != nil && a.SandboxRun.Health != "" { - sessionDown = a.SandboxRun.Health == SandboxHealthFailed || a.SandboxRun.Health == SandboxHealthOverdue + sessionDown = a.SandboxRun.Health == SandboxHealthFailed || a.SandboxRun.Health == SandboxHealthNodeUnhealthy || a.SandboxRun.Health == SandboxHealthOverdue } down := sessionDown && !paused && !off && !a.OnDemand && !starting working := a.Busy == hiveAgentBusyWorking && !paused && !off && !down diff --git a/src/pkg/dashboard/sandbox_health.go b/src/pkg/dashboard/sandbox_health.go index b49fd23285..1317e20495 100644 --- a/src/pkg/dashboard/sandbox_health.go +++ b/src/pkg/dashboard/sandbox_health.go @@ -4,6 +4,7 @@ import ( "time" "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/crux/sandbox" "github.com/hivecommons/hive/pkg/config" ) @@ -22,6 +23,14 @@ const ( SandboxHealthRunning = "running" // SandboxHealthFailed: the last run failed or timed out. SandboxHealthFailed = "failed" + // SandboxHealthWaitingForNode: a kick is in flight but its Job's pod is + // Pending and Unschedulable (no node has taken it yet), distinct from + // running so a stuck placement is not mistaken for a working agent. + SandboxHealthWaitingForNode = "waiting_for_node" + // SandboxHealthNodeUnhealthy: the last kick never reached the agent + // because the node preflight failed (on the retry node too); the node is + // in FrontendSandboxRun.Node and the failing command in Error. + SandboxHealthNodeUnhealthy = "node_unhealthy" // SandboxHealthOverdue: a scheduled kick is due and the grace has passed // without a run starting — the cadence was missed. SandboxHealthOverdue = "overdue" @@ -70,6 +79,13 @@ type FrontendSandboxRun struct { // DueAt is the scheduled occurrence that has been missed when Health is // overdue; empty otherwise. DueAt string `json:"dueAt,omitempty"` + // FailReason is node_unhealthy or waiting_for_node when the last run + // failed for the infrastructure's reason, not the agent's; Node is the + // node the pod ran on, when known (#18811). + FailReason string `json:"failReason,omitempty"` + Node string `json:"node,omitempty"` + // WaitDetail is the scheduler's message while Health is waiting_for_node. + WaitDetail string `json:"waitDetail,omitempty"` } // sandboxOverdueGrace is how long after a due occurrence a missed cadence is @@ -122,13 +138,22 @@ func sandboxRunHealth(proc *crux.Process, cadence config.Cadence, onDemand bool, out.Summary = redactTokens(run.Summary) out.Tokens = run.Usage.Total() out.Model = run.Usage.Model + out.FailReason = run.FailReason + out.Node = run.Node } if proc.State == crux.StateRunning { out.Health = SandboxHealthRunning + if ph := proc.SandboxPhase; ph != nil && ph.Phase == sandbox.FailReasonWaitingForNode { + out.Health = SandboxHealthWaitingForNode + out.WaitDetail = redactTokens(ph.Detail) + } return out } if proc.SandboxLastRun != nil && !proc.SandboxLastRun.Succeeded { out.Health = SandboxHealthFailed + if proc.SandboxLastRun.FailReason == sandbox.FailReasonNodeUnhealthy { + out.Health = SandboxHealthNodeUnhealthy + } return out } if onDemand || proc.Paused || proc.State == crux.StatePaused || cadence == "" || cadence.IsPaused() || cadence.IsContinuous() { diff --git a/src/pkg/dashboard/sandbox_health_test.go b/src/pkg/dashboard/sandbox_health_test.go index 757cf55df6..5e1d716dae 100644 --- a/src/pkg/dashboard/sandbox_health_test.go +++ b/src/pkg/dashboard/sandbox_health_test.go @@ -118,3 +118,30 @@ func TestBuildAgents_SandboxedAgentCarriesLastRun(t *testing.T) { t.Errorf("non-sandboxed agent must not carry sandboxRun") } } + +// A Job whose pod cannot be scheduled and a node whose preflight failed are +// their own verdicts, distinct from running and from a failed agent (#18811). +func TestSandboxRunHealthInfrastructureStates(t *testing.T) { + now := time.Date(2026, 11, 17, 12, 0, 0, 0, time.UTC) + phase := &crux.SandboxPhase{Phase: "waiting_for_node", Detail: "0/38 nodes are available: 38 node(s) didn't match Pod's node affinity/selector.", Since: now} + waiting := sandboxRunHealth(&crux.Process{State: crux.StateRunning, SandboxPhase: phase}, "1h", false, now) + if waiting.Health != SandboxHealthWaitingForNode || !strings.Contains(waiting.WaitDetail, "0/38 nodes") { + t.Errorf("waiting: %+v", waiting) + } + if running := sandboxRunHealth(&crux.Process{State: crux.StateRunning}, "1h", false, now); running.Health != SandboxHealthRunning || running.WaitDetail != "" { + t.Errorf("a running kick with no phase must stay running: %+v", running) + } + bad := &crux.SandboxRun{Runtime: "job", StartedAt: now.Add(-time.Minute), EndedAt: now.Add(-50 * time.Second), Error: "sandbox: node_unhealthy on node p1-worker-93: smoke failed", FailReason: "node_unhealthy", Node: "p1-worker-93"} + got := sandboxRunHealth(&crux.Process{State: crux.StateIdle, SandboxLastRun: bad}, "1h", false, now) + if got.Health != SandboxHealthNodeUnhealthy || got.FailReason != "node_unhealthy" || got.Node != "p1-worker-93" || got.Succeeded { + t.Errorf("node unhealthy: %+v", got) + } + noNode := &crux.SandboxRun{Runtime: "job", StartedAt: now.Add(-time.Hour), EndedAt: now.Add(-50 * time.Minute), Error: "sandbox: waiting_for_node: ...", FailReason: "waiting_for_node"} + if got := sandboxRunHealth(&crux.Process{State: crux.StateIdle, SandboxLastRun: noNode}, "1h", false, now); got.Health != SandboxHealthFailed || got.FailReason != "waiting_for_node" { + t.Errorf("a kick that never got a node reads as a failed run carrying its reason: %+v", got) + } + a := FrontendAgent{Sandboxed: true, State: "idle", SandboxRun: &FrontendSandboxRun{Health: SandboxHealthNodeUnhealthy}} + if got := hiveAgentClassOf(a); got != hiveAgentDown { + t.Errorf("a node_unhealthy agent must read as down in the agent roster, got %q", got) + } +} diff --git a/src/pkg/dashboard/settings_parity_allowlist.txt b/src/pkg/dashboard/settings_parity_allowlist.txt index acf9474de7..b530de75ae 100644 --- a/src/pkg/dashboard/settings_parity_allowlist.txt +++ b/src/pkg/dashboard/settings_parity_allowlist.txt @@ -40,6 +40,11 @@ agent_sandbox.openshell.workspace_claim # TODO #17645 (OpenShell runtime; Settin agent_sandbox.openshell.workspace_claim_mount # TODO #17645 (OpenShell runtime; Settings control pending) agent_sandbox.publish.verify[] # TODO #14999 (policy-file knob; Settings control pending) agent_sandbox.publish.verify_timeout_s # TODO #14999 (policy-file knob; Settings control pending) +agent_sandbox.preflight.commands[] # TODO #18811 (policy-file knob; Settings control pending) +agent_sandbox.preflight.retry_delay_s # TODO #18811 (policy-file knob; Settings control pending) +agent_sandbox.preflight.timeout_s # TODO #18811 (policy-file knob; Settings control pending) +agent_sandbox.preflight.total_timeout_s # TODO #18811 (policy-file knob; Settings control pending) +agent_sandbox.job.schedule_timeout_s # TODO #18811 (policy-file knob; Settings control pending) agents..acmm_levels[] # TODO #11854 agents..agent_spec # TODO #11854 agents..aliases[] # TODO #11854 @@ -171,6 +176,11 @@ agents..sandbox.runtime # TODO #11854 agents..sandbox.timeout_s # TODO #11854 agents..sandbox.publish.verify[] # TODO #14999 (policy-file knob; Settings control pending) agents..sandbox.publish.verify_timeout_s # TODO #14999 (policy-file knob; Settings control pending) +agents..sandbox.preflight.commands[] # TODO #18811 (policy-file knob; Settings control pending) +agents..sandbox.preflight.retry_delay_s # TODO #18811 (policy-file knob; Settings control pending) +agents..sandbox.preflight.timeout_s # TODO #18811 (policy-file knob; Settings control pending) +agents..sandbox.preflight.total_timeout_s # TODO #18811 (policy-file knob; Settings control pending) +agents..sandbox.job.schedule_timeout_s # TODO #18811 (policy-file knob; Settings control pending) agents..cli_memory.auto_memory # TODO #11854 agents..cli_memory.clear_on_kick # TODO #11854 agents..cli_memory.drain_on_write # TODO #11854 diff --git a/src/pkg/dashboard/static-src/css/agent-fields.css b/src/pkg/dashboard/static-src/css/agent-fields.css index 56eda913ba..fb9b449882 100644 --- a/src/pkg/dashboard/static-src/css/agent-fields.css +++ b/src/pkg/dashboard/static-src/css/agent-fields.css @@ -4,6 +4,7 @@ .agent-blockers .ab-sep { opacity: 0.35; } .agent-blockers .ab-ok { color: var(--green); } .agent-blockers .ab-bad { color: var(--red); font-weight: 600; } + .agent-blockers .ab-warn { color: var(--yellow); font-weight: 600; } .oc-detail-blockers { margin-top: var(--sp-3); display: flex; flex-wrap: wrap; align-items: center; gap: var(--sp-3); } .never-scheduled-chip { font-size: var(--fs-2xs); font-weight: 700; text-transform: uppercase; letter-spacing: 0.4px; color: var(--yellow); background: none; border: 1px solid var(--yellow); border-radius: 3px; padding: 1px 5px; margin-left: var(--sp-3); cursor: pointer; vertical-align: middle; white-space: nowrap; } .never-scheduled-chip:hover { background: color-mix(in srgb, var(--yellow) 18%, transparent); } diff --git a/src/pkg/dashboard/static-src/js/agent-cards.js b/src/pkg/dashboard/static-src/js/agent-cards.js index 6caf9980ba..2ae99d0cd4 100644 --- a/src/pkg/dashboard/static-src/js/agent-cards.js +++ b/src/pkg/dashboard/static-src/js/agent-cards.js @@ -839,13 +839,14 @@ } function agentSessionDown(a) { const sr = agentSandboxRun(a); - if (sr) return sr.health === 'failed' || sr.health === 'overdue'; + if (sr) return sr.health === 'failed' || sr.health === 'node_unhealthy' || sr.health === 'overdue'; return a.state !== 'running'; } function agentSandboxDownLabel(a) { const sr = agentSandboxRun(a); if (!sr) return ''; - if (sr.health === 'failed') return sr.timedOut ? 'last run timed out' : 'last run failed'; + if (sr.health === 'node_unhealthy') return 'node unhealthy'; + if (sr.health === 'failed') return sr.failReason === 'waiting_for_node' ? 'no node available' : (sr.timedOut ? 'last run timed out' : 'last run failed'); if (sr.health === 'overdue') return 'run overdue'; return ''; } @@ -854,7 +855,7 @@ if (!sr || !sr.lastRunAt) return ''; const ago = agentNowAgo((hiveNowMs || Date.now()) - Date.parse(sr.lastRunAt)); const dur = sr.durationS >= 60 ? Math.round(sr.durationS / 60) + ' min' : (sr.durationS || 0) + ' s'; - let text = (sr.succeeded ? 'last run succeeded' : (sr.timedOut ? 'last run timed out' : 'last run failed')) + ' \u00b7 ' + ago + ' \u00b7 ' + dur; + let text = (sr.succeeded ? 'last run succeeded' : (sr.failReason === 'node_unhealthy' ? 'node unhealthy' : (sr.failReason === 'waiting_for_node' ? 'no node available' : (sr.timedOut ? 'last run timed out' : 'last run failed')))) + (sr.node ? ' on ' + sr.node : '') + ' \u00b7 ' + ago + ' \u00b7 ' + dur; if (sr.commitCount) text += ' \u00b7 ' + sr.commitCount + (sr.commitCount === 1 ? ' commit' : ' commits'); return text; } @@ -885,6 +886,8 @@ } else if (sandboxRun) { const rt = sandboxRun.runtime === 'job' ? 'Job' : 'sandbox'; if (sandboxRun.health === 'running') { session = rt + ' running'; sessionCls = 'ab-ok'; sessionTip = 'A sandboxed kick is in flight.'; } + else if (sandboxRun.health === 'waiting_for_node') { session = rt + ' waiting for a node'; sessionCls = 'ab-warn'; sessionTip = 'The Job is created but its pod is Pending and Unschedulable' + (sandboxRun.waitDetail ? ': ' + sandboxRun.waitDetail : '.'); } + else if (sandboxRun.health === 'node_unhealthy') { session = 'node unhealthy'; sessionCls = 'ab-bad'; sessionTip = 'The node preflight failed' + (sandboxRun.node ? ' on ' + sandboxRun.node : '') + ', also on the retry: ' + (sandboxRun.error || 'see the full log.'); } else if (sandboxRun.health === 'failed') { session = agentSandboxDownLabel(a); sessionCls = 'ab-bad'; sessionTip = 'The last sandboxed run failed: ' + (sandboxRun.error || 'see the full log.'); } else if (sandboxRun.health === 'overdue') { session = 'run overdue'; sessionCls = 'ab-bad'; sessionTip = 'A scheduled kick was due' + (sandboxRun.dueAt ? ' at ' + sandboxRun.dueAt : '') + ' and no run has started.'; } else { session = sandboxRun.lastRunAt ? rt + ' ok' : rt + ' ready'; sessionCls = 'ab-ok'; sessionTip = 'Sandboxed agent: each kick runs in its own container, so there is no session between kicks. ' + (agentSandboxRunText(a) || 'No run yet.'); } diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html index 32ff65a3f9..a5f6865111 100644 --- a/src/pkg/dashboard/static/index.html +++ b/src/pkg/dashboard/static/index.html @@ -2705,6 +2705,7 @@ .agent-blockers .ab-sep { opacity: 0.35; } .agent-blockers .ab-ok { color: var(--green); } .agent-blockers .ab-bad { color: var(--red); font-weight: 600; } + .agent-blockers .ab-warn { color: var(--yellow); font-weight: 600; } .oc-detail-blockers { margin-top: var(--sp-3); display: flex; flex-wrap: wrap; align-items: center; gap: var(--sp-3); } .never-scheduled-chip { font-size: var(--fs-2xs); font-weight: 700; text-transform: uppercase; letter-spacing: 0.4px; color: var(--yellow); background: none; border: 1px solid var(--yellow); border-radius: 3px; padding: 1px 5px; margin-left: var(--sp-3); cursor: pointer; vertical-align: middle; white-space: nowrap; } .never-scheduled-chip:hover { background: color-mix(in srgb, var(--yellow) 18%, transparent); } @@ -32403,13 +32404,14 @@

All labels

} function agentSessionDown(a) { const sr = agentSandboxRun(a); - if (sr) return sr.health === 'failed' || sr.health === 'overdue'; + if (sr) return sr.health === 'failed' || sr.health === 'node_unhealthy' || sr.health === 'overdue'; return a.state !== 'running'; } function agentSandboxDownLabel(a) { const sr = agentSandboxRun(a); if (!sr) return ''; - if (sr.health === 'failed') return sr.timedOut ? 'last run timed out' : 'last run failed'; + if (sr.health === 'node_unhealthy') return 'node unhealthy'; + if (sr.health === 'failed') return sr.failReason === 'waiting_for_node' ? 'no node available' : (sr.timedOut ? 'last run timed out' : 'last run failed'); if (sr.health === 'overdue') return 'run overdue'; return ''; } @@ -32418,7 +32420,7 @@

All labels

if (!sr || !sr.lastRunAt) return ''; const ago = agentNowAgo((hiveNowMs || Date.now()) - Date.parse(sr.lastRunAt)); const dur = sr.durationS >= 60 ? Math.round(sr.durationS / 60) + ' min' : (sr.durationS || 0) + ' s'; - let text = (sr.succeeded ? 'last run succeeded' : (sr.timedOut ? 'last run timed out' : 'last run failed')) + ' \u00b7 ' + ago + ' \u00b7 ' + dur; + let text = (sr.succeeded ? 'last run succeeded' : (sr.failReason === 'node_unhealthy' ? 'node unhealthy' : (sr.failReason === 'waiting_for_node' ? 'no node available' : (sr.timedOut ? 'last run timed out' : 'last run failed')))) + (sr.node ? ' on ' + sr.node : '') + ' \u00b7 ' + ago + ' \u00b7 ' + dur; if (sr.commitCount) text += ' \u00b7 ' + sr.commitCount + (sr.commitCount === 1 ? ' commit' : ' commits'); return text; } @@ -32449,6 +32451,8 @@

All labels

} else if (sandboxRun) { const rt = sandboxRun.runtime === 'job' ? 'Job' : 'sandbox'; if (sandboxRun.health === 'running') { session = rt + ' running'; sessionCls = 'ab-ok'; sessionTip = 'A sandboxed kick is in flight.'; } + else if (sandboxRun.health === 'waiting_for_node') { session = rt + ' waiting for a node'; sessionCls = 'ab-warn'; sessionTip = 'The Job is created but its pod is Pending and Unschedulable' + (sandboxRun.waitDetail ? ': ' + sandboxRun.waitDetail : '.'); } + else if (sandboxRun.health === 'node_unhealthy') { session = 'node unhealthy'; sessionCls = 'ab-bad'; sessionTip = 'The node preflight failed' + (sandboxRun.node ? ' on ' + sandboxRun.node : '') + ', also on the retry: ' + (sandboxRun.error || 'see the full log.'); } else if (sandboxRun.health === 'failed') { session = agentSandboxDownLabel(a); sessionCls = 'ab-bad'; sessionTip = 'The last sandboxed run failed: ' + (sandboxRun.error || 'see the full log.'); } else if (sandboxRun.health === 'overdue') { session = 'run overdue'; sessionCls = 'ab-bad'; sessionTip = 'A scheduled kick was due' + (sandboxRun.dueAt ? ' at ' + sandboxRun.dueAt : '') + ' and no run has started.'; } else { session = sandboxRun.lastRunAt ? rt + ' ok' : rt + ' ready'; sessionCls = 'ab-ok'; sessionTip = 'Sandboxed agent: each kick runs in its own container, so there is no session between kicks. ' + (agentSandboxRunText(a) || 'No run yet.'); } From 662be0e032e0cee671526e203e329343edd67b1a Mon Sep 17 00:00:00 2001 From: Andrew Anderson Date: Tue, 17 Nov 2026 23:35:22 -0500 Subject: [PATCH 2/2] =?UTF-8?q?=F0=9F=8C=B1=20test:=20poll=20sandbox=20pre?= =?UTF-8?q?flight=20kicks=20on=20a=20ticker,=20not=20time.Sleep?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The sleep ratchet counts time.Sleep calls in tests. Signed-off-by: Andrew Anderson --- src/crux/agent/sandbox_job_preflight_test.go | 32 ++++++++++++-------- 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/src/crux/agent/sandbox_job_preflight_test.go b/src/crux/agent/sandbox_job_preflight_test.go index ff6c845246..1e9b612657 100644 --- a/src/crux/agent/sandbox_job_preflight_test.go +++ b/src/crux/agent/sandbox_job_preflight_test.go @@ -65,17 +65,28 @@ func infraManager(t *testing.T, launcher sandbox.Launcher) (*Manager, *auditLog) return m, audits } -func waitLastRun(t *testing.T, m *Manager) *crux.Process { +// pollUntil re-checks cond on a ticker until it holds or the deadline passes. +func pollUntil(t *testing.T, what string, cond func() bool) { t.Helper() - deadline := time.Now().Add(3 * time.Second) - for time.Now().Before(deadline) { - if p := m.AllStatuses()["deps"]; p.SandboxLastRun != nil && p.State != StateRunning { - return p + tick := time.NewTicker(10 * time.Millisecond) + defer tick.Stop() + deadline := time.After(3 * time.Second) + for !cond() { + select { + case <-deadline: + t.Fatalf("timed out waiting for %s", what) + case <-tick.C: } - time.Sleep(10 * time.Millisecond) } - t.Fatalf("the kick never finished: %+v", m.AllStatuses()["deps"]) - return nil +} + +func waitLastRun(t *testing.T, m *Manager) *crux.Process { + t.Helper() + pollUntil(t, "the kick to finish", func() bool { + p := m.AllStatuses()["deps"] + return p.SandboxLastRun != nil && p.State != StateRunning + }) + return m.AllStatuses()["deps"] } func TestNodeUnhealthyKickIsNotAnAgentFailure(t *testing.T) { @@ -130,10 +141,7 @@ func TestOrdinaryJobFailureStillFailsTheAgent(t *testing.T) { if err := m.SendKick("deps", "bump"); err != nil { t.Fatalf("SendKick: %v", err) } - deadline := time.Now().Add(3 * time.Second) - for time.Now().Before(deadline) && m.AllStatuses()["deps"].State != StateFailed { - time.Sleep(10 * time.Millisecond) - } + pollUntil(t, "the agent to fail", func() bool { return m.AllStatuses()["deps"].State == StateFailed }) p := m.AllStatuses()["deps"] if p.State != StateFailed || p.SandboxLastRun == nil || p.SandboxLastRun.FailReason != "" { t.Fatalf("state=%s run=%+v, want failed with no infrastructure reason", p.State, p.SandboxLastRun)