Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- Add `sandbox.preflight`, a configurable node health check (for example a one-line device smoke test) that runs in a Job-runtime sandbox kick before the agent starts and, when it fails, ends the kick in seconds as `node_unhealthy` with the node recorded and retries once on a different node via a hostname `NotIn` affinity, without counting as an agent failure; and add `job.schedule_timeout_s` (default 10 minutes), which fails a kick whose Job pod stays Unschedulable as `waiting_for_node` with the scheduler's message, deletes the Job and releases the agent instead of waiting out the 45 minute timeout; the dashboard shows "waiting for a node" and "node unhealthy" as their own states. Preflight is off unless configured (#18811).
1 change: 1 addition & 0 deletions src/crux/agent/manager_pane_signals.go
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,7 @@ func (a *AgentProcess) snapshot() crux.Process {
LastError: a.LastError,
KickOutcome: a.KickOutcome,
SandboxLastRun: cloneSandboxRun(a.SandboxLastRun),
SandboxPhase: sandboxPhaseOf(a),
NeedsLogin: needsLogin,
LoginURL: loginURL,
Starting: a.startupLaunchQueued || a.launching,
Expand Down
9 changes: 9 additions & 0 deletions src/crux/agent/manager_sandbox.go
Original file line number Diff line number Diff line change
Expand Up @@ -174,6 +174,7 @@ func (m *Manager) startSandboxKickLocked(agent *AgentProcess, message string) er
case config.SandboxRuntimeSubstrate:
launcher = m.substrateLauncherLocked(agent.Config.SandboxSubstrate(m.sandboxConfig))
}
m.prepareJobKickLocked(launcher, agent, runtime)
repo := m.project.PrimaryRepo()
now := time.Now()
agent.State = StateRunning
Expand Down Expand Up @@ -338,6 +339,7 @@ func (m *Manager) runSandboxKick(ctx context.Context, name, runtime string, spec
}
agent.cancel = nil
agent.sandboxKickActive = false
clearSandboxPhase(agent)
// The run record is what a sandboxed agent's health and activity derive
// from: there is no session to probe between kicks. Recorded on every
// outcome but an operator cancel, and persisted so a pod roll does not
Expand Down Expand Up @@ -375,6 +377,9 @@ func (m *Manager) runSandboxKick(ctx context.Context, name, runtime string, spec
m.auditSandbox(name, "sandbox_cancelled", "sandbox kick cancelled while resuming")
return
}
if m.sandboxInfraFailureLocked(agent, name, res, err) {
return
}
if agent.Paused || agent.State == StatePaused {
agent.State = StatePaused
} else if agent.State != StateStopped {
Expand Down Expand Up @@ -411,6 +416,9 @@ func (m *Manager) runSandboxKick(ctx context.Context, name, runtime string, spec
agent.OutputBuffer.Write(msg)
}
detail := fmt.Sprintf("workspace=%s commits=%d branch=%s", res.Workspace, res.CommitCount, res.Branch)
if res.Sandbox.Node != "" {
detail += " node=" + res.Sandbox.Node
}
if res.PR != nil && res.PR.URL != "" {
detail += " pr=" + res.PR.URL
}
Expand Down Expand Up @@ -537,6 +545,7 @@ func JobOptionsFromConfig(job config.SandboxJobConfig) kubejob.Options {
EnvFromSecrets: append([]string(nil), job.EnvFromSecrets...),
ImagePullSecrets: append([]string(nil), job.ImagePullSecrets...),
TTLSeconds: job.TTLSeconds,
ScheduleTimeout: time.Duration(job.ScheduleTimeoutS) * time.Second,
RelayImage: job.RelayImage,
RelayRunAsUser: job.RelayRunAsUser,
Resources: kubejob.Resources{
Expand Down
122 changes: 122 additions & 0 deletions src/crux/agent/manager_sandbox_job.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,122 @@
package agent

import (
"fmt"
"sync"
"time"

"github.com/hivecommons/hive/crux"
"github.com/hivecommons/hive/crux/config"
"github.com/hivecommons/hive/crux/kubejob"
"github.com/hivecommons/hive/crux/sandbox"
)

// Audit actions for the Job runtime's infrastructure failures
// (hivecommons/hive#18811). Each detail names the node when it is known.
const (
auditSandboxNodeUnhealthy = "sandbox_node_unhealthy"
auditSandboxNodeRetry = "sandbox_node_unhealthy_retry"
auditSandboxWaitingForNode = "sandbox_waiting_for_node"
auditSandboxWaitingForNodeFail = "sandbox_waiting_for_node_failed"
)

// sandboxPhases holds the live phase of each in-flight sandboxed kick, keyed by
// the agent. It lives beside AgentProcess rather than on it because manager.go
// is at its crux file-size ratchet; every read and write is under m.mu anyway,
// the sync.Map only keeps the package-level table race-free across Managers.
var sandboxPhases sync.Map

func setSandboxPhase(a *AgentProcess, p crux.SandboxPhase) { sandboxPhases.Store(a, p) }

func clearSandboxPhase(a *AgentProcess) { sandboxPhases.Delete(a) }

// sandboxPhaseOf returns a copy of the agent's live phase, or nil.
func sandboxPhaseOf(a *AgentProcess) *crux.SandboxPhase {
v, ok := sandboxPhases.Load(a)
if !ok {
return nil
}
p, ok := v.(crux.SandboxPhase)
if !ok {
return nil
}
return &p
}

// prepareJobKickLocked wires sandbox.preflight and the scheduling events into
// the launcher of a job-runtime kick. A test factory's launcher is left alone.
// A preflight configured for another runtime is reported and ignored: it only
// has a meaning on the Job runtime, where it runs in the Job's own container.
func (m *Manager) prepareJobKickLocked(launcher sandbox.Launcher, agent *AgentProcess, runtime string) {
preflight := agent.Config.SandboxPreflight(m.sandboxConfig)
if runtime != config.SandboxRuntimeJob {
if len(preflight.Commands) > 0 {
m.logger.Warn("sandbox.preflight is set but the runtime is not job; the preflight is ignored", "name", agent.Name, "runtime", runtime)
}
return
}
kl, ok := launcher.(*kubejob.Launcher)
if !ok {
return
}
kl.Options.Preflight = preflight.Commands
kl.Options.PreflightTimeout = time.Duration(preflight.TimeoutS) * time.Second
kl.Options.PreflightTotalTimeout = time.Duration(preflight.TotalTimeoutS) * time.Second
kl.Options.PreflightRetryDelay = time.Duration(preflight.RetryDelayS) * time.Second
name := agent.Name
kl.Options.OnEvent = func(e kubejob.Event) { m.onJobEvent(name, e) }
}

// onJobEvent records what the launcher reports while a Job is placed: the
// pod waiting for a node (shown on the dashboard until it lands) and a
// preflight failure that is being retried on another node.
func (m *Manager) onJobEvent(name string, e kubejob.Event) {
m.mu.Lock()
defer m.mu.Unlock()
agent, ok := m.agents[name]
if !ok {
return
}
switch e.Kind {
case kubejob.EventWaitingForNode:
setSandboxPhase(agent, crux.SandboxPhase{Phase: sandbox.FailReasonWaitingForNode, Detail: e.Detail, Since: time.Now()})
if agent.OutputBuffer != nil {
agent.OutputBuffer.Write("sandbox Job waiting for a node: " + e.Detail)
}
m.auditSandbox(name, auditSandboxWaitingForNode, "reason="+e.Detail)
case kubejob.EventScheduled:
clearSandboxPhase(agent)
case kubejob.EventNodeRetry:
if agent.OutputBuffer != nil {
agent.OutputBuffer.Write("sandbox node preflight failed on " + e.Node + "; retrying on another node")
}
m.auditSandbox(name, auditSandboxNodeRetry, fmt.Sprintf("node=%s reason=%s", e.Node, e.Detail))
}
}

// sandboxInfraFailureLocked handles a kick that failed for the infrastructure's
// reason (node_unhealthy, waiting_for_node): the agent never ran, so it goes
// back to Idle with the reason, not to Failed, and nothing that watches Failed
// agents (restart, escalation, a failure-fix loop) sees it. The next cadence
// tries again. It reports whether it handled the failure.
func (m *Manager) sandboxInfraFailureLocked(agent *AgentProcess, name string, res SandboxKickResult, err error) bool {
reason := res.Sandbox.FailReason
if reason == "" {
return false
}
if agent.Paused || agent.State == StatePaused {
agent.State = StatePaused
} else if agent.State != StateStopped {
agent.State = StateIdle
}
agent.LastError = err.Error()
if agent.OutputBuffer != nil {
agent.OutputBuffer.Write("sandbox kick not run: " + err.Error())
}
action := auditSandboxNodeUnhealthy
if reason == sandbox.FailReasonWaitingForNode {
action = auditSandboxWaitingForNodeFail
}
m.auditSandbox(name, action, fmt.Sprintf("node=%s reason=%s workspace=%s", res.Sandbox.Node, err.Error(), res.Workspace))
return true
}
Loading
Loading