From c6b9faba25874c5f6ccd3adb6884c9a31b7bad5b Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Mon, 16 Nov 2026 19:55:26 -0500 Subject: [PATCH 1/2] =?UTF-8?q?=F0=9F=90=9B=20Sandbox=20toggle=20writes=20?= =?UTF-8?q?a=20complete=20block;=20reap=20orphaned=20sandbox=20Jobs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The per-agent Run in sandbox toggle now resolves the agent's effective sandbox block (own fields, then agent_sandbox defaults) and refuses with 400 when no image or usable runtime resolves. copy_from copies another agent's complete block minus publish.verify. The General tab shows a source picker with an inline error when there is none. Toggling off keeps the block with enabled: false. Switching an agent into sandbox mode now stops its tmux session, and the running-state kick refusal names the real state. Every sandbox Job gets activeDeadlineSeconds (kick timeout plus a named margin), and hive sweeps orphaned hive-kubejob sandbox Jobs at boot, logging and auditing each. Fixes #18525 Signed-off-by: Andy Anderson --- ...fixed-18525-sandbox-toggle-and-job-reap.md | 1 + src/cmd/hive/main.go | 3 + src/cmd/hive/sandbox_job_sweep.go | 129 +++++++++ src/cmd/hive/sandbox_job_sweep_test.go | 150 +++++++++++ src/crux/agent/manager.go | 13 + src/crux/agent/manager_sandbox.go | 31 ++- src/crux/agent/manager_sandbox_mode.go | 59 +++++ src/crux/agent/manager_sandbox_mode_test.go | 170 ++++++++++++ src/crux/agent/sandbox_executor.go | 2 +- src/crux/agent/sandbox_job_runtime_test.go | 19 +- src/crux/config/agent_sandbox_block.go | 86 ++++++ src/crux/config/agent_sandbox_block_test.go | 90 +++++++ src/crux/kubejob/kubejob.go | 57 +++- src/crux/kubejob/sweep.go | 156 +++++++++++ src/crux/kubejob/sweep_test.go | 247 ++++++++++++++++++ .../dashboard-sections/workflow-and-agents.md | 2 + src/docs/sandbox-isolation.md | 15 ++ src/pkg/config/crux_aliases.go | 2 + src/pkg/dashboard/api_agent_config.go | 16 +- src/pkg/dashboard/api_agent_sandbox_toggle.go | 101 +++++++ .../api_agent_sandbox_toggle_test.go | 148 +++++++++++ .../dashboard/api_governor_security_test.go | 1 + .../dashboard/static-src/js/config-agent.js | 37 ++- .../dashboard/static-src/js/config-helpers.js | 1 + src/pkg/dashboard/static/index.html | 38 ++- 25 files changed, 1560 insertions(+), 14 deletions(-) create mode 100644 changelog.d/fixed-18525-sandbox-toggle-and-job-reap.md create mode 100644 src/cmd/hive/sandbox_job_sweep.go create mode 100644 src/cmd/hive/sandbox_job_sweep_test.go create mode 100644 src/crux/agent/manager_sandbox_mode.go create mode 100644 src/crux/agent/manager_sandbox_mode_test.go create mode 100644 src/crux/config/agent_sandbox_block.go create mode 100644 src/crux/config/agent_sandbox_block_test.go create mode 100644 src/crux/kubejob/sweep.go create mode 100644 src/crux/kubejob/sweep_test.go create mode 100644 src/pkg/dashboard/api_agent_sandbox_toggle.go create mode 100644 src/pkg/dashboard/api_agent_sandbox_toggle_test.go diff --git a/changelog.d/fixed-18525-sandbox-toggle-and-job-reap.md b/changelog.d/fixed-18525-sandbox-toggle-and-job-reap.md new file mode 100644 index 0000000000..cc3025a653 --- /dev/null +++ b/changelog.d/fixed-18525-sandbox-toggle-and-job-reap.md @@ -0,0 +1 @@ +- The dashboard's per-agent **Run in sandbox** toggle no longer wedges an agent ([#18525](https://github.com/hivecommons/hive/issues/18525)). Turning it on used to write `sandbox.enabled: true` alone, so with no `agent_sandbox` image every kick failed with "sandbox image is not configured". It now writes a complete block, prefilled from the `agent_sandbox` defaults or copied (`copy_from`) from another agent that has one, minus that agent's `publish.verify`, and refuses with a 400 that names what is missing when neither exists. The General tab shows a picker for the source when the toggle is turned on, with an inline error when there is none. Turning it off keeps the block and sets `enabled: false`. Switching an agent into sandbox mode now stops its tmux session, and a refused kick names the real state ("agent has a live interactive session; sandbox kicks are blocked until it stops") instead of "sandbox execution already running". Every sandbox Kubernetes Job now carries `activeDeadlineSeconds` (the kick timeout plus a 15-minute margin), and at boot hive reaps unfinished `app.kubernetes.io/managed-by=hive-kubejob` sandbox Jobs that no live kick owns, logging and auditing each one (`sandbox_job_reaped`). Before this, one Job sat Pending for 11 days. diff --git a/src/cmd/hive/main.go b/src/cmd/hive/main.go index 3cb03c5ba8..05021627c5 100644 --- a/src/cmd/hive/main.go +++ b/src/cmd/hive/main.go @@ -2662,6 +2662,9 @@ func (b *boot) bootDashboardWith(deps bootDashboardDeps) { } } }) + // Reap sandbox Jobs the previous process lost track of, now that the + // audit log can record each one (hivecommons/hive#18525). + startSandboxJobBootSweep(b.ctx, b.cfg, b.agentMgr, b.dashSrv.AuditLog, b.logger) // Persist per-user dashboard sessions on the PVC (/data) so direct-route // users aren't logged out by pod restarts. NOTE: use /data explicitly, NOT diff --git a/src/cmd/hive/sandbox_job_sweep.go b/src/cmd/hive/sandbox_job_sweep.go new file mode 100644 index 0000000000..1006bcddfc --- /dev/null +++ b/src/cmd/hive/sandbox_job_sweep.go @@ -0,0 +1,129 @@ +package main + +import ( + "context" + "fmt" + "log/slog" + "strings" + "time" + + "github.com/hivecommons/hive/crux" + cruxconfig "github.com/hivecommons/hive/crux/config" + "github.com/hivecommons/hive/crux/kubejob" + "github.com/hivecommons/hive/crux/sandbox" + "github.com/hivecommons/hive/pkg/config" +) + +// sandboxJobSweepTimeout bounds the boot sweep's API calls as a whole. +const sandboxJobSweepTimeout = 2 * time.Minute + +// sandboxJobSweepAuditAction is the audit-log action of one reaped Job. +const sandboxJobSweepAuditAction = "sandbox_job_reaped" + +// sandboxJobSweepAuditUser is who the audit log names for a reap: hive +// itself, at boot, not an operator. +const sandboxJobSweepAuditUser = "hive" + +// sandboxJobSweeper is the part of kubejob.Launcher the boot sweep uses. +type sandboxJobSweeper interface { + SweepOrphans(ctx context.Context, opts kubejob.SweepOptions) ([]kubejob.SweptJob, error) +} + +// newSandboxJobSweeper builds the boot sweep's sweeper: the in-cluster Job +// launcher. Tests replace it so a test run on a cluster-hosted CI runner can +// never list or delete real Jobs. +var newSandboxJobSweeper = func() sandboxJobSweeper { return &kubejob.Launcher{} } + +// sandboxJobRuntimeConfigured reports whether any agent (or the global +// default) selects the Job sandbox runtime, enabled or not: a Job orphaned +// before an agent was switched off still needs reaping. +func sandboxJobRuntimeConfigured(cfg *config.Config) bool { + if strings.TrimSpace(cfg.AgentSandbox.Runtime) == cruxconfig.SandboxRuntimeJob { + return true + } + for _, a := range cfg.Agents { + if a.SandboxRuntime(cfg.AgentSandbox) == cruxconfig.SandboxRuntimeJob { + return true + } + } + return false +} + +// sandboxJobSweepMaxAge is the longest kick timeout any Job-runtime agent +// has: a Job younger than that may still be a kick that has not timed out. +func sandboxJobSweepMaxAge(cfg *config.Config) time.Duration { + maxAge := kubejob.DefaultKickTimeout + for _, a := range cfg.Agents { + if a.SandboxRuntime(cfg.AgentSandbox) != cruxconfig.SandboxRuntimeJob { + continue + } + if t := time.Duration(a.SandboxTimeoutS(cfg.AgentSandbox)) * time.Second; t > maxAge { + maxAge = t + } + } + return maxAge +} + +// sandboxJobLive reports whether a Job's agent label belongs to an agent the +// runtime says is mid-kick. It errs towards live: the label of every Job of +// a running agent's slug counts, whichever launch (kick or verify) made it. +func sandboxJobLive(agentNames []string, agentMgr crux.Runtime) func(string) bool { + return func(label string) bool { + if agentMgr == nil { + return false + } + for _, name := range agentNames { + if !strings.HasPrefix(label, "hive-"+sandbox.AgentSlug(name)+"-") { + continue + } + if p, err := agentMgr.GetStatusFast(name); err == nil && p != nil && p.State == crux.StateRunning { + return true + } + } + return false + } +} + +// startSandboxJobBootSweep runs the boot sweep in the background when any +// agent selects the Job runtime. The config is read here, synchronously, so +// the goroutine never touches the live config a reload may be rewriting. +func startSandboxJobBootSweep(ctx context.Context, cfg *config.Config, agentMgr crux.Runtime, audit func(user, action, detail, agent string), logger *slog.Logger) bool { + if cfg == nil || !sandboxJobRuntimeConfigured(cfg) { + return false + } + maxAge := sandboxJobSweepMaxAge(cfg) + names := make([]string, 0, len(cfg.Agents)) + for name := range cfg.Agents { + names = append(names, name) + } + go runSandboxJobBootSweep(ctx, maxAge, names, agentMgr, newSandboxJobSweeper(), audit, logger) + return true +} + +// runSandboxJobBootSweep reaps sandbox Jobs the previous hive process lost +// track of (hivecommons/hive#18525). A kick's timeout lives in memory, so a +// restart mid-kick leaves its Job with no owner; one sat Pending for 11 +// days. Each reaped Job is logged and audited. Jobs without the +// app.kubernetes.io/managed-by=hive-kubejob label are never touched. +func runSandboxJobBootSweep(ctx context.Context, maxAge time.Duration, agentNames []string, agentMgr crux.Runtime, sweeper sandboxJobSweeper, audit func(user, action, detail, agent string), logger *slog.Logger) int { + if sweeper == nil { + sweeper = newSandboxJobSweeper() + } + ctx, cancel := context.WithTimeout(ctx, sandboxJobSweepTimeout) + defer cancel() + swept, err := sweeper.SweepOrphans(ctx, kubejob.SweepOptions{MaxAge: maxAge, Live: sandboxJobLive(agentNames, agentMgr)}) + for _, j := range swept { + logger.Warn("audit: reaped orphaned sandbox job", + "job", j.Name, "agent_label", j.Agent, "age", j.Age.Round(time.Second).String(), "reason", j.Reason) + if audit != nil { + audit(sandboxJobSweepAuditUser, sandboxJobSweepAuditAction, + fmt.Sprintf("job=%s age=%s reason=%s", j.Name, j.Age.Round(time.Second), j.Reason), j.Agent) + } + } + if err != nil { + logger.Warn("sandbox job boot sweep incomplete", "error", err, "reaped", len(swept)) + } else { + logger.Info("sandbox job boot sweep done", "reaped", len(swept), "max_age", maxAge.String()) + } + return len(swept) +} diff --git a/src/cmd/hive/sandbox_job_sweep_test.go b/src/cmd/hive/sandbox_job_sweep_test.go new file mode 100644 index 0000000000..876b5c0c0d --- /dev/null +++ b/src/cmd/hive/sandbox_job_sweep_test.go @@ -0,0 +1,150 @@ +package main + +import ( + "bytes" + "context" + "errors" + "io" + "log/slog" + "strings" + "sync" + "testing" + "time" + + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/crux/cruxtest" + "github.com/hivecommons/hive/crux/kubejob" + "github.com/hivecommons/hive/pkg/config" +) + +type fakeJobSweeper struct { + opts kubejob.SweepOptions + calls int + swept []kubejob.SweptJob + err error +} + +func (f *fakeJobSweeper) SweepOrphans(_ context.Context, opts kubejob.SweepOptions) ([]kubejob.SweptJob, error) { + f.calls++ + f.opts = opts + return f.swept, f.err +} + +func jobSweepConfig() *config.Config { + on := true + return &config.Config{AgentRuntimeConfig: config.AgentRuntimeConfig{ + AgentSandbox: config.AgentSandboxConfig{Enabled: true, Image: "img"}, + Agents: map[string]config.AgentConfig{ + "reviewer": {Backend: "claude", Sandbox: &config.AgentSandboxOverride{Enabled: &on, Runtime: "job", TimeoutS: 7200}}, + "scanner": {Backend: "claude"}, + }, + }} +} + +func TestSandboxJobBootSweepAuditsEachReap(t *testing.T) { + var logBuf bytes.Buffer + logger := slog.New(slog.NewTextHandler(&logBuf, nil)) + sw := &fakeJobSweeper{swept: []kubejob.SweptJob{{Name: "hive-kick-reviewer-1", Agent: "hive-reviewer-sandbox", Age: 11 * 24 * time.Hour, Reason: kubejob.SweepReasonPendingPastDeadline}}} + var audits []string + audit := func(user, action, detail, agent string) { + audits = append(audits, user+"|"+action+"|"+agent+"|"+detail) + } + cfg := jobSweepConfig() + n := runSandboxJobBootSweep(context.Background(), sandboxJobSweepMaxAge(cfg), []string{"reviewer", "scanner"}, nil, sw, audit, logger) + if n != 1 || sw.calls != 1 { + t.Fatalf("reaped = %d, calls = %d", n, sw.calls) + } + if sw.opts.MaxAge != 2*time.Hour { + t.Fatalf("MaxAge = %s, want the longest Job-runtime kick timeout", sw.opts.MaxAge) + } + if len(audits) != 1 || !strings.Contains(audits[0], "hive|sandbox_job_reaped|hive-reviewer-sandbox|job=hive-kick-reviewer-1") { + t.Fatalf("audits = %v", audits) + } + if !strings.Contains(logBuf.String(), "reaped orphaned sandbox job") { + t.Fatalf("reap not logged: %s", logBuf.String()) + } + if sw.opts.Live == nil || sw.opts.Live("hive-reviewer-sandbox") { + t.Fatal("with no runtime, no Job is live") + } +} + +func TestSandboxJobBootSweepStartsOnlyWithJobRuntime(t *testing.T) { + quiet := slog.New(slog.NewTextHandler(io.Discard, nil)) + cfg := &config.Config{} + cfg.Agents = map[string]config.AgentConfig{"scanner": {Backend: "claude"}} + if startSandboxJobBootSweep(context.Background(), cfg, nil, nil, quiet) { + t.Fatal("sweep started with no Job runtime configured") + } + if startSandboxJobBootSweep(context.Background(), nil, nil, nil, quiet) { + t.Fatal("nil config must not sweep") + } + sw := &fakeJobSweeper{} + done := make(chan struct{}) + prev := newSandboxJobSweeper + newSandboxJobSweeper = func() sandboxJobSweeper { return sw } + t.Cleanup(func() { newSandboxJobSweeper = prev }) + audit := func(string, string, string, string) {} + if !startSandboxJobBootSweep(context.Background(), jobSweepConfig(), nil, audit, slog.New(slog.NewTextHandler(&signalWriter{done: done}, nil))) { + t.Fatal("sweep did not start with a Job runtime configured") + } + select { + case <-done: + case <-time.After(10 * time.Second): + t.Fatal("background sweep never finished") + } + if sw.calls != 1 || sw.opts.MaxAge != 2*time.Hour { + t.Fatalf("sweep calls = %d, MaxAge = %s", sw.calls, sw.opts.MaxAge) + } + + // A nil sweeper falls back to the factory. + sw2 := &fakeJobSweeper{} + newSandboxJobSweeper = func() sandboxJobSweeper { return sw2 } + runSandboxJobBootSweep(context.Background(), time.Hour, nil, nil, nil, nil, quiet) + if sw2.calls != 1 { + t.Fatal("nil sweeper did not use the factory") + } +} + +// signalWriter closes done on the first write: the background sweep's +// completion log line. +type signalWriter struct { + once sync.Once + done chan struct{} +} + +func (w *signalWriter) Write(p []byte) (int, error) { + if strings.Contains(string(p), "sandbox job boot sweep") { + w.once.Do(func() { close(w.done) }) + } + return len(p), nil +} + +func TestSandboxJobBootSweepLogsError(t *testing.T) { + var logBuf bytes.Buffer + sw := &fakeJobSweeper{err: errors.New("jobs.batch is forbidden")} + cfg := &config.Config{} + cfg.AgentSandbox = config.AgentSandboxConfig{Runtime: "job"} + if !sandboxJobRuntimeConfigured(cfg) { + t.Fatal("a global job runtime must count as configured") + } + runSandboxJobBootSweep(context.Background(), sandboxJobSweepMaxAge(cfg), nil, nil, sw, nil, slog.New(slog.NewTextHandler(&logBuf, nil))) + if !strings.Contains(logBuf.String(), "sandbox job boot sweep incomplete") { + t.Fatalf("error not logged: %s", logBuf.String()) + } + if sw.opts.MaxAge != kubejob.DefaultKickTimeout { + t.Fatalf("MaxAge = %s, want default", sw.opts.MaxAge) + } +} + +func TestSandboxJobLiveChecksRunningAgents(t *testing.T) { + cfg := jobSweepConfig() + rt := cruxtest.NewRuntime(cfg.Agents, slog.New(slog.NewTextHandler(&bytes.Buffer{}, nil)), crux.ProjectContext{}) + live := sandboxJobLive([]string{"reviewer", "scanner"}, rt) + // Freshly built agents are stopped, so nothing is live. + if live("hive-reviewer-sandbox") { + t.Fatal("a stopped agent's Job must not count as live") + } + if live("crux-run-abc") { + t.Fatal("a label matching no agent must not count as live") + } +} diff --git a/src/crux/agent/manager.go b/src/crux/agent/manager.go index 3122127029..5a486eea94 100644 --- a/src/crux/agent/manager.go +++ b/src/crux/agent/manager.go @@ -483,6 +483,11 @@ type AgentProcess struct { // The completion handler then turns the expected cancellation into Idle // instead of Failed. sandboxResumeAfterCancel bool + // sandboxKickActive is true while a sandbox kick's goroutine is in + // flight. StateRunning alone cannot tell that apart from a live + // interactive tmux session, and the kick refusal must name which one is + // blocking it (hivecommons/hive#18525). Guarded by m.mu. + sandboxKickActive bool // awaitingBobKey marks an agent that launchInTmux parked in StateFailed // for the single, fully-recoverable reason "bob backend with no API key". @@ -1476,9 +1481,13 @@ func (m *Manager) UpdateConfig(name string, cfg config.AgentConfig) error { return fmt.Errorf("agent %s not found", name) } + wasSandboxed := m.agentSandboxEnabledLocked(agent) agent.clearAgentSpecPromptIfSpecRemoved(cfg) agent.Config = cfg agent.baseConfig = cfg + if !wasSandboxed && m.agentSandboxEnabledLocked(agent) { + m.enterSandboxModeLocked(agent) + } return nil } @@ -1498,9 +1507,13 @@ func (m *Manager) ReconcileAgents(configs map[string]config.AgentConfig) []strin allowedConfigs[name] = cfg if existing, ok := m.agents[name]; ok { delete(m.idToName, existing.ID) + wasSandboxed := m.agentSandboxEnabledLocked(existing) existing.clearAgentSpecPromptIfSpecRemoved(cfg) existing.Config = cfg existing.baseConfig = cfg + if !wasSandboxed && m.agentSandboxEnabledLocked(existing) { + m.enterSandboxModeLocked(existing) + } if cfg.ID != "" { existing.ID = cfg.ID } else { diff --git a/src/crux/agent/manager_sandbox.go b/src/crux/agent/manager_sandbox.go index a365511161..92bb75c1da 100644 --- a/src/crux/agent/manager_sandbox.go +++ b/src/crux/agent/manager_sandbox.go @@ -42,7 +42,18 @@ func openShellPreflightWarnings(blocks []config.SandboxOpenShellConfig) []string func (m *Manager) SetSandboxConfig(cfg config.AgentSandboxConfig) { m.mu.Lock() defer m.mu.Unlock() + was := make(map[string]bool, len(m.agents)) + for name, agent := range m.agents { + was[name] = m.agentSandboxEnabledLocked(agent) + } m.sandboxConfig = cfg + // Turning the global gate on can move agents into sandbox mode just as a + // per-agent toggle does; their interactive sessions go the same way. + for name, agent := range m.agents { + if !was[name] && m.agentSandboxEnabledLocked(agent) { + m.enterSandboxModeLocked(agent) + } + } } func (m *Manager) SetSandboxLauncher(l sandbox.Launcher) { @@ -99,7 +110,7 @@ func (m *Manager) validateSandboxKickLocked(agent *AgentProcess) (string, error) return "", fmt.Errorf("agent %s cannot be kicked: %s", agent.Name, notRunningReason(agent)) } if agent.State == StateRunning { - return "", fmt.Errorf("agent %s cannot be kicked: sandbox execution already running", agent.Name) + return "", sandboxKickBlockedError(agent) } if strings.TrimSpace(m.sandboxConfig.Image) == "" && strings.TrimSpace(agent.Config.SandboxImage(m.sandboxConfig)) == "" { return "", fmt.Errorf("agent %s sandbox image is not configured", agent.Name) @@ -156,7 +167,7 @@ func (m *Manager) startSandboxKickLocked(agent *AgentProcess, message string) er launcher := m.sandboxLauncher switch runtime { case config.SandboxRuntimeJob: - launcher = m.jobLauncherLocked(agent.Config.SandboxJob(m.sandboxConfig)) + launcher = m.jobLauncherLocked(agent.Config.SandboxJob(m.sandboxConfig), time.Duration(agent.Config.SandboxTimeoutS(m.sandboxConfig))*time.Second) case config.SandboxRuntimeOpenShell: launcher = m.openshellLauncherLocked(agent.Config.SandboxOpenShell(m.sandboxConfig)) case config.SandboxRuntimeSubstrate: @@ -165,6 +176,7 @@ func (m *Manager) startSandboxKickLocked(agent *AgentProcess, message string) er repo := m.project.PrimaryRepo() now := time.Now() agent.State = StateRunning + agent.sandboxKickActive = true agent.StartedAt = &now agent.LastKick = &now agent.LastKickMessage = message @@ -307,6 +319,7 @@ func (m *Manager) runSandboxKick(ctx context.Context, name, runtime string, spec return } agent.cancel = nil + agent.sandboxKickActive = false // The run record is what a sandboxed agent's health and activity derive // from: there is no session to probe between kicks. Recorded on every // outcome but an operator cancel, and persisted so a pod roll does not @@ -397,11 +410,21 @@ func (m *Manager) setSandboxJobLauncherFactoryForTest(f func(config.SandboxJobCo // jobLauncherLocked returns the launcher for a sandbox.runtime: job kick: // the injected factory's, or a real in-cluster kubejob.Launcher. -func (m *Manager) jobLauncherLocked(job config.SandboxJobConfig) sandbox.Launcher { +func (m *Manager) jobLauncherLocked(job config.SandboxJobConfig, kickTimeout time.Duration) sandbox.Launcher { if m.sandboxJobLauncherFactory != nil { return m.sandboxJobLauncherFactory(job) } - return &kubejob.Launcher{Options: JobOptionsFromConfig(job)} + return &kubejob.Launcher{Options: jobOptionsForKick(job, kickTimeout)} +} + +// jobOptionsForKick is JobOptionsFromConfig plus the Job's +// activeDeadlineSeconds, derived from the kick timeout, so a Job that sits +// Pending or Unschedulable after hive lost track of it fails on its own +// instead of waiting forever (hivecommons/hive#18525). +func jobOptionsForKick(job config.SandboxJobConfig, kickTimeout time.Duration) kubejob.Options { + opts := JobOptionsFromConfig(job) + opts.ActiveDeadlineSeconds = kubejob.ActiveDeadlineSeconds(kickTimeout) + return opts } func (m *Manager) setSandboxOpenShellLauncherFactoryForTest(f func(config.SandboxOpenShellConfig) sandbox.Launcher) { diff --git a/src/crux/agent/manager_sandbox_mode.go b/src/crux/agent/manager_sandbox_mode.go new file mode 100644 index 0000000000..efe93b0f69 --- /dev/null +++ b/src/crux/agent/manager_sandbox_mode.go @@ -0,0 +1,59 @@ +package agent + +import ( + "fmt" + "strings" + "time" +) + +// sandboxModeArchiveReason labels the kick-log archive written when an agent's +// interactive session is retired because it switched to sandbox execution. +const sandboxModeArchiveReason = "sandbox-mode" + +// sandboxKickBlockedError is the refusal for a sandbox kick to an agent in +// StateRunning. It names what is actually holding the agent: a sandbox kick +// still in flight, or an interactive tmux session left over from before the +// agent was switched to sandbox mode (hivecommons/hive#18525). The second +// used to read "sandbox execution already running", which sent operators +// looking for a sandbox run that did not exist. +func sandboxKickBlockedError(agent *AgentProcess) error { + if agent.sandboxKickActive { + return fmt.Errorf("agent %s cannot be kicked: a sandbox kick is already running", agent.Name) + } + return fmt.Errorf("agent %s cannot be kicked: agent has a live interactive session; sandbox kicks are blocked until it stops", agent.Name) +} + +// enterSandboxModeLocked retires the interactive tmux session of an agent +// whose config just switched it to sandbox execution. Sandboxed agents never +// run a CLI in tmux, and a session left running keeps the agent in +// StateRunning, which refuses every sandbox kick (hivecommons/hive#18525). +// The session's scrollback is archived first, the agent's CLI is reaped and +// the session killed, and a running agent drops to StateIdle — the state a +// sandboxed agent waits in between kicks. Paused and stopped agents keep +// their state. Callers must hold m.mu. +func (m *Manager) enterSandboxModeLocked(agent *AgentProcess) { + if agent == nil || agent.sandboxKickActive || agent.State != StateRunning { + return + } + m.archiveKickLogLocked(agent, sandboxModeArchiveReason) + if agent.cancel != nil { + agent.cancel() + agent.cancel = nil + } + reaped := m.reapAgentCLI(agent) + if output, err := m.tmuxCmd(agent, "kill-session", "-t", agent.tmuxSession).CombinedOutput(); err != nil && !tmuxKillSessionNothingToKill(string(output)) { + m.logger.Warn("tmux kill-session on entering sandbox mode failed", + "name", agent.Name, + "session", agent.tmuxSession, + "error", err, + "stderr", strings.TrimSpace(string(output))) + } + now := time.Now() + agent.State = StateIdle + agent.StartedAt = &now + agent.HasLaunched = true + agent.LaunchedMode = m.agentMode(agent) + m.logger.Info("audit: agent switched to sandbox mode; interactive session stopped", + "name", agent.Name, "session", agent.tmuxSession, "reaped_cli", reaped) + m.auditSandbox(agent.Name, "sandbox_mode_entered", "interactive tmux session "+agent.tmuxSession+" stopped") +} diff --git a/src/crux/agent/manager_sandbox_mode_test.go b/src/crux/agent/manager_sandbox_mode_test.go new file mode 100644 index 0000000000..d7ca4e6e5c --- /dev/null +++ b/src/crux/agent/manager_sandbox_mode_test.go @@ -0,0 +1,170 @@ +package agent + +import ( + "log/slog" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/hivecommons/hive/crux/config" +) + +// recordingTmux installs a fake tmux that appends each invocation's argv to a +// log file, and returns a reader for that log. +func recordingTmux(t *testing.T) func() string { + t.Helper() + logPath := filepath.Join(t.TempDir(), "tmux.log") + writeFakeTmux(t, "#!/bin/sh\necho \"$@\" >> '"+logPath+"'\nexit 0\n") + return func() string { + raw, _ := os.ReadFile(logPath) + return string(raw) + } +} + +func sandboxOptIn() *config.AgentSandboxOverride { + on := true + return &config.AgentSandboxOverride{Enabled: &on, Image: "img"} +} + +func TestUpdateConfigIntoSandboxStopsTmuxSession(t *testing.T) { + tmuxLog := recordingTmux(t) + m := NewManager(map[string]config.AgentConfig{"reviewer": {Backend: "claude"}}, quietTestLogger(), ProjectContext{}) + m.SetSandboxConfig(config.AgentSandboxConfig{Enabled: true}) + var audits []string + m.SetSandboxAuditCallback(func(agent, action, detail string) { audits = append(audits, agent+":"+action) }) + m.mu.Lock() + agent := m.agents["reviewer"] + agent.State = StateRunning + cancelled := false + agent.cancel = func() { cancelled = true } + cfg := agent.Config + m.mu.Unlock() + + cfg.Sandbox = sandboxOptIn() + if err := m.UpdateConfig("reviewer", cfg); err != nil { + t.Fatal(err) + } + + m.mu.RLock() + state := agent.State + m.mu.RUnlock() + if state != StateIdle { + t.Fatalf("state = %s, want %s after entering sandbox mode", state, StateIdle) + } + if !cancelled { + t.Fatal("the interactive session's context was not cancelled") + } + if got := tmuxLog(); !strings.Contains(got, "kill-session -t hive-reviewer") { + t.Fatalf("tmux kill-session not run; tmux saw:\n%s", got) + } + if len(audits) != 1 || audits[0] != "reviewer:sandbox_mode_entered" { + t.Fatalf("audits = %v, want one sandbox_mode_entered", audits) + } + + // A kick now reaches the sandbox validator past the running-state gate. + m.mu.Lock() + _, err := m.validateSandboxKickLocked(agent) + m.mu.Unlock() + if err != nil && strings.Contains(err.Error(), "interactive session") { + t.Fatalf("kick still blocked by the interactive session: %v", err) + } +} + +func TestReconcileAgentsIntoSandboxStopsTmuxSession(t *testing.T) { + tmuxLog := recordingTmux(t) + m := NewManager(map[string]config.AgentConfig{"reviewer": {Backend: "claude"}}, quietTestLogger(), ProjectContext{}) + m.SetSandboxConfig(config.AgentSandboxConfig{Enabled: true}) + m.mu.Lock() + m.agents["reviewer"].State = StateRunning + m.mu.Unlock() + + m.ReconcileAgents(map[string]config.AgentConfig{"reviewer": {Backend: "claude", Sandbox: sandboxOptIn()}}) + + m.mu.RLock() + state := m.agents["reviewer"].State + m.mu.RUnlock() + if state != StateIdle { + t.Fatalf("state = %s, want %s", state, StateIdle) + } + if !strings.Contains(tmuxLog(), "kill-session") { + t.Fatal("tmux kill-session not run") + } +} + +func TestGlobalSandboxGateStopsTmuxSession(t *testing.T) { + tmuxLog := recordingTmux(t) + m := NewManager(map[string]config.AgentConfig{"reviewer": {Backend: "claude", Sandbox: sandboxOptIn()}}, quietTestLogger(), ProjectContext{}) + m.mu.Lock() + m.agents["reviewer"].State = StateRunning + m.mu.Unlock() + + m.SetSandboxConfig(config.AgentSandboxConfig{Enabled: true}) + + m.mu.RLock() + state := m.agents["reviewer"].State + m.mu.RUnlock() + if state != StateIdle { + t.Fatalf("state = %s, want %s", state, StateIdle) + } + if !strings.Contains(tmuxLog(), "kill-session") { + t.Fatal("tmux kill-session not run") + } +} + +// An agent already in sandbox mode, or one mid sandbox kick, is not touched. +func TestEnterSandboxModeLeavesSandboxKickAlone(t *testing.T) { + tmuxLog := recordingTmux(t) + m := NewManager(map[string]config.AgentConfig{"reviewer": {Backend: "claude"}}, quietTestLogger(), ProjectContext{}) + m.mu.Lock() + agent := m.agents["reviewer"] + agent.State = StateRunning + agent.sandboxKickActive = true + m.enterSandboxModeLocked(agent) + agent.sandboxKickActive = false + agent.State = StatePaused + m.enterSandboxModeLocked(agent) + m.enterSandboxModeLocked(nil) + state := agent.State + m.mu.Unlock() + if state != StatePaused { + t.Fatalf("state = %s, want paused kept", state) + } + if got := tmuxLog(); strings.Contains(got, "kill-session") { + t.Fatalf("kill-session ran for an agent it must leave alone:\n%s", got) + } +} + +func TestEnterSandboxModeLogsKillFailure(t *testing.T) { + writeFakeTmux(t, "#!/bin/sh\necho 'permission denied' >&2\nexit 1\n") + logBuf := &syncBuffer{} + m := NewManager(map[string]config.AgentConfig{"reviewer": {Backend: "claude"}}, slog.New(slog.NewTextHandler(logBuf, nil)), ProjectContext{}) + m.mu.Lock() + agent := m.agents["reviewer"] + agent.State = StateRunning + m.enterSandboxModeLocked(agent) + m.mu.Unlock() + if !strings.Contains(logBuf.String(), "tmux kill-session on entering sandbox mode failed") { + t.Fatalf("kill failure not logged: %s", logBuf.String()) + } +} + +func TestSandboxKickRefusalNamesRealState(t *testing.T) { + m := NewManager(map[string]config.AgentConfig{"reviewer": {Backend: "claude", Sandbox: sandboxOptIn()}}, quietTestLogger(), ProjectContext{}) + m.SetSandboxConfig(config.AgentSandboxConfig{Enabled: true}) + m.mu.Lock() + defer m.mu.Unlock() + agent := m.agents["reviewer"] + agent.State = StateRunning + + _, err := m.validateSandboxKickLocked(agent) + if err == nil || !strings.Contains(err.Error(), "agent has a live interactive session; sandbox kicks are blocked until it stops") { + t.Fatalf("err = %v, want the live-interactive-session refusal", err) + } + + agent.sandboxKickActive = true + _, err = m.validateSandboxKickLocked(agent) + if err == nil || !strings.Contains(err.Error(), "a sandbox kick is already running") { + t.Fatalf("err = %v, want the sandbox-kick-in-flight refusal", err) + } +} diff --git a/src/crux/agent/sandbox_executor.go b/src/crux/agent/sandbox_executor.go index 69d218e3d1..166f672473 100644 --- a/src/crux/agent/sandbox_executor.go +++ b/src/crux/agent/sandbox_executor.go @@ -25,7 +25,7 @@ import ( ) const ( - DefaultSandboxKickTimeout = 45 * time.Minute + DefaultSandboxKickTimeout = kubejob.DefaultKickTimeout defaultSandboxNetworkMode = sandbox.NetworkModeRestricted sandboxPromptRelPath = ".hive/kick-prompt.txt" sandboxTranscriptRelPath = sandbox.TranscriptRelPath diff --git a/src/crux/agent/sandbox_job_runtime_test.go b/src/crux/agent/sandbox_job_runtime_test.go index 02010eddc3..e7a584e982 100644 --- a/src/crux/agent/sandbox_job_runtime_test.go +++ b/src/crux/agent/sandbox_job_runtime_test.go @@ -8,6 +8,7 @@ import ( "time" "github.com/hivecommons/hive/crux/config" + "github.com/hivecommons/hive/crux/kubejob" "github.com/hivecommons/hive/crux/sandbox" ) @@ -170,7 +171,7 @@ func TestJobOptionsFromConfig(t *testing.T) { // the executor and silently run the kick on the hive node. func TestJobLauncherDefaultsToKubejob(t *testing.T) { m := NewManager(map[string]config.AgentConfig{}, quietTestLogger(), ProjectContext{}) - l := m.jobLauncherLocked(config.SandboxJobConfig{WorkspaceClaim: "hive-data"}) + l := m.jobLauncherLocked(config.SandboxJobConfig{WorkspaceClaim: "hive-data"}, 0) if l == nil { t.Fatal("jobLauncherLocked returned nil") } @@ -178,3 +179,19 @@ func TestJobLauncherDefaultsToKubejob(t *testing.T) { t.Fatal("job runtime must not resolve to the Podman launcher") } } + +// Every sandbox Job carries activeDeadlineSeconds derived from the kick +// timeout, so a Job orphaned by a hive restart fails on its own +// (hivecommons/hive#18525). +func TestJobLauncherSetsDeadlineFromKickTimeout(t *testing.T) { + m := NewManager(map[string]config.AgentConfig{}, quietTestLogger(), ProjectContext{}) + const kickTimeout = 20 * time.Minute + l, ok := m.jobLauncherLocked(config.SandboxJobConfig{WorkspaceClaim: "hive-data"}, kickTimeout).(*kubejob.Launcher) + if !ok { + t.Fatal("jobLauncherLocked did not return a kubejob launcher") + } + want := int64((kickTimeout + kubejob.DeadlineMargin) / time.Second) + if l.Options.ActiveDeadlineSeconds != want { + t.Fatalf("ActiveDeadlineSeconds = %d, want %d", l.Options.ActiveDeadlineSeconds, want) + } +} diff --git a/src/crux/config/agent_sandbox_block.go b/src/crux/config/agent_sandbox_block.go new file mode 100644 index 0000000000..d6e8b4340c --- /dev/null +++ b/src/crux/config/agent_sandbox_block.go @@ -0,0 +1,86 @@ +package config + +import ( + "encoding/json" + "fmt" + "strings" +) + +// SandboxBlockProblem reports why an agent's effective sandbox block — its own +// sandbox fields, then the agent_sandbox defaults — cannot run a kick, or "" +// when the block is complete. It checks only what config alone decides (the +// same config checks the manager makes before a kick), so the dashboard can +// refuse to turn the sandbox on for an agent whose every kick would fail +// (hivecommons/hive#18525). +func (a *AgentConfig) SandboxBlockProblem(global AgentSandboxConfig) string { + if strings.TrimSpace(a.SandboxImage(global)) == "" { + return "no sandbox image: neither the agent's sandbox.image nor agent_sandbox.image is set" + } + runtime := a.SandboxRuntime(global) + if !ValidSandboxRuntimes[runtime] { + return fmt.Sprintf("sandbox runtime %q is not one hive has (%s, %s, %s or %s)", runtime, SandboxRuntimePodman, SandboxRuntimeJob, SandboxRuntimeOpenShell, SandboxRuntimeSubstrate) + } + if runtime == SandboxRuntimeJob && strings.TrimSpace(a.SandboxJob(global).WorkspaceClaim) == "" { + return "the job sandbox runtime needs job.workspace_claim, in the agent's sandbox.job or in agent_sandbox.job" + } + return "" +} + +// PrefilledSandboxBlock returns the agent's own sandbox block with the image +// and runtime pinned to what they resolve to now (the agent's value, else the +// agent_sandbox default) and enabled set to true. Every other field is left +// as the agent wrote it, so it keeps inheriting the global default. +func (a *AgentConfig) PrefilledSandboxBlock(global AgentSandboxConfig) *AgentSandboxOverride { + out := &AgentSandboxOverride{} + if a != nil && a.Sandbox != nil { + out = cloneSandboxOverride(a.Sandbox) + } + out.Image = a.SandboxImage(global) + out.Runtime = a.SandboxRuntime(global) + on := true + out.Enabled = &on + return out +} + +// CopiedSandboxBlock returns src's sandbox block for another agent to run +// with: src's own fields (deep-copied) with the image and runtime pinned to +// what they resolve to for src, enabled set to true. The publish block is +// dropped, because its verify commands are written for src's repository and +// work, not the receiving agent's; the receiving agent's own publish block, +// passed as keepPublish, is kept instead. +func CopiedSandboxBlock(src *AgentConfig, global AgentSandboxConfig, keepPublish *SandboxPublishConfig) *AgentSandboxOverride { + out := &AgentSandboxOverride{} + if src != nil && src.Sandbox != nil { + out = cloneSandboxOverride(src.Sandbox) + } + out.Image = src.SandboxImage(global) + out.Runtime = src.SandboxRuntime(global) + out.Publish = nil + if keepPublish != nil { + p := *keepPublish + p.Verify = append([]string(nil), keepPublish.Verify...) + out.Publish = &p + } + on := true + out.Enabled = &on + return out +} + +// cloneSandboxOverride deep-copies a sandbox block. Every field carries a +// JSON tag, so a round trip copies the nested maps and slices too. +func cloneSandboxOverride(in *AgentSandboxOverride) *AgentSandboxOverride { + out := &AgentSandboxOverride{} + if in == nil { + return out + } + raw, err := json.Marshal(in) + if err != nil { + cp := *in + return &cp + } + if err := json.Unmarshal(raw, out); err != nil { + cp := *in + return &cp + } + return out +} diff --git a/src/crux/config/agent_sandbox_block_test.go b/src/crux/config/agent_sandbox_block_test.go new file mode 100644 index 0000000000..854c4d29ed --- /dev/null +++ b/src/crux/config/agent_sandbox_block_test.go @@ -0,0 +1,90 @@ +package config + +import ( + "strings" + "testing" +) + +func TestSandboxBlockProblem(t *testing.T) { + tests := []struct { + name string + agent AgentConfig + global AgentSandboxConfig + want string + }{ + {name: "no image anywhere", want: "no sandbox image"}, + {name: "global image, default podman", global: AgentSandboxConfig{Image: "img"}}, + {name: "agent image", agent: AgentConfig{Sandbox: &AgentSandboxOverride{Image: "img"}}}, + {name: "bad runtime", global: AgentSandboxConfig{Image: "img", Runtime: "vm"}, want: `runtime "vm"`}, + {name: "job without claim", global: AgentSandboxConfig{Image: "img", Runtime: SandboxRuntimeJob}, want: "workspace_claim"}, + {name: "job with global claim", global: AgentSandboxConfig{Image: "img", Runtime: SandboxRuntimeJob, Job: &SandboxJobConfig{WorkspaceClaim: "data"}}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tt.agent.SandboxBlockProblem(tt.global) + if tt.want == "" && got != "" { + t.Fatalf("problem = %q, want none", got) + } + if tt.want != "" && !strings.Contains(got, tt.want) { + t.Fatalf("problem = %q, want it to mention %q", got, tt.want) + } + }) + } +} + +func TestPrefilledSandboxBlockPinsDefaults(t *testing.T) { + global := AgentSandboxConfig{Image: "global-img", Runtime: SandboxRuntimeJob, TimeoutS: 600} + a := AgentConfig{Sandbox: &AgentSandboxOverride{EnvAllowlist: []string{"FOO"}}} + got := a.PrefilledSandboxBlock(global) + if got.Image != "global-img" || got.Runtime != SandboxRuntimeJob { + t.Fatalf("prefill = %+v, want global image and runtime pinned", got) + } + if got.Enabled == nil || !*got.Enabled { + t.Fatal("prefill must enable the block") + } + if len(got.EnvAllowlist) != 1 || got.EnvAllowlist[0] != "FOO" { + t.Fatalf("own fields lost: %+v", got.EnvAllowlist) + } + got.EnvAllowlist[0] = "BAR" + if a.Sandbox.EnvAllowlist[0] != "FOO" { + t.Fatal("prefill aliased the agent's slice") + } + + var bare AgentConfig + if got := bare.PrefilledSandboxBlock(AgentSandboxConfig{Image: "i"}); got.Image != "i" || got.Runtime != SandboxRuntimePodman { + t.Fatalf("bare prefill = %+v", got) + } +} + +func TestCopiedSandboxBlockDropsPublish(t *testing.T) { + src := AgentConfig{Sandbox: &AgentSandboxOverride{ + Image: "spyre-img", + Runtime: SandboxRuntimeJob, + Job: &SandboxJobConfig{WorkspaceClaim: "data", NodeSelector: map[string]string{"accel": "spyre"}}, + Publish: &SandboxPublishConfig{Verify: []string{"pytest -x"}}, + }} + got := CopiedSandboxBlock(&src, AgentSandboxConfig{}, nil) + if got.Image != "spyre-img" || got.Runtime != SandboxRuntimeJob || got.Job == nil || got.Job.NodeSelector["accel"] != "spyre" { + t.Fatalf("copy = %+v", got) + } + if got.Publish != nil { + t.Fatalf("copy kept the source's publish block: %+v", got.Publish) + } + if got.Enabled == nil || !*got.Enabled { + t.Fatal("copy must enable the block") + } + got.Job.NodeSelector["accel"] = "other" + if src.Sandbox.Job.NodeSelector["accel"] != "spyre" { + t.Fatal("copy aliased the source's job block") + } + + keep := &SandboxPublishConfig{Verify: []string{"make check"}, VerifyTimeoutS: 30} + got = CopiedSandboxBlock(&src, AgentSandboxConfig{}, keep) + if got.Publish == nil || got.Publish.Verify[0] != "make check" || got.Publish.VerifyTimeoutS != 30 { + t.Fatalf("receiver publish not kept: %+v", got.Publish) + } + + if got := cloneSandboxOverride(nil); got == nil { + t.Fatal("clone(nil) = nil, want empty block") + } +} diff --git a/src/crux/kubejob/kubejob.go b/src/crux/kubejob/kubejob.go index 5f742ce7ed..ba5252b403 100644 --- a/src/crux/kubejob/kubejob.go +++ b/src/crux/kubejob/kubejob.go @@ -62,6 +62,18 @@ const ( // DefaultPollInterval is how often the launcher re-reads the Job status. DefaultPollInterval = 5 * time.Second + // DefaultKickTimeout is the sandbox kick timeout hive applies when none is + // configured; crux/agent's DefaultSandboxKickTimeout is this value. + DefaultKickTimeout = 45 * time.Minute + + // DeadlineMargin is added to a kick's timeout to give the Job's + // activeDeadlineSeconds. Hive cancels a kick at its timeout and deletes + // the Job itself; the deadline is the cluster-side backstop for a Job + // hive lost track of (a restart mid-kick, a Job left Pending or + // Unschedulable), so it must land after hive's own timeout, never before + // (hivecommons/hive#18525). + DeadlineMargin = 15 * time.Minute + // requestTimeout bounds every single API call; the overall run is bounded // by the caller's context (the sandbox kick timeout). requestTimeout = 30 * time.Second @@ -138,6 +150,37 @@ type Options struct { // RelayRunAsUser pins the relay init container's uid. Nil leaves it to // the cluster on OpenShift and uses DefaultRelayRunAsUser elsewhere. RelayRunAsUser *int64 + // ActiveDeadlineSeconds is the Job's activeDeadlineSeconds. Non-positive + // derives it from the run's context deadline, else from + // DefaultKickTimeout; either way DeadlineMargin is added. Run uses + // whichever of this and the context-derived value is later, so the + // deadline never cuts a run short of the timeout hive gave it. + ActiveDeadlineSeconds int64 +} + +// ActiveDeadlineSeconds returns the activeDeadlineSeconds for a Job running +// one kick with the given timeout: the timeout (DefaultKickTimeout when +// non-positive) plus DeadlineMargin. +func ActiveDeadlineSeconds(kickTimeout time.Duration) int64 { + if kickTimeout <= 0 { + kickTimeout = DefaultKickTimeout + } + return int64((kickTimeout + DeadlineMargin) / time.Second) +} + +// activeDeadline resolves the deadline for one Run: the configured value or +// the one derived from ctx's remaining time, whichever is later. +func (l *Launcher) activeDeadline(ctx context.Context) int64 { + deadline := l.Options.ActiveDeadlineSeconds + if dl, ok := ctx.Deadline(); ok { + if fromCtx := ActiveDeadlineSeconds(time.Until(dl)); fromCtx > deadline { + deadline = fromCtx + } + } + if deadline <= 0 { + deadline = ActiveDeadlineSeconds(DefaultKickTimeout) + } + return deadline } // network is Options.Network with the empty default applied. @@ -264,7 +307,11 @@ func (l *Launcher) Run(ctx context.Context, spec sandbox.LaunchSpec) (sandbox.Re } } name := jobName(spec.Name, l.now()) - body, err := json.Marshal(l.jobManifest(ns, name, subPath, spec, relays)) + manifest := l.jobManifest(ns, name, subPath, spec, relays) + if jobSpec, ok := manifest["spec"].(map[string]any); ok { + jobSpec["activeDeadlineSeconds"] = l.activeDeadline(ctx) + } + body, err := json.Marshal(manifest) if err != nil { return res, err } @@ -526,6 +573,10 @@ func (l *Launcher) jobManifest(ns, name, subPath string, spec sandbox.LaunchSpec if ttl == 0 { ttl = DefaultTTLSeconds } + deadline := l.Options.ActiveDeadlineSeconds + if deadline <= 0 { + deadline = ActiveDeadlineSeconds(DefaultKickTimeout) + } // Same filter as the Podman path; explicit spec.Env entries come last and win. merged := map[string]string{} @@ -644,6 +695,10 @@ func (l *Launcher) jobManifest(ns, name, subPath string, spec sandbox.LaunchSpec // workspace the first attempt already changed. "backoffLimit": 0, "ttlSecondsAfterFinished": ttl, + // The cluster fails a Job still running (or never scheduled) + // past this, so one hive lost track of cannot sit Pending + // forever (hivecommons/hive#18525). + "activeDeadlineSeconds": deadline, "template": map[string]any{ "metadata": map[string]any{"labels": labels}, "spec": podSpec, diff --git a/src/crux/kubejob/sweep.go b/src/crux/kubejob/sweep.go new file mode 100644 index 0000000000..4d3360c0d2 --- /dev/null +++ b/src/crux/kubejob/sweep.go @@ -0,0 +1,156 @@ +package kubejob + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/url" + "strings" + "time" +) + +// Sweep reasons, as reported on each SweptJob. +const ( + SweepReasonPendingPastDeadline = "pending past its deadline" + SweepReasonOrphaned = "not tied to a live kick and older than the kick timeout" +) + +// Launch-name shapes of the Jobs the agent sandbox creates: a kick is +// hive--sandbox, a publish preflight hive--verify. Contained +// runs (crux/runs) share the managed-by label but are named crux-run- +// and belong to the runs service, so the sweep leaves them alone. +const ( + sandboxJobAgentPrefix = "hive-" + sandboxJobKickSuffix = "-sandbox" + sandboxJobVerifySuffix = "-verify" +) + +// SweepOptions bounds an orphan sweep. +type SweepOptions struct { + // MaxAge is the kick timeout: an unfinished Job older than this that no + // live kick owns is an orphan. Non-positive uses DefaultKickTimeout. + MaxAge time.Duration + // Live reports whether a live kick owns the Job with this agent label + // (hive--sandbox / -verify). Nil treats nothing as live, which is + // the truth at boot: the kicks of the previous hive process are gone. + Live func(agentLabel string) bool +} + +// SweptJob is one Job the sweep deleted. +type SweptJob struct { + Name string + Agent string + Age time.Duration + Reason string +} + +type jobList struct { + Items []struct { + Metadata struct { + Name string `json:"name"` + CreationTimestamp time.Time `json:"creationTimestamp"` + Labels map[string]string `json:"labels"` + } `json:"metadata"` + Status struct { + Active int `json:"active"` + Ready int `json:"ready"` + Conditions []struct { + Type string `json:"type"` + Status string `json:"status"` + } `json:"conditions"` + } `json:"status"` + } `json:"items"` +} + +// SweepOrphans deletes sandbox Jobs hive lost track of (hivecommons/hive#18525): +// a Job's in-memory kick timeout dies with the hive process, and +// ttlSecondsAfterFinished only reaps a Job that finished, so a Job left +// Pending by a restart could sit there for days. It lists Jobs labelled +// app.kubernetes.io/managed-by=hive-kubejob in the hive's namespace and +// deletes each unfinished agent-sandbox Job that no live kick owns and that +// is older than the kick timeout, and each one that never got a ready pod +// and is past the kick deadline (timeout plus DeadlineMargin). Finished Jobs +// are left to their TTL so an operator can still inspect a failure; Jobs +// without the label, and contained runs, are never touched. Deletion errors do not stop the sweep; the first +// is returned with whatever was deleted. +func (l *Launcher) SweepOrphans(ctx context.Context, opts SweepOptions) ([]SweptJob, error) { + maxAge := opts.MaxAge + if maxAge <= 0 { + maxAge = DefaultKickTimeout + } + pendingDeadline := time.Duration(ActiveDeadlineSeconds(maxAge)) * time.Second + ns, err := l.namespace() + if err != nil { + return nil, err + } + jobsPath := fmt.Sprintf("/apis/batch/v1/namespaces/%s/jobs", ns) + selector := url.QueryEscape(fmt.Sprintf(labelSelectorFmt, managedByLabel, managedByValue)) + raw, err := l.do(ctx, http.MethodGet, jobsPath+"?labelSelector="+selector, nil) + if err != nil { + return nil, fmt.Errorf("kubejob: listing sandbox jobs: %w", err) + } + var list jobList + if err := json.Unmarshal(raw, &list); err != nil { + return nil, fmt.Errorf("kubejob: decoding sandbox job list: %w", err) + } + now := l.now() + var swept []SweptJob + var firstErr error + for _, item := range list.Items { + md := item.Metadata + // The selector already asked for this; check again so a server that + // ignored it can never get an unlabelled Job deleted. + if md.Labels[managedByLabel] != managedByValue || md.Name == "" { + continue + } + agent := md.Labels[agentLabel] + if !isAgentSandboxJob(agent) { + continue + } + finished := false + for _, c := range item.Status.Conditions { + if (c.Type == condComplete || c.Type == condFailed) && c.Status == condStatusTrue { + finished = true + } + } + if finished { + continue + } + if md.CreationTimestamp.IsZero() { + continue + } + age := now.Sub(md.CreationTimestamp) + var reason string + switch { + case item.Status.Ready == 0 && age > pendingDeadline: + // Never got a ready pod and outlived any kick's deadline: no + // kick can still be waiting on it. + reason = SweepReasonPendingPastDeadline + case age > maxAge && (opts.Live == nil || !opts.Live(agent)): + reason = SweepReasonOrphaned + default: + continue + } + if err := l.delete(ctx, jobsPath+"/"+md.Name); err != nil { + if firstErr == nil { + firstErr = fmt.Errorf("kubejob: deleting orphaned job %s: %w", md.Name, err) + } + continue + } + // A network-none Job had a per-Job NetworkPolicy of the same name; + // best effort, as a Job on the default network has none. + l.deletePolicy(ctx, fmt.Sprintf("/apis/networking.k8s.io/v1/namespaces/%s/networkpolicies/%s", ns, md.Name)) + swept = append(swept, SweptJob{Name: md.Name, Agent: agent, Age: age, Reason: reason}) + } + return swept, firstErr +} + +// isAgentSandboxJob reports whether an agent label names an agent-sandbox +// kick or publish-preflight Job, as opposed to a contained run. +func isAgentSandboxJob(agent string) bool { + if !strings.HasPrefix(agent, sandboxJobAgentPrefix) { + return false + } + return strings.HasSuffix(agent, sandboxJobKickSuffix) || strings.HasSuffix(agent, sandboxJobVerifySuffix) +} diff --git a/src/crux/kubejob/sweep_test.go b/src/crux/kubejob/sweep_test.go new file mode 100644 index 0000000000..8323f788bd --- /dev/null +++ b/src/crux/kubejob/sweep_test.go @@ -0,0 +1,247 @@ +package kubejob + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + "time" +) + +// sweepAPI serves a fixed Job list and records deletes. +type sweepAPI struct { + mu sync.Mutex + list string + listErr int + selector string + deleted []string + delErr map[string]bool +} + +func (f *sweepAPI) handler(t *testing.T) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + f.mu.Lock() + defer f.mu.Unlock() + switch { + case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/jobs"): + if f.listErr > 0 { + w.WriteHeader(f.listErr) + _, _ = w.Write([]byte(`{"message":"jobs.batch is forbidden"}`)) + return + } + f.selector = r.URL.Query().Get("labelSelector") + _, _ = w.Write([]byte(f.list)) + case r.Method == http.MethodDelete && strings.Contains(r.URL.Path, "/jobs/"): + name := r.URL.Path[strings.LastIndex(r.URL.Path, "/")+1:] + if f.delErr[name] { + w.WriteHeader(http.StatusInternalServerError) + return + } + f.deleted = append(f.deleted, name) + _, _ = w.Write([]byte(`{"kind":"Status","status":"Success"}`)) + case r.Method == http.MethodDelete && strings.Contains(r.URL.Path, "/networkpolicies/"): + w.WriteHeader(http.StatusNotFound) + default: + t.Errorf("unexpected request %s %s", r.Method, r.URL) + w.WriteHeader(http.StatusNotFound) + } + } +} + +type sweepItem struct { + name, agent string + managed bool + age time.Duration + ready int + finished string +} + +func sweepList(t *testing.T, now time.Time, items ...sweepItem) string { + t.Helper() + out := make([]map[string]any, 0, len(items)) + for _, it := range items { + labels := map[string]string{agentLabel: it.agent} + if it.managed { + labels[managedByLabel] = managedByValue + } + status := map[string]any{"ready": it.ready} + if it.finished != "" { + status["conditions"] = []map[string]string{{"type": it.finished, "status": condStatusTrue}} + } + out = append(out, map[string]any{ + "metadata": map[string]any{ + "name": it.name, + "creationTimestamp": now.Add(-it.age).UTC().Format(time.RFC3339), + "labels": labels, + }, + "status": status, + }) + } + raw, err := json.Marshal(map[string]any{"items": out}) + if err != nil { + t.Fatal(err) + } + return string(raw) +} + +func TestSweepOrphansDeletesOrphansOnly(t *testing.T) { + const kickTimeout = 45 * time.Minute + now := time.Unix(1700000000, 0) + api := &sweepAPI{} + api.list = sweepList(t, now, + // Pending for 11 days: the incident this sweep exists for. + sweepItem{name: "hive-kick-reviewer-1", agent: "hive-reviewer-sandbox", managed: true, age: 11 * 24 * time.Hour}, + // Running, older than the timeout, no live kick: an orphan. + sweepItem{name: "hive-kick-deps-2", agent: "hive-deps-sandbox", managed: true, age: time.Hour, ready: 1}, + // An orphaned publish preflight. + sweepItem{name: "hive-kick-deps-3", agent: "hive-deps-verify", managed: true, age: 2 * time.Hour, ready: 1}, + // Young: may still belong to a kick that is about to be re-adopted. + sweepItem{name: "hive-kick-deps-4", agent: "hive-deps-sandbox", managed: true, age: 10 * time.Minute}, + // Owned by a live kick. + sweepItem{name: "hive-kick-live-5", agent: "hive-live-sandbox", managed: true, age: time.Hour, ready: 1}, + // Finished: left to its TTL for inspection. + sweepItem{name: "hive-kick-done-6", agent: "hive-done-sandbox", managed: true, age: 2 * time.Hour, finished: condFailed}, + // No managed-by label: never ours to touch. + sweepItem{name: "someone-else-7", agent: "hive-other-sandbox", managed: false, age: 30 * 24 * time.Hour}, + // A contained run: belongs to the runs service. + sweepItem{name: "hive-kick-crux-run-8", agent: "crux-run-abc", managed: true, age: 30 * 24 * time.Hour}, + ) + srv := httptest.NewServer(api.handler(t)) + defer srv.Close() + l := newLauncher(srv, Options{}) + l.Now = func() time.Time { return now } + + swept, err := l.SweepOrphans(context.Background(), SweepOptions{ + MaxAge: kickTimeout, + Live: func(agent string) bool { return agent == "hive-live-sandbox" }, + }) + if err != nil { + t.Fatalf("SweepOrphans: %v", err) + } + want := map[string]string{ + "hive-kick-reviewer-1": SweepReasonPendingPastDeadline, + "hive-kick-deps-2": SweepReasonOrphaned, + "hive-kick-deps-3": SweepReasonOrphaned, + } + if len(swept) != len(want) { + t.Fatalf("swept = %+v, want %v", swept, want) + } + for _, s := range swept { + if want[s.Name] != s.Reason { + t.Errorf("swept %s with reason %q, want %q", s.Name, s.Reason, want[s.Name]) + } + } + if strings.Join(api.deleted, ",") != "hive-kick-reviewer-1,hive-kick-deps-2,hive-kick-deps-3" { + t.Fatalf("deleted = %v", api.deleted) + } + if api.selector != managedByLabel+"="+managedByValue { + t.Fatalf("labelSelector = %q", api.selector) + } +} + +func TestSweepOrphansPendingPastDeadlineEvenIfLive(t *testing.T) { + now := time.Unix(1700000000, 0) + api := &sweepAPI{list: sweepList(t, now, sweepItem{name: "hive-kick-a-1", agent: "hive-a-sandbox", managed: true, age: 48 * time.Hour})} + srv := httptest.NewServer(api.handler(t)) + defer srv.Close() + l := newLauncher(srv, Options{}) + l.Now = func() time.Time { return now } + swept, err := l.SweepOrphans(context.Background(), SweepOptions{Live: func(string) bool { return true }}) + if err != nil || len(swept) != 1 || swept[0].Reason != SweepReasonPendingPastDeadline { + t.Fatalf("swept = %+v, err = %v", swept, err) + } +} + +func TestSweepOrphansReportsErrors(t *testing.T) { + now := time.Unix(1700000000, 0) + + api := &sweepAPI{listErr: http.StatusForbidden} + srv := httptest.NewServer(api.handler(t)) + defer srv.Close() + l := newLauncher(srv, Options{}) + if _, err := l.SweepOrphans(context.Background(), SweepOptions{}); err == nil || !strings.Contains(err.Error(), "listing sandbox jobs") { + t.Fatalf("list error = %v", err) + } + + bad := &sweepAPI{list: "not json"} + srvBad := httptest.NewServer(bad.handler(t)) + defer srvBad.Close() + if _, err := newLauncher(srvBad, Options{}).SweepOrphans(context.Background(), SweepOptions{}); err == nil || !strings.Contains(err.Error(), "decoding") { + t.Fatalf("decode error = %v", err) + } + + del := &sweepAPI{ + list: sweepList(t, now, + sweepItem{name: "hive-kick-a-1", agent: "hive-a-sandbox", managed: true, age: 48 * time.Hour}, + sweepItem{name: "hive-kick-b-2", agent: "hive-b-sandbox", managed: true, age: 48 * time.Hour}, + ), + delErr: map[string]bool{"hive-kick-a-1": true}, + } + srvDel := httptest.NewServer(del.handler(t)) + defer srvDel.Close() + ld := newLauncher(srvDel, Options{}) + ld.Now = func() time.Time { return now } + swept, err := ld.SweepOrphans(context.Background(), SweepOptions{}) + if err == nil || !strings.Contains(err.Error(), "hive-kick-a-1") { + t.Fatalf("delete error = %v", err) + } + if len(swept) != 1 || swept[0].Name != "hive-kick-b-2" { + t.Fatalf("a failed delete must not stop the sweep: %+v", swept) + } + + noNS := &Launcher{NamespacePath: "/nonexistent/namespace"} + if _, err := noNS.SweepOrphans(context.Background(), SweepOptions{}); err == nil { + t.Fatal("missing namespace must be an error") + } +} + +func TestActiveDeadlineSeconds(t *testing.T) { + if got, want := ActiveDeadlineSeconds(0), int64((DefaultKickTimeout+DeadlineMargin)/time.Second); got != want { + t.Fatalf("ActiveDeadlineSeconds(0) = %d, want %d", got, want) + } + if got, want := ActiveDeadlineSeconds(10*time.Minute), int64((10*time.Minute+DeadlineMargin)/time.Second); got != want { + t.Fatalf("ActiveDeadlineSeconds(10m) = %d, want %d", got, want) + } +} + +func TestActiveDeadlinePrefersLaterOfConfigAndContext(t *testing.T) { + l := &Launcher{Options: Options{ActiveDeadlineSeconds: 100}} + if got := l.activeDeadline(context.Background()); got != 100 { + t.Fatalf("configured deadline = %d, want 100", got) + } + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Hour) + defer cancel() + if got := l.activeDeadline(ctx); got < int64((2*time.Hour)/time.Second) { + t.Fatalf("context deadline not honoured: %d", got) + } + empty := &Launcher{} + if got := empty.activeDeadline(context.Background()); got != ActiveDeadlineSeconds(DefaultKickTimeout) { + t.Fatalf("default deadline = %d", got) + } +} + +// Every Job hive creates carries activeDeadlineSeconds, so one hive lost +// track of fails on its own (hivecommons/hive#18525). +func TestRunSetsActiveDeadlineOnJob(t *testing.T) { + api := newFakeAPI() + api.statuses = []string{statusComplete} + srv := httptest.NewServer(api.handler(t)) + defer srv.Close() + const deadline = 1234 + l := newLauncher(srv, Options{ActiveDeadlineSeconds: deadline}) + if _, err := l.Run(context.Background(), specFor("/data/agents/sandbox/deps-123")); err != nil { + t.Fatalf("Run: %v", err) + } + spec, _ := api.created["spec"].(map[string]any) + if got, _ := spec["activeDeadlineSeconds"].(float64); got != deadline { + t.Fatalf("activeDeadlineSeconds = %v, want %d", spec["activeDeadlineSeconds"], deadline) + } + + m := (&Launcher{}).manifest("hive", "j", "sub", specFor("/data/x")) + if got := m["spec"].(map[string]any)["activeDeadlineSeconds"]; got != ActiveDeadlineSeconds(DefaultKickTimeout) { + t.Fatalf("default manifest deadline = %v", got) + } +} diff --git a/src/docs/dashboard-sections/workflow-and-agents.md b/src/docs/dashboard-sections/workflow-and-agents.md index ed61c83aaa..a53589e0c2 100644 --- a/src/docs/dashboard-sections/workflow-and-agents.md +++ b/src/docs/dashboard-sections/workflow-and-agents.md @@ -350,6 +350,8 @@ More about the line: **Sandboxed worker's last run.** A sandboxed worker's card also carries a **last run** line: when it ran, how long it took, whether it succeeded, and the pull request it opened. If the worker's own report said the run failed, or the sandbox exited with an error, Hive keeps any commits the run made off GitHub and the line reads `no PR opened: ` — for example `no PR opened: agent report status=failed: RPM install returned 401`. The same line appears when a configured publish preflight (`publish.verify`, for example a whole-suite `pytest --collect-only`) failed after the run: `no PR opened: verify failed (1/2): uv run pytest --collect-only -q (exit 2)`, with the command's output tail in the tooltip. The full reason is in the line's tooltip and in the full log's outcome block, where each verify command has its own `---- verify: … ----` section. +**Turning on Run in sandbox.** The **Run in sandbox** switch on a worker's General settings tab now asks where the sandbox settings come from before it saves: the hive-wide sandbox defaults (Governor → Security) when they name an image, or a copy of another worker that already runs in a sandbox. A copy leaves out that worker's own publish checks (`publish.verify`). If there are no usable defaults and no worker to copy from, the switch shows a red message saying what is missing and the save is refused, instead of leaving a worker that fails every round. When the switch takes effect, the worker's terminal session is stopped and it waits idle for its next sandboxed round. Turning the switch off keeps the settings for next time. See [Credential-free sandbox isolation](../sandbox-isolation.md#current-wiring). + **Example.** A card says "needs login". You click **Login** and sign in again. Another card says `Now: working — no issue or pull request yet · started 50 min ago`. That worker has changed nothing through the hive for 50 minutes, so you open its log to see what it is doing. **When it appears.** Always. diff --git a/src/docs/sandbox-isolation.md b/src/docs/sandbox-isolation.md index 5a7b30eebe..c2665def05 100644 --- a/src/docs/sandbox-isolation.md +++ b/src/docs/sandbox-isolation.md @@ -124,6 +124,16 @@ agents: Hive logs an `agent sandbox posture` warning at boot and on every config reload when the sandbox is enabled globally but some or all agents are not opted in (`config.AgentSandboxGateWarnings`), and — as of the #4918 fix — the same diagnosis also reaches the dashboard itself: `GET /api/config/governor`'s `security.sandboxWarnings` array carries it, and the Security tab renders it both in the page's "Coherence warnings" box and as an inline warning directly beneath the sandbox toggle, naming which agents are still unconfined and the exact `sandbox: {enabled: true}` key that fixes it. The array is empty (and nothing renders) on the documented default — sandbox off globally — and on a fully opted-in hive, so the warning appears only when it is actionable. +**The per-agent toggle writes a complete block, or refuses (#18525).** The dashboard General tab's **Run in sandbox** toggle (`PUT /api/config/agent/{name}/general` with `sandboxEnabled`) used to write `sandbox.enabled: true` and nothing else, so an agent turned on while `agent_sandbox` had no image fell back to Podman with no image and failed every kick with "sandbox image is not configured". Turning the toggle on now resolves the agent's effective block — its own `sandbox` fields first, then the `agent_sandbox` defaults — and: + +- when that block is complete (an image, a runtime hive has, and `job.workspace_claim` for the Job runtime), writes it with the image and runtime pinned and `enabled: true`; +- when the request names another agent with `copy_from` (the dashboard sends `sandboxCopyFrom`), copies that agent's complete block instead, **minus its `publish` block**: `publish.verify` commands are written for one repository's work, so the receiving agent keeps its own; +- otherwise refuses with HTTP 400 naming what is missing and the agents a block could be copied from. + +When the toggle is switched on, the General tab shows a **Sandbox settings from** picker: the `agent_sandbox` defaults when they are complete, plus every agent that already has a complete block. If neither exists it shows an inline error instead, and the save would be refused. `GET /api/config/agent/{name}` carries the inputs as `general.sandboxDefaultsProblem` (empty when the defaults are complete) and `general.sandboxCopySources`. Turning the toggle off keeps the block and only sets `enabled: false`. + +**Switching an agent into sandbox mode stops its tmux session.** A sandboxed agent never runs a CLI in tmux, but the session it had before the switch used to stay alive and keep the agent in `running`, so every governor kick was refused. When an agent's config changes it into sandbox mode (the toggle, a config reload, or the global gate turning on), the manager archives the session's scrollback as a kick log, reaps the CLI, kills the session and parks the agent in `idle`, and writes a `sandbox_mode_entered` audit line. A kick refused because the agent is still `running` now says which state is holding it: `a sandbox kick is already running`, or `agent has a live interactive session; sandbox kicks are blocked until it stops`. + The second gate is deliberate rather than an oversight, and it is not safe to simply collapse. A sandboxed agent runs a different execution model — no tmux CLI at all, and every kick is a Podman run against the primary repo — and `startSandboxKickLocked` has **no fallback to the tmux path**: an agent opted in without a resolvable image fails every kick outright rather than degrading. Making the global flag sufficient would therefore convert working agents into permanently failing ones on any hive that set it without an image. Changing that default is a fleet-affecting decision that wants measurement, not a code-reading; the warning above is the part that is safe today. **Cline's provider capability is separate from the Git workspace.** Its bounded @@ -201,6 +211,11 @@ agents: **What the operator sees.** The kick shows on the agent card like any sandboxed kick, with the runtime named in the output buffer and the audit line. The pod log is the transcript — hive collects it before the Job is deleted and archives it as the kick log, so "full log" on the card and the kick history keep working after the Job is gone — the container exit code is the sandbox exit code, and the kick timeout deletes a Job that is still running. A Job that **failed** is left in place until `ttl_seconds` (default one hour) so `kubectl describe job` and `kubectl logs` still work; a completed Job is deleted at once. Jobs carry the label `app.kubernetes.io/managed-by=hive-kubejob`. +**Orphaned Jobs fail and are reaped (#18525).** The kick timeout lives in hive's memory, so a hive restart mid-kick used to leave its Job with no owner, and `ttl_seconds` only applies after a Job finishes; one Job sat `Pending` for 11 days. Two backstops now cover that: + +- Every Job carries `activeDeadlineSeconds`: the kick timeout (`timeout_s`, default 45 minutes) plus a fixed 15-minute margin (`kubejob.DeadlineMargin`), or the run's own context deadline plus the margin when that is later. Hive's own timeout always fires first; the deadline makes the cluster fail a Job that is still `Pending`, `Unschedulable` or running once hive has lost track of it. +- At boot, when any agent (or `agent_sandbox`) selects the Job runtime, hive lists the Jobs labelled `app.kubernetes.io/managed-by=hive-kubejob` in its namespace and deletes each unfinished agent-sandbox Job (`hive--sandbox` kicks and `hive--verify` preflights) that no live kick owns and that is older than the longest Job-runtime kick timeout, plus any that never got a ready pod and are past the deadline. Each one is logged (`audit: reaped orphaned sandbox job`) and recorded in the audit log as `sandbox_job_reaped`. Finished Jobs are left to `ttl_seconds`; Jobs without the label, and contained runs (`crux-run-`), are never touched. The sweep uses the same `list` and `delete` grants on `batch/jobs` as the launcher. + **The terminal pane.** A job-runtime agent has no tmux session in the hive pod, so the dashboard's ▶ terminal does not attach ttyd for it. Instead it opens `/terminal/job/`, a read-only viewer that streams the newest Job pod's log through the same in-cluster client and RBAC the launcher already uses ([#14798](https://github.com/hivecommons/hive/issues/14798)): it follows the pod while the kick runs (waiting while the pod is `Pending`, e.g. for an accelerator node), prints the container exit code when it ends, and between kicks, when the completed Job has already been deleted, shows the last run's persisted transcript (`/.hive/sandbox-transcript.log`, which now ends with an `exit code: N` line) or a clear "no sandbox run yet" state. The pane is observation only; there is no input path into the Job. The same terminal handoff and role check as the tmux pane apply. **RBAC.** The spoke's ServiceAccount needs a Role for `batch/jobs` (create, get, list, delete) and `pods` plus `pods/log` (get, list); the terminal viewer needs nothing beyond that same `pods/log` grant. The standalone kustomize base ships it as `sandbox-job-rbac.yaml`, and the hub provisioner grants it to every hosted spoke as `hive-sandbox-jobs`. diff --git a/src/pkg/config/crux_aliases.go b/src/pkg/config/crux_aliases.go index 96b1271e6a..c7561e6c40 100644 --- a/src/pkg/config/crux_aliases.go +++ b/src/pkg/config/crux_aliases.go @@ -36,6 +36,7 @@ type ( ReviewModelsConfig = cruxconfig.ReviewModelsConfig RotationConfig = cruxconfig.RotationConfig SandboxJobConfig = cruxconfig.SandboxJobConfig + SandboxPublishConfig = cruxconfig.SandboxPublishConfig SandboxJobResources = cruxconfig.SandboxJobResources SandboxJobToleration = cruxconfig.SandboxJobToleration SandboxJobVolume = cruxconfig.SandboxJobVolume @@ -168,6 +169,7 @@ var ( var ( APIKeySHA256 = cruxconfig.APIKeySHA256 + CopiedSandboxBlock = cruxconfig.CopiedSandboxBlock AllowSecretFileRoot = cruxconfig.AllowSecretFileRoot DropDanglingModelFlag = cruxconfig.DropDanglingModelFlag IsCLIBackend = cruxconfig.IsCLIBackend diff --git a/src/pkg/dashboard/api_agent_config.go b/src/pkg/dashboard/api_agent_config.go index 17ab41cf89..3db1f074d1 100644 --- a/src/pkg/dashboard/api_agent_config.go +++ b/src/pkg/dashboard/api_agent_config.go @@ -187,7 +187,12 @@ func (s *Server) handleAgentConfigGet(w http.ResponseWriter, r *http.Request) { "sandboxSubstrate": agentSandboxSubstrateOrEmpty(agentCfg), "sandboxEnabled": agentCfg.Sandbox != nil && agentCfg.Sandbox.Enabled != nil && *agentCfg.Sandbox.Enabled, "sandboxEffective": agentCfg.SandboxEnabled(s.deps.Config.AgentSandbox), - "replicas": agentCfg.Replicas, + // What turning the sandbox on would resolve to (#18525): whether + // the agent's own fields plus agent_sandbox defaults are a + // complete block, and which other agents a block can be copied from. + "sandboxDefaultsProblem": agentCfg.SandboxBlockProblem(s.deps.Config.AgentSandbox), + "sandboxCopySources": s.sandboxCopySources(name), + "replicas": agentCfg.Replicas, }, "cadences": cadences, "models": models, @@ -655,11 +660,12 @@ func (s *Server) handleAgentConfigGeneral(w http.ResponseWriter, r *http.Request } if v, ok := body["sandboxEnabled"]; ok { if b, ok := v.(bool); ok { - if agentCfg.Sandbox == nil { - agentCfg.Sandbox = &config.AgentSandboxOverride{} + block, err := s.sandboxBlockForToggle(name, agentCfg, b, sandboxCopyFromBody(body)) + if err != nil { + jsonError(w, err.Error(), http.StatusBadRequest) + return } - bb := b - agentCfg.Sandbox.Enabled = &bb + agentCfg.Sandbox = block } } if v, ok := body["sandboxSubstrate"]; ok { diff --git a/src/pkg/dashboard/api_agent_sandbox_toggle.go b/src/pkg/dashboard/api_agent_sandbox_toggle.go new file mode 100644 index 0000000000..30e5491bce --- /dev/null +++ b/src/pkg/dashboard/api_agent_sandbox_toggle.go @@ -0,0 +1,101 @@ +package dashboard + +import ( + "fmt" + "sort" + "strings" + + "github.com/hivecommons/hive/pkg/config" +) + +// sandboxCopyFromKeys are the General-tab body keys naming the agent whose +// sandbox block a toggle-on copies: the dashboard's camelCase spelling and +// the documented copy_from. +var sandboxCopyFromKeys = []string{"sandboxCopyFrom", "copy_from"} + +// sandboxCopyFromBody returns the copy-from agent a General-tab body names, +// or "" for none. +func sandboxCopyFromBody(body map[string]interface{}) string { + for _, k := range sandboxCopyFromKeys { + if v, ok := body[k].(string); ok { + if name := strings.TrimSpace(sanitizeString(v)); name != "" { + return name + } + } + } + return "" +} + +// sandboxCopySources lists the other agents whose sandbox block is complete +// (their own fields plus agent_sandbox defaults resolve an image and a +// usable runtime), sorted, for the General tab's copy-from picker. +func (s *Server) sandboxCopySources(exclude string) []string { + out := []string{} + for name, a := range s.deps.Config.Agents { + if name == exclude || a.Sandbox == nil { + continue + } + if a.SandboxBlockProblem(s.deps.Config.AgentSandbox) == "" { + out = append(out, name) + } + } + sort.Strings(out) + return out +} + +// sandboxBlockForToggle returns the sandbox block the General tab's "Run in +// sandbox" toggle writes (hivecommons/hive#18525). Writing enabled: true +// alone produced an agent whose every kick failed with "sandbox image is not +// configured" when agent_sandbox had no image, so turning the toggle on now +// writes a complete block or refuses: +// +// - with copyFrom, the named agent's complete block is copied, minus its +// publish preflight (verify commands are per-repository work); the +// receiving agent keeps its own publish block; +// - otherwise the agent's own fields, then agent_sandbox defaults, must +// resolve a complete block, and the image and runtime are pinned into it. +// +// Turning it off keeps the block and only sets enabled: false. +func (s *Server) sandboxBlockForToggle(name string, agentCfg config.AgentConfig, enable bool, copyFrom string) (*config.AgentSandboxOverride, error) { + global := s.deps.Config.AgentSandbox + if !enable { + block := &config.AgentSandboxOverride{} + if agentCfg.Sandbox != nil { + cp := *agentCfg.Sandbox + block = &cp + } + off := false + block.Enabled = &off + return block, nil + } + if copyFrom != "" { + if copyFrom == name { + return nil, fmt.Errorf("cannot copy a sandbox block from %q into itself", name) + } + src, ok := s.deps.Config.Agents[copyFrom] + if !ok { + return nil, fmt.Errorf("copy_from: agent %q not found", copyFrom) + } + if src.Sandbox == nil { + return nil, fmt.Errorf("copy_from: agent %q has no sandbox block to copy", copyFrom) + } + if problem := src.SandboxBlockProblem(global); problem != "" { + return nil, fmt.Errorf("copy_from: agent %q has no complete sandbox block: %s", copyFrom, problem) + } + var keep *config.SandboxPublishConfig + if agentCfg.Sandbox != nil { + keep = agentCfg.Sandbox.Publish + } + return config.CopiedSandboxBlock(&src, global, keep), nil + } + if problem := agentCfg.SandboxBlockProblem(global); problem != "" { + msg := fmt.Sprintf("cannot run %s in a sandbox: %s. Set the agent_sandbox defaults (Governor → Security), or pass copy_from naming an agent with a complete sandbox block", name, problem) + if sources := s.sandboxCopySources(name); len(sources) > 0 { + msg += " (candidates: " + strings.Join(sources, ", ") + ")" + } else { + msg += " (no agent has one yet)" + } + return nil, fmt.Errorf("%s", msg) + } + return agentCfg.PrefilledSandboxBlock(global), nil +} diff --git a/src/pkg/dashboard/api_agent_sandbox_toggle_test.go b/src/pkg/dashboard/api_agent_sandbox_toggle_test.go new file mode 100644 index 0000000000..4afdf6223a --- /dev/null +++ b/src/pkg/dashboard/api_agent_sandbox_toggle_test.go @@ -0,0 +1,148 @@ +package dashboard + +import ( + "net/http" + "strings" + "testing" + + "github.com/hivecommons/hive/pkg/config" +) + +// hivecommons/hive#18525: the "Run in sandbox" toggle used to write +// enabled: true alone, which wedged an agent when agent_sandbox had no image. + +func TestSandboxToggleRefusedWithoutDefaults(t *testing.T) { + s, deps := apiServer(t) + rec := doPut(s, "/api/config/agent/scanner/general", map[string]any{"sandboxEnabled": true}) + if rec.Code != http.StatusBadRequest { + t.Fatalf("PUT = %d, want 400: %s", rec.Code, rec.Body.String()) + } + body := rec.Body.String() + for _, want := range []string{"no sandbox image", "copy_from", "no agent has one yet"} { + if !strings.Contains(body, want) { + t.Errorf("refusal %q does not mention %q", body, want) + } + } + if got := deps.Config.Agents["scanner"].Sandbox; got != nil { + t.Fatalf("a refused toggle must not write a block: %#v", got) + } +} + +func TestSandboxToggleRefusalListsCopySources(t *testing.T) { + s, deps := apiServer(t) + deps.Config.Agents["spyre"] = config.AgentConfig{Backend: "claude", Sandbox: &config.AgentSandboxOverride{Image: "spyre-img"}} + rec := doPut(s, "/api/config/agent/scanner/general", map[string]any{"sandboxEnabled": true}) + if rec.Code != http.StatusBadRequest || !strings.Contains(rec.Body.String(), "candidates: spyre") { + t.Fatalf("PUT = %d %s, want 400 naming spyre", rec.Code, rec.Body.String()) + } +} + +func TestSandboxTogglePrefillsFromDefaults(t *testing.T) { + s, deps := apiServer(t) + deps.Config.AgentSandbox = config.AgentSandboxConfig{Enabled: true, Image: "registry.example/sandbox:1", Runtime: "job", Job: &config.SandboxJobConfig{WorkspaceClaim: "hive-data"}} + rec := doPut(s, "/api/config/agent/scanner/general", map[string]any{"sandboxEnabled": true}) + if rec.Code != http.StatusOK { + t.Fatalf("PUT = %d: %s", rec.Code, rec.Body.String()) + } + got := deps.Config.Agents["scanner"].Sandbox + if got == nil || got.Enabled == nil || !*got.Enabled { + t.Fatalf("not enabled: %#v", got) + } + if got.Image != "registry.example/sandbox:1" || got.Runtime != "job" { + t.Fatalf("block not prefilled from agent_sandbox: %#v", got) + } +} + +func TestSandboxToggleCopiesFromAnotherAgent(t *testing.T) { + s, deps := apiServer(t) + deps.Config.Agents["spyre"] = config.AgentConfig{Backend: "claude", Sandbox: &config.AgentSandboxOverride{ + Image: "spyre-img", + Runtime: "job", + Job: &config.SandboxJobConfig{WorkspaceClaim: "hive-data", NodeSelector: map[string]string{"accel": "spyre"}}, + Publish: &config.SandboxPublishConfig{Verify: []string{"pytest tests/spyre"}}, + }} + sc := deps.Config.Agents["scanner"] + sc.Sandbox = &config.AgentSandboxOverride{Publish: &config.SandboxPublishConfig{Verify: []string{"make lint"}}} + deps.Config.Agents["scanner"] = sc + + rec := doPut(s, "/api/config/agent/scanner/general", map[string]any{"sandboxEnabled": true, "copy_from": "spyre"}) + if rec.Code != http.StatusOK { + t.Fatalf("PUT = %d: %s", rec.Code, rec.Body.String()) + } + got := deps.Config.Agents["scanner"].Sandbox + if got == nil || got.Image != "spyre-img" || got.Runtime != "job" || got.Job == nil || got.Job.NodeSelector["accel"] != "spyre" { + t.Fatalf("block not copied: %#v", got) + } + if got.Publish == nil || len(got.Publish.Verify) != 1 || got.Publish.Verify[0] != "make lint" { + t.Fatalf("publish.verify must stay the receiver's own: %#v", got.Publish) + } + if got.Enabled == nil || !*got.Enabled { + t.Fatal("copied block not enabled") + } +} + +func TestSandboxToggleCopyFromRefusals(t *testing.T) { + s, deps := apiServer(t) + deps.Config.Agents["bare"] = config.AgentConfig{Backend: "claude"} + deps.Config.Agents["partial"] = config.AgentConfig{Backend: "claude", Sandbox: &config.AgentSandboxOverride{Runtime: "job", Image: "img"}} + for _, tt := range []struct{ from, want string }{ + {"ghost", "not found"}, + {"scanner", "into itself"}, + {"bare", "no sandbox block"}, + {"partial", "workspace_claim"}, + } { + rec := doPut(s, "/api/config/agent/scanner/general", map[string]any{"sandboxEnabled": true, "sandboxCopyFrom": tt.from}) + if rec.Code != http.StatusBadRequest || !strings.Contains(rec.Body.String(), tt.want) { + t.Errorf("copy from %s: %d %s, want 400 mentioning %q", tt.from, rec.Code, rec.Body.String(), tt.want) + } + } +} + +func TestSandboxToggleOffKeepsBlock(t *testing.T) { + s, deps := apiServer(t) + on := true + sc := deps.Config.Agents["scanner"] + sc.Sandbox = &config.AgentSandboxOverride{Enabled: &on, Image: "img", Runtime: "podman"} + deps.Config.Agents["scanner"] = sc + rec := doPut(s, "/api/config/agent/scanner/general", map[string]any{"sandboxEnabled": false}) + if rec.Code != http.StatusOK { + t.Fatalf("PUT = %d: %s", rec.Code, rec.Body.String()) + } + got := deps.Config.Agents["scanner"].Sandbox + if got == nil || got.Enabled == nil || *got.Enabled || got.Image != "img" || got.Runtime != "podman" { + t.Fatalf("toggle off must keep the block with enabled: false: %#v", got) + } + + // Off on an agent with no block writes a minimal disabled one. + deps.Config.Agents["fresh"] = config.AgentConfig{Backend: "claude"} + rec = doPut(s, "/api/config/agent/fresh/general", map[string]any{"sandboxEnabled": false}) + if rec.Code != http.StatusOK { + t.Fatalf("PUT fresh = %d: %s", rec.Code, rec.Body.String()) + } + if got := deps.Config.Agents["fresh"].Sandbox; got == nil || got.Enabled == nil || *got.Enabled { + t.Fatalf("fresh block = %#v", got) + } +} + +func TestAgentConfigGetReportsSandboxResolution(t *testing.T) { + s, deps := apiServer(t) + deps.Config.Agents["spyre"] = config.AgentConfig{Backend: "claude", Sandbox: &config.AgentSandboxOverride{Image: "spyre-img"}} + body := decodeJSON(t, doGet(s, "/api/config/agent/scanner")) + general, _ := body["general"].(map[string]any) + if p, _ := general["sandboxDefaultsProblem"].(string); !strings.Contains(p, "no sandbox image") { + t.Fatalf("sandboxDefaultsProblem = %v", general["sandboxDefaultsProblem"]) + } + sources, _ := general["sandboxCopySources"].([]any) + if len(sources) != 1 || sources[0] != "spyre" { + t.Fatalf("sandboxCopySources = %v, want [spyre]", general["sandboxCopySources"]) + } +} + +func TestSandboxCopyFromBody(t *testing.T) { + if got := sandboxCopyFromBody(map[string]interface{}{"copy_from": " spyre "}); got != "spyre" { + t.Fatalf("copy_from = %q", got) + } + if got := sandboxCopyFromBody(map[string]interface{}{"sandboxCopyFrom": 3}); got != "" { + t.Fatalf("non-string = %q", got) + } +} diff --git a/src/pkg/dashboard/api_governor_security_test.go b/src/pkg/dashboard/api_governor_security_test.go index 26ab68457f..02721c7809 100644 --- a/src/pkg/dashboard/api_governor_security_test.go +++ b/src/pkg/dashboard/api_governor_security_test.go @@ -148,6 +148,7 @@ func TestGovernorSecurityRejectsInvalidAdvancedValues(t *testing.T) { func TestAgentConfigGeneralPersistsSandboxOptIn(t *testing.T) { s, deps := apiServer(t) + deps.Config.AgentSandbox.Image = "registry.example/sandbox:1" rec := doPut(s, "/api/config/agent/scanner/general", map[string]any{"sandboxEnabled": true}) if rec.Code != http.StatusOK { t.Fatalf("PUT agent general = %d, want 200: %s", rec.Code, rec.Body.String()) diff --git a/src/pkg/dashboard/static-src/js/config-agent.js b/src/pkg/dashboard/static-src/js/config-agent.js index ef85124de9..c811e0ef1f 100644 --- a/src/pkg/dashboard/static-src/js/config-agent.js +++ b/src/pkg/dashboard/static-src/js/config-agent.js @@ -121,8 +121,9 @@
- Run in sandbox ${settingHelpMark('sandbox-isolation', '', 'Run in sandbox', `Opt this agent into the credential-free sandbox runner. Requires the hive-wide sandbox gate in Governor → Security to be enabled.`)}${g.sandboxEffective ? ' ' + statusBadgeHTML('ready', 'sandboxed', '', 'done') : ''} + Run in sandbox ${settingHelpMark('sandbox-isolation', '', 'Run in sandbox', `Opt this agent into the credential-free sandbox runner. Requires the hive-wide sandbox gate in Governor → Security to be enabled. Turning it on writes a complete sandbox block: the agent_sandbox defaults, or a copy of another sandboxed agent's block (its publish.verify commands excluded).`)}${g.sandboxEffective ? ' ' + statusBadgeHTML('ready', 'sandboxed', '', 'done') : ''}
+ ${sandboxSourcePickerHTML(g)}
Substrate sandbox overrides ${substrateSandboxFields('general', g.sandboxSubstrate)}
@@ -1452,3 +1453,37 @@ else note = 'No hive default set, so agents that leave Explain Mode unset do not explain.'; return 'In force now: ' + esc(effective) + '. ' + esc(note) + ' Applies from the next kick — no restart needed.'; } + + + // Run-in-sandbox source picker (hivecommons/hive#18525). Shown while the + // toggle is being turned on: the server refuses a toggle that would leave + // the agent with no sandbox image or runtime, so the operator picks where + // the block comes from — the agent_sandbox defaults when they are + // complete, or another agent that already has a complete block. + function sandboxSourcePickerHTML(g) { + const sources = Array.isArray(g.sandboxCopySources) ? g.sandboxCopySources : []; + const defaultsOk = !g.sandboxDefaultsProblem; + const hidden = g.sandboxEnabled ? ' style="display:none"' : ''; + let inner; + if (!defaultsOk && sources.length === 0) { + inner = ''; + } else { + const opts = (defaultsOk ? '' : '') + + sources.map(n => '').join(''); + inner = '' + + '' + + (defaultsOk ? '' : '
' + esc(g.sandboxDefaultsProblem) + ', so a block must be copied from another agent.
'); + } + return ''; + } + + // Called by toggleConfigSwitch when the Run-in-sandbox toggle flips. + function onSandboxToggle(isOn) { + const wrap = document.getElementById('cfg-agent-sandbox-source-wrap'); + if (!wrap) return; + wrap.style.display = isOn ? '' : 'none'; + const sel = document.getElementById('cfg-agent-sandbox-source'); + if (isOn && sel && sel.value) markDirty('general', 'sandboxCopyFrom', sel.value); + if (!isOn && _configState.dirty.general) delete _configState.dirty.general.sandboxCopyFrom; + } diff --git a/src/pkg/dashboard/static-src/js/config-helpers.js b/src/pkg/dashboard/static-src/js/config-helpers.js index 6ebb10bd3d..d09da706e6 100644 --- a/src/pkg/dashboard/static-src/js/config-helpers.js +++ b/src/pkg/dashboard/static-src/js/config-helpers.js @@ -25,6 +25,7 @@ } markDirty(section, key, isOn); if (section === 'review' && (key === 'fan_out' || key === 'combined_perspectives')) refreshReviewerAgentSummary(); + if (section === 'general' && key === 'sandboxEnabled' && typeof onSandboxToggle === 'function') onSandboxToggle(isOn); } function markDirtyMergeTrain(key, value) { diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html index 98ef17edbd..3af018ed08 100644 --- a/src/pkg/dashboard/static/index.html +++ b/src/pkg/dashboard/static/index.html @@ -65743,8 +65743,9 @@

Hub settings

- Run in sandbox ${settingHelpMark('sandbox-isolation', '', 'Run in sandbox', `Opt this agent into the credential-free sandbox runner. Requires the hive-wide sandbox gate in Governor → Security to be enabled.`)}${g.sandboxEffective ? ' ' + statusBadgeHTML('ready', 'sandboxed', '', 'done') : ''} + Run in sandbox ${settingHelpMark('sandbox-isolation', '', 'Run in sandbox', `Opt this agent into the credential-free sandbox runner. Requires the hive-wide sandbox gate in Governor → Security to be enabled. Turning it on writes a complete sandbox block: the agent_sandbox defaults, or a copy of another sandboxed agent's block (its publish.verify commands excluded).`)}${g.sandboxEffective ? ' ' + statusBadgeHTML('ready', 'sandboxed', '', 'done') : ''}
+ ${sandboxSourcePickerHTML(g)}
Substrate sandbox overrides ${substrateSandboxFields('general', g.sandboxSubstrate)}
@@ -67074,6 +67075,40 @@

Hub settings

else note = 'No hive default set, so agents that leave Explain Mode unset do not explain.'; return 'In force now: ' + esc(effective) + '. ' + esc(note) + ' Applies from the next kick — no restart needed.'; } + + + // Run-in-sandbox source picker (hivecommons/hive#18525). Shown while the + // toggle is being turned on: the server refuses a toggle that would leave + // the agent with no sandbox image or runtime, so the operator picks where + // the block comes from — the agent_sandbox defaults when they are + // complete, or another agent that already has a complete block. + function sandboxSourcePickerHTML(g) { + const sources = Array.isArray(g.sandboxCopySources) ? g.sandboxCopySources : []; + const defaultsOk = !g.sandboxDefaultsProblem; + const hidden = g.sandboxEnabled ? ' style="display:none"' : ''; + let inner; + if (!defaultsOk && sources.length === 0) { + inner = ''; + } else { + const opts = (defaultsOk ? '' : '') + + sources.map(n => '').join(''); + inner = '' + + '' + + (defaultsOk ? '' : '
' + esc(g.sandboxDefaultsProblem) + ', so a block must be copied from another agent.
'); + } + return ''; + } + + // Called by toggleConfigSwitch when the Run-in-sandbox toggle flips. + function onSandboxToggle(isOn) { + const wrap = document.getElementById('cfg-agent-sandbox-source-wrap'); + if (!wrap) return; + wrap.style.display = isOn ? '' : 'none'; + const sel = document.getElementById('cfg-agent-sandbox-source'); + if (isOn && sel && sel.value) markDirty('general', 'sandboxCopyFrom', sel.value); + if (!isOn && _configState.dirty.general) delete _configState.dirty.general.sandboxCopyFrom; + } // ── Governor Tab Renderers ── function renderGovGeneral() { const currentId = (window._lastStatus || {}).hiveId || ''; @@ -77026,6 +77061,7 @@

${model.editIndex === null ? 'Add Event trigger' : 'Edit Event trigger'}

Date: Mon, 16 Nov 2026 20:13:37 -0500 Subject: [PATCH 2/2] =?UTF-8?q?=F0=9F=8C=B1=20Move=20UpdateConfig=20next?= =?UTF-8?q?=20to=20the=20sandbox-mode=20hook=20to=20keep=20manager.go=20un?= =?UTF-8?q?der=20its=20size=20ratchet?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Andy Anderson --- src/crux/agent/manager.go | 22 ---------------------- src/crux/agent/manager_sandbox_mode.go | 24 ++++++++++++++++++++++++ 2 files changed, 24 insertions(+), 22 deletions(-) diff --git a/src/crux/agent/manager.go b/src/crux/agent/manager.go index 5a486eea94..027927bee7 100644 --- a/src/crux/agent/manager.go +++ b/src/crux/agent/manager.go @@ -1469,28 +1469,6 @@ func (m *Manager) removeAgentsBelowACMMGateLocked(level int) { } } -// UpdateConfig updates the stored config for a running agent process so that -// status builders (which read from AgentProcess.Config) reflect changes made -// via the config dialog (which writes to the global Config.Agents map). -func (m *Manager) UpdateConfig(name string, cfg config.AgentConfig) error { - m.mu.Lock() - defer m.mu.Unlock() - - agent, ok := m.agents[name] - if !ok { - return fmt.Errorf("agent %s not found", name) - } - - wasSandboxed := m.agentSandboxEnabledLocked(agent) - agent.clearAgentSpecPromptIfSpecRemoved(cfg) - agent.Config = cfg - agent.baseConfig = cfg - if !wasSandboxed && m.agentSandboxEnabledLocked(agent) { - m.enterSandboxModeLocked(agent) - } - return nil -} - // ReconcileAgents makes the manager's name-keyed process table match the // enabled config set. New agents are added, existing agents get fresh config, // and removed agents have only their own hive- tmux session retired. diff --git a/src/crux/agent/manager_sandbox_mode.go b/src/crux/agent/manager_sandbox_mode.go index efe93b0f69..5c74734568 100644 --- a/src/crux/agent/manager_sandbox_mode.go +++ b/src/crux/agent/manager_sandbox_mode.go @@ -4,6 +4,8 @@ import ( "fmt" "strings" "time" + + "github.com/hivecommons/hive/crux/config" ) // sandboxModeArchiveReason labels the kick-log archive written when an agent's @@ -57,3 +59,25 @@ func (m *Manager) enterSandboxModeLocked(agent *AgentProcess) { "name", agent.Name, "session", agent.tmuxSession, "reaped_cli", reaped) m.auditSandbox(agent.Name, "sandbox_mode_entered", "interactive tmux session "+agent.tmuxSession+" stopped") } + +// UpdateConfig updates the stored config for a running agent process so that +// status builders (which read from AgentProcess.Config) reflect changes made +// via the config dialog (which writes to the global Config.Agents map). +func (m *Manager) UpdateConfig(name string, cfg config.AgentConfig) error { + m.mu.Lock() + defer m.mu.Unlock() + + agent, ok := m.agents[name] + if !ok { + return fmt.Errorf("agent %s not found", name) + } + + wasSandboxed := m.agentSandboxEnabledLocked(agent) + agent.clearAgentSpecPromptIfSpecRemoved(cfg) + agent.Config = cfg + agent.baseConfig = cfg + if !wasSandboxed && m.agentSandboxEnabledLocked(agent) { + m.enterSandboxModeLocked(agent) + } + return nil +}