Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/fixed-18525-sandbox-toggle-and-job-reap.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- The dashboard's per-agent **Run in sandbox** toggle no longer wedges an agent ([#18525](https://github.com/hivecommons/hive/issues/18525)). Turning it on used to write `sandbox.enabled: true` alone, so with no `agent_sandbox` image every kick failed with "sandbox image is not configured". It now writes a complete block, prefilled from the `agent_sandbox` defaults or copied (`copy_from`) from another agent that has one, minus that agent's `publish.verify`, and refuses with a 400 that names what is missing when neither exists. The General tab shows a picker for the source when the toggle is turned on, with an inline error when there is none. Turning it off keeps the block and sets `enabled: false`. Switching an agent into sandbox mode now stops its tmux session, and a refused kick names the real state ("agent has a live interactive session; sandbox kicks are blocked until it stops") instead of "sandbox execution already running". Every sandbox Kubernetes Job now carries `activeDeadlineSeconds` (the kick timeout plus a 15-minute margin), and at boot hive reaps unfinished `app.kubernetes.io/managed-by=hive-kubejob` sandbox Jobs that no live kick owns, logging and auditing each one (`sandbox_job_reaped`). Before this, one Job sat Pending for 11 days.
3 changes: 3 additions & 0 deletions src/cmd/hive/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -2663,6 +2663,9 @@ func (b *boot) bootDashboardWith(deps bootDashboardDeps) {
}
}
})
// Reap sandbox Jobs the previous process lost track of, now that the
// audit log can record each one (hivecommons/hive#18525).
startSandboxJobBootSweep(b.ctx, b.cfg, b.agentMgr, b.dashSrv.AuditLog, b.logger)

// Persist per-user dashboard sessions on the PVC (/data) so direct-route
// users aren't logged out by pod restarts. NOTE: use /data explicitly, NOT
Expand Down
129 changes: 129 additions & 0 deletions src/cmd/hive/sandbox_job_sweep.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,129 @@
package main

import (
"context"
"fmt"
"log/slog"
"strings"
"time"

"github.com/hivecommons/hive/crux"
cruxconfig "github.com/hivecommons/hive/crux/config"
"github.com/hivecommons/hive/crux/kubejob"
"github.com/hivecommons/hive/crux/sandbox"
"github.com/hivecommons/hive/pkg/config"
)

// sandboxJobSweepTimeout bounds the boot sweep's API calls as a whole.
const sandboxJobSweepTimeout = 2 * time.Minute

// sandboxJobSweepAuditAction is the audit-log action of one reaped Job.
const sandboxJobSweepAuditAction = "sandbox_job_reaped"

// sandboxJobSweepAuditUser is who the audit log names for a reap: hive
// itself, at boot, not an operator.
const sandboxJobSweepAuditUser = "hive"

// sandboxJobSweeper is the part of kubejob.Launcher the boot sweep uses.
type sandboxJobSweeper interface {
SweepOrphans(ctx context.Context, opts kubejob.SweepOptions) ([]kubejob.SweptJob, error)
}

// newSandboxJobSweeper builds the boot sweep's sweeper: the in-cluster Job
// launcher. Tests replace it so a test run on a cluster-hosted CI runner can
// never list or delete real Jobs.
var newSandboxJobSweeper = func() sandboxJobSweeper { return &kubejob.Launcher{} }

// sandboxJobRuntimeConfigured reports whether any agent (or the global
// default) selects the Job sandbox runtime, enabled or not: a Job orphaned
// before an agent was switched off still needs reaping.
func sandboxJobRuntimeConfigured(cfg *config.Config) bool {
if strings.TrimSpace(cfg.AgentSandbox.Runtime) == cruxconfig.SandboxRuntimeJob {
return true
}
for _, a := range cfg.Agents {
if a.SandboxRuntime(cfg.AgentSandbox) == cruxconfig.SandboxRuntimeJob {
return true
}
}
return false
}

// sandboxJobSweepMaxAge is the longest kick timeout any Job-runtime agent
// has: a Job younger than that may still be a kick that has not timed out.
func sandboxJobSweepMaxAge(cfg *config.Config) time.Duration {
maxAge := kubejob.DefaultKickTimeout
for _, a := range cfg.Agents {
if a.SandboxRuntime(cfg.AgentSandbox) != cruxconfig.SandboxRuntimeJob {
continue
}
if t := time.Duration(a.SandboxTimeoutS(cfg.AgentSandbox)) * time.Second; t > maxAge {
maxAge = t
}
}
return maxAge
}

// sandboxJobLive reports whether a Job's agent label belongs to an agent the
// runtime says is mid-kick. It errs towards live: the label of every Job of
// a running agent's slug counts, whichever launch (kick or verify) made it.
func sandboxJobLive(agentNames []string, agentMgr crux.Runtime) func(string) bool {
return func(label string) bool {
if agentMgr == nil {
return false
}
for _, name := range agentNames {
if !strings.HasPrefix(label, "hive-"+sandbox.AgentSlug(name)+"-") {
continue
}
if p, err := agentMgr.GetStatusFast(name); err == nil && p != nil && p.State == crux.StateRunning {
return true
}
}
return false
}
}

// startSandboxJobBootSweep runs the boot sweep in the background when any
// agent selects the Job runtime. The config is read here, synchronously, so
// the goroutine never touches the live config a reload may be rewriting.
func startSandboxJobBootSweep(ctx context.Context, cfg *config.Config, agentMgr crux.Runtime, audit func(user, action, detail, agent string), logger *slog.Logger) bool {
if cfg == nil || !sandboxJobRuntimeConfigured(cfg) {
return false
}
maxAge := sandboxJobSweepMaxAge(cfg)
names := make([]string, 0, len(cfg.Agents))
for name := range cfg.Agents {
names = append(names, name)
}
go runSandboxJobBootSweep(ctx, maxAge, names, agentMgr, newSandboxJobSweeper(), audit, logger)
return true
}

// runSandboxJobBootSweep reaps sandbox Jobs the previous hive process lost
// track of (hivecommons/hive#18525). A kick's timeout lives in memory, so a
// restart mid-kick leaves its Job with no owner; one sat Pending for 11
// days. Each reaped Job is logged and audited. Jobs without the
// app.kubernetes.io/managed-by=hive-kubejob label are never touched.
func runSandboxJobBootSweep(ctx context.Context, maxAge time.Duration, agentNames []string, agentMgr crux.Runtime, sweeper sandboxJobSweeper, audit func(user, action, detail, agent string), logger *slog.Logger) int {
if sweeper == nil {
sweeper = newSandboxJobSweeper()
}
ctx, cancel := context.WithTimeout(ctx, sandboxJobSweepTimeout)
defer cancel()
swept, err := sweeper.SweepOrphans(ctx, kubejob.SweepOptions{MaxAge: maxAge, Live: sandboxJobLive(agentNames, agentMgr)})
for _, j := range swept {
logger.Warn("audit: reaped orphaned sandbox job",
"job", j.Name, "agent_label", j.Agent, "age", j.Age.Round(time.Second).String(), "reason", j.Reason)
if audit != nil {
audit(sandboxJobSweepAuditUser, sandboxJobSweepAuditAction,
fmt.Sprintf("job=%s age=%s reason=%s", j.Name, j.Age.Round(time.Second), j.Reason), j.Agent)
}
}
if err != nil {
logger.Warn("sandbox job boot sweep incomplete", "error", err, "reaped", len(swept))
} else {
logger.Info("sandbox job boot sweep done", "reaped", len(swept), "max_age", maxAge.String())
}
return len(swept)
}
150 changes: 150 additions & 0 deletions src/cmd/hive/sandbox_job_sweep_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,150 @@
package main

import (
"bytes"
"context"
"errors"
"io"
"log/slog"
"strings"
"sync"
"testing"
"time"

"github.com/hivecommons/hive/crux"
"github.com/hivecommons/hive/crux/cruxtest"
"github.com/hivecommons/hive/crux/kubejob"
"github.com/hivecommons/hive/pkg/config"
)

type fakeJobSweeper struct {
opts kubejob.SweepOptions
calls int
swept []kubejob.SweptJob
err error
}

func (f *fakeJobSweeper) SweepOrphans(_ context.Context, opts kubejob.SweepOptions) ([]kubejob.SweptJob, error) {
f.calls++
f.opts = opts
return f.swept, f.err
}

func jobSweepConfig() *config.Config {
on := true
return &config.Config{AgentRuntimeConfig: config.AgentRuntimeConfig{
AgentSandbox: config.AgentSandboxConfig{Enabled: true, Image: "img"},
Agents: map[string]config.AgentConfig{
"reviewer": {Backend: "claude", Sandbox: &config.AgentSandboxOverride{Enabled: &on, Runtime: "job", TimeoutS: 7200}},
"scanner": {Backend: "claude"},
},
}}
}

func TestSandboxJobBootSweepAuditsEachReap(t *testing.T) {
var logBuf bytes.Buffer
logger := slog.New(slog.NewTextHandler(&logBuf, nil))
sw := &fakeJobSweeper{swept: []kubejob.SweptJob{{Name: "hive-kick-reviewer-1", Agent: "hive-reviewer-sandbox", Age: 11 * 24 * time.Hour, Reason: kubejob.SweepReasonPendingPastDeadline}}}
var audits []string
audit := func(user, action, detail, agent string) {
audits = append(audits, user+"|"+action+"|"+agent+"|"+detail)
}
cfg := jobSweepConfig()
n := runSandboxJobBootSweep(context.Background(), sandboxJobSweepMaxAge(cfg), []string{"reviewer", "scanner"}, nil, sw, audit, logger)
if n != 1 || sw.calls != 1 {
t.Fatalf("reaped = %d, calls = %d", n, sw.calls)
}
if sw.opts.MaxAge != 2*time.Hour {
t.Fatalf("MaxAge = %s, want the longest Job-runtime kick timeout", sw.opts.MaxAge)
}
if len(audits) != 1 || !strings.Contains(audits[0], "hive|sandbox_job_reaped|hive-reviewer-sandbox|job=hive-kick-reviewer-1") {
t.Fatalf("audits = %v", audits)
}
if !strings.Contains(logBuf.String(), "reaped orphaned sandbox job") {
t.Fatalf("reap not logged: %s", logBuf.String())
}
if sw.opts.Live == nil || sw.opts.Live("hive-reviewer-sandbox") {
t.Fatal("with no runtime, no Job is live")
}
}

func TestSandboxJobBootSweepStartsOnlyWithJobRuntime(t *testing.T) {
quiet := slog.New(slog.NewTextHandler(io.Discard, nil))
cfg := &config.Config{}
cfg.Agents = map[string]config.AgentConfig{"scanner": {Backend: "claude"}}
if startSandboxJobBootSweep(context.Background(), cfg, nil, nil, quiet) {
t.Fatal("sweep started with no Job runtime configured")
}
if startSandboxJobBootSweep(context.Background(), nil, nil, nil, quiet) {
t.Fatal("nil config must not sweep")
}
sw := &fakeJobSweeper{}
done := make(chan struct{})
prev := newSandboxJobSweeper
newSandboxJobSweeper = func() sandboxJobSweeper { return sw }
t.Cleanup(func() { newSandboxJobSweeper = prev })
audit := func(string, string, string, string) {}
if !startSandboxJobBootSweep(context.Background(), jobSweepConfig(), nil, audit, slog.New(slog.NewTextHandler(&signalWriter{done: done}, nil))) {
t.Fatal("sweep did not start with a Job runtime configured")
}
select {
case <-done:
case <-time.After(10 * time.Second):
t.Fatal("background sweep never finished")
}
if sw.calls != 1 || sw.opts.MaxAge != 2*time.Hour {
t.Fatalf("sweep calls = %d, MaxAge = %s", sw.calls, sw.opts.MaxAge)
}

// A nil sweeper falls back to the factory.
sw2 := &fakeJobSweeper{}
newSandboxJobSweeper = func() sandboxJobSweeper { return sw2 }
runSandboxJobBootSweep(context.Background(), time.Hour, nil, nil, nil, nil, quiet)
if sw2.calls != 1 {
t.Fatal("nil sweeper did not use the factory")
}
}

// signalWriter closes done on the first write: the background sweep's
// completion log line.
type signalWriter struct {
once sync.Once
done chan struct{}
}

func (w *signalWriter) Write(p []byte) (int, error) {
if strings.Contains(string(p), "sandbox job boot sweep") {
w.once.Do(func() { close(w.done) })
}
return len(p), nil
}

func TestSandboxJobBootSweepLogsError(t *testing.T) {
var logBuf bytes.Buffer
sw := &fakeJobSweeper{err: errors.New("jobs.batch is forbidden")}
cfg := &config.Config{}
cfg.AgentSandbox = config.AgentSandboxConfig{Runtime: "job"}
if !sandboxJobRuntimeConfigured(cfg) {
t.Fatal("a global job runtime must count as configured")
}
runSandboxJobBootSweep(context.Background(), sandboxJobSweepMaxAge(cfg), nil, nil, sw, nil, slog.New(slog.NewTextHandler(&logBuf, nil)))
if !strings.Contains(logBuf.String(), "sandbox job boot sweep incomplete") {
t.Fatalf("error not logged: %s", logBuf.String())
}
if sw.opts.MaxAge != kubejob.DefaultKickTimeout {
t.Fatalf("MaxAge = %s, want default", sw.opts.MaxAge)
}
}

func TestSandboxJobLiveChecksRunningAgents(t *testing.T) {
cfg := jobSweepConfig()
rt := cruxtest.NewRuntime(cfg.Agents, slog.New(slog.NewTextHandler(&bytes.Buffer{}, nil)), crux.ProjectContext{})
live := sandboxJobLive([]string{"reviewer", "scanner"}, rt)
// Freshly built agents are stopped, so nothing is live.
if live("hive-reviewer-sandbox") {
t.Fatal("a stopped agent's Job must not count as live")
}
if live("crux-run-abc") {
t.Fatal("a label matching no agent must not count as live")
}
}
27 changes: 9 additions & 18 deletions src/crux/agent/manager.go
Original file line number Diff line number Diff line change
Expand Up @@ -483,6 +483,11 @@ type AgentProcess struct {
// The completion handler then turns the expected cancellation into Idle
// instead of Failed.
sandboxResumeAfterCancel bool
// sandboxKickActive is true while a sandbox kick's goroutine is in
// flight. StateRunning alone cannot tell that apart from a live
// interactive tmux session, and the kick refusal must name which one is
// blocking it (hivecommons/hive#18525). Guarded by m.mu.
sandboxKickActive bool

// awaitingBobKey marks an agent that launchInTmux parked in StateFailed
// for the single, fully-recoverable reason "bob backend with no API key".
Expand Down Expand Up @@ -1464,24 +1469,6 @@ func (m *Manager) removeAgentsBelowACMMGateLocked(level int) {
}
}

// UpdateConfig updates the stored config for a running agent process so that
// status builders (which read from AgentProcess.Config) reflect changes made
// via the config dialog (which writes to the global Config.Agents map).
func (m *Manager) UpdateConfig(name string, cfg config.AgentConfig) error {
m.mu.Lock()
defer m.mu.Unlock()

agent, ok := m.agents[name]
if !ok {
return fmt.Errorf("agent %s not found", name)
}

agent.clearAgentSpecPromptIfSpecRemoved(cfg)
agent.Config = cfg
agent.baseConfig = cfg
return nil
}

// ReconcileAgents makes the manager's name-keyed process table match the
// enabled config set. New agents are added, existing agents get fresh config,
// and removed agents have only their own hive-<name> tmux session retired.
Expand All @@ -1498,9 +1485,13 @@ func (m *Manager) ReconcileAgents(configs map[string]config.AgentConfig) []strin
allowedConfigs[name] = cfg
if existing, ok := m.agents[name]; ok {
delete(m.idToName, existing.ID)
wasSandboxed := m.agentSandboxEnabledLocked(existing)
existing.clearAgentSpecPromptIfSpecRemoved(cfg)
existing.Config = cfg
existing.baseConfig = cfg
if !wasSandboxed && m.agentSandboxEnabledLocked(existing) {
m.enterSandboxModeLocked(existing)
}
if cfg.ID != "" {
existing.ID = cfg.ID
} else {
Expand Down
Loading
Loading