Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/added-shared-agent-session-pool.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- Add an opt-in Hub-coordinated CLI session pool for spokes. Participating Hives keep separate workspaces, UIDs, credentials, and tmux sessions; the Hub leases a global live-session slot and idle sessions are terminated after a turn. Configure `HIVE_SHARED_AGENT_POOL_ENABLED` on spokes and `HIVE_SHARED_AGENT_POOL_MAX_SESSIONS` (optionally `HIVE_SHARED_AGENT_POOL_MAX_PER_HIVE` for a per-tenant cap) on the Hub.
4 changes: 3 additions & 1 deletion src/cmd/hive/eval_cycle_seams.go
Original file line number Diff line number Diff line change
Expand Up @@ -644,7 +644,9 @@ func stoppedAgentStartErrorIsBenign(err error) bool {
return true
}
msg := err.Error()
return strings.Contains(msg, "already running") || strings.Contains(msg, "launch already in progress")
return strings.Contains(msg, "already running") ||
strings.Contains(msg, "launch already in progress") ||
strings.Contains(msg, "shared session pool is busy")
}

// advisoryIngestDeps are the side effects of recording an ioscan canary leak
Expand Down
26 changes: 23 additions & 3 deletions src/cmd/hive/main.go
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
package main

import (
Expand Down Expand Up @@ -2192,10 +2192,25 @@
b.logger.Warn("cleared invalid inference launch_cmd; inference and gateway backends launch through claude", "agent", name)
}
rtCfg := cruxconfig.RuntimeConfigFromEnv()
b.agentMgr, err = deps.newRuntime(rtCfg,
runtimeOptions := []cruxruntime.Option{
cruxruntime.WithDefinitions(b.cfg.EnabledAgents()),
cruxruntime.WithLogger(b.logger),
cruxruntime.WithProject(b.projectCtx))
cruxruntime.WithProject(b.projectCtx),
}
if sharedAgentPoolEnabled() {
hubURL := strings.TrimSpace(os.Getenv("HIVE_HUB_URL"))
if hubURL == "" {
hubURL = b.cfg.Hub.URL
}
pool, poolErr := cruxruntime.NewSharedSessionPoolClient(hubURL, b.cfg.HiveID, hub.SpokeHeartbeatKey())
if poolErr != nil {
b.logger.Error("shared agent pool is enabled but cannot be configured", "error", poolErr)
deps.exit(1)
return
}
runtimeOptions = append(runtimeOptions, cruxruntime.WithSharedSessionPool(pool))
}
b.agentMgr, err = deps.newRuntime(rtCfg, runtimeOptions...)
if err != nil {
b.logger.Error("crux runtime unavailable", "mode", rtCfg.Mode, "error", err)
deps.exit(1)
Expand Down Expand Up @@ -4638,7 +4653,12 @@
// shutdown the ctx-aware stagger returns immediately instead of leaking a
// goroutine parked in a bare time.Sleep.
enabledAgents := b.cfg.EnabledAgents()
startupAgents := startupLaunchNames(enabledAgents, b.onDemandFromPack)
startupAgents := []string{}
if sharedAgentPoolEnabled() {
b.logger.Info("shared agent pool enabled; persistent agents will start on demand")
} else {
startupAgents = startupLaunchNames(enabledAgents, b.onDemandFromPack)
}
b.agentMgr.MarkStartupLaunchQueued(startupAgents)
deps.spawn("agent-launch", func() {
agentIndex := 0
Expand Down
15 changes: 15 additions & 0 deletions src/cmd/hive/shared_pool.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
package main

import (
"os"
"strings"
)

func sharedAgentPoolEnabled() bool {
switch strings.ToLower(strings.TrimSpace(os.Getenv("HIVE_SHARED_AGENT_POOL_ENABLED"))) {
case "1", "true", "yes", "on":
return true
default:
return false
}
}
27 changes: 27 additions & 0 deletions src/cmd/hive/shared_pool_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
package main

import (
"testing"
)

func TestSharedAgentPoolEnabledEnv(t *testing.T) {
for _, tc := range []struct {
val string
want bool
}{
{"", false},
{"0", false},
{"false", false},
{"no", false},
{"1", true},
{"true", true},
{"TRUE", true},
{"yes", true},
{"on", true},
} {
t.Setenv("HIVE_SHARED_AGENT_POOL_ENABLED", tc.val)
if got := sharedAgentPoolEnabled(); got != tc.want {
t.Errorf("sharedAgentPoolEnabled() for %q = %v, want %v", tc.val, got, tc.want)
}
}
}
25 changes: 20 additions & 5 deletions src/crux/agent/kick_async.go
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
package agent

import (
Expand Down Expand Up @@ -223,7 +223,9 @@

func (m *Manager) validatePaneKickLocked(agent *AgentProcess, name string) error {
if agent.State != StateRunning {
return fmt.Errorf("agent %s cannot be kicked: %s", name, notRunningReason(agent))
if m.sharedPool == nil || agent.State != StateStopped || agent.Paused || agent.OperatorStopped {
return fmt.Errorf("agent %s cannot be kicked: %s", name, notRunningReason(agent))
}
}
if remaining := m.providerErrorBackoffRemainingLocked(agent, time.Now()); remaining > 0 {
class, line := agent.ProviderErrorClass, agent.ProviderErrorLine
Expand All @@ -233,7 +235,7 @@
if err := m.restartKickHoldErrLocked(agent, time.Now()); err != nil {
return err
}
if !m.tmuxSessionExistsForAgent(agent) {
if agent.State == StateRunning && !m.tmuxSessionExistsForAgent(agent) {
return fmt.Errorf("tmux session %s not found", agent.tmuxSession)
}
return nil
Expand All @@ -255,9 +257,22 @@
return fmt.Errorf("agent %s not found", name)
}
if agent.State != StateRunning {
reason := notRunningReason(agent)
m.mu.Unlock()
return fmt.Errorf("agent %s cannot be kicked: %s", name, reason)
if m.sharedPool != nil && agent.State == StateStopped && !agent.Paused && !agent.OperatorStopped {
m.mu.Unlock()
if err := m.Start(context.Background(), name); err != nil {
return fmt.Errorf("auto-start stopped agent for kick: %w", err)
}
m.mu.Lock()
agent, ok = m.agents[name]
if !ok || agent.State != StateRunning {
m.mu.Unlock()
return fmt.Errorf("agent %s failed to enter running state", name)
}
} else {
reason := notRunningReason(agent)
m.mu.Unlock()
return fmt.Errorf("agent %s cannot be kicked: %s", name, reason)
}
}
if remaining := m.providerErrorBackoffRemainingLocked(agent, time.Now()); remaining > 0 {
class, line := agent.ProviderErrorClass, agent.ProviderErrorLine
Expand Down
20 changes: 16 additions & 4 deletions src/crux/agent/kick_outcome.go
Original file line number Diff line number Diff line change
Expand Up @@ -241,9 +241,21 @@ func (m *Manager) SetKickOutcomeObserver(fn func(agentName string, outcome KickO
}

func (m *Manager) notifyKickOutcome(agentName string, outcome KickOutcome) {
fn := m.kickOutcomeObserver.Load()
if fn == nil {
return
if fn := m.kickOutcomeObserver.Load(); fn != nil {
go (*fn)(agentName, outcome)
}
if m.sharedPool != nil {
go func() {
m.mu.RLock()
agent := m.agents[agentName]
leaseID := ""
if agent != nil {
leaseID = agent.poolLeaseID
}
m.mu.RUnlock()
if leaseID != "" {
m.evictSharedPoolSessionAfterTurn(agentName, leaseID, outcome)
}
}()
}
go (*fn)(agentName, outcome)
}
102 changes: 76 additions & 26 deletions src/crux/agent/manager.go
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
package agent

import (
Expand Down Expand Up @@ -258,6 +258,8 @@
KickRefusalReason string
LaunchedMode AgentMode
HasLaunched bool
poolLeaseID string
poolLeaseCancel context.CancelFunc
baseConfig config.AgentConfig
agentSpecPrompt string
tmuxSession string
Expand Down Expand Up @@ -575,6 +577,7 @@
copilotAuthTokenRejected bool
claudeAuthToken string
uidMap *UIDMap
sharedPool SharedSessionPool
uidMapPath string
appAuth AppTokenMinter
appNotInstalledWarned sync.Map // agent name -> struct{}: not-installed mint failure already logged
Expand Down Expand Up @@ -789,6 +792,7 @@
type managerOptions struct {
uidMapPath string
uidMapPathSet bool
sharedPool SharedSessionPool
}

// WithUIDMapPath points a manager at a specific persisted UID map. Production
Expand All @@ -800,6 +804,13 @@
}
}

// WithSharedSessionPool opts a manager into a Hub-coordinated live CLI slot.
// Every Hive retains separate sessions and tenant data; the lease only limits
// how many CLI sessions across the participating Hives remain live.
func WithSharedSessionPool(pool SharedSessionPool) ManagerOption {
return func(opts *managerOptions) { opts.sharedPool = pool }
}

func (m *Manager) effectiveUIDMapPath() string {
if m.uidMapPath != "" {
return m.uidMapPath
Expand Down Expand Up @@ -889,6 +900,7 @@
copilotAuthTokenSource: copilotTokenSource,
claudeAuthToken: claudeToken,
uidMap: uidMap,
sharedPool: opts.sharedPool,
uidMapPath: selectedUIDMapPath,
kickLogDir: kickLogDir,
kickLogRetention: kickLogRetention,
Expand Down Expand Up @@ -1148,22 +1160,62 @@
//
// The remaining subprocess sends below are the actual tmux launch boundary;
// Bob's shared-home/file probes have already run lock-free above.
m.mu.RLock()
leaseID := agent.poolLeaseID
hasLeaseWatcher := agent.poolLeaseCancel != nil
m.mu.RUnlock()
if m.sharedPool != nil && leaseID == "" {
var err error
leaseID, err = m.sharedPool.Acquire(ctx, name)
if err != nil {
return fmt.Errorf("acquire shared CLI session slot: %w", err)
}
}
launchSucceeded := false
defer func() {
if leaseID != "" && !launchSucceeded {
m.releaseSharedPoolLease(name, leaseID)
}
}()

m.mu.Lock()
defer m.mu.Unlock()
// Re-verify under the re-acquired lock: while m.mu was released for Phase 2,
// a concurrent Stop/Remove could have deleted this agent from the map or a
// racing path could have started it. The launching guard prevents a second
// concurrent Start of THIS agent, but not a Stop/delete, so re-check both
// before mutating launch state. (agent still points at the same struct; the
// map re-lookup is what detects a delete.)
// racing path could have started it.
if cur, ok := m.agents[name]; !ok || cur != agent {
return fmt.Errorf("agent %s removed during launch", name)
}
if agent.State == StateRunning {
// Another path won the launch race while we were unlocked; nothing to do.
if leaseID != "" {
agent.poolLeaseID = leaseID
launchSucceeded = true
if !hasLeaseWatcher {
m.startSharedPoolLeaseWatcherLocked(agent, name, leaseID)
}
}
return nil
}
return m.launchInTmux(ctx, agent)
if agent.OperatorStopped {
return fmt.Errorf("agent %s was stopped during launch", name)
}
agent.poolLeaseID = leaseID
err := m.launchInTmux(ctx, agent)
if err == nil && leaseID != "" {
launchSucceeded = true
if !hasLeaseWatcher {
m.startSharedPoolLeaseWatcherLocked(agent, name, leaseID)
}
}
return err
}

func (m *Manager) startSharedPoolLeaseWatcherLocked(agent *AgentProcess, name, leaseID string) {
if agent.poolLeaseCancel == nil {
leaseCtx, cancel := context.WithCancel(context.Background())
agent.poolLeaseCancel = cancel
go m.watchSharedPoolLease(leaseCtx, name, leaseID)
}
}

// kickOutcomePollEvery throttles the post-kick turn-ended check to one visible
Expand Down Expand Up @@ -1262,35 +1314,33 @@

func (m *Manager) Stop(name string) error {
m.mu.Lock()
defer m.mu.Unlock()

agent, ok := m.agents[name]
if !ok {
m.mu.Unlock()
return fmt.Errorf("agent %s not found", name)
}

if agent.State == StateStopped {
agent.OperatorStopped = true
}
if agent.State != StateRunning {
return nil
leaseID := agent.poolLeaseID
if agent.State == StateRunning {
if agent.cancel != nil {
agent.cancel()
}
m.tmuxSendKeysForAgent(agent, "C-c", "")
agent.State = StateStopped
agent.OperatorStopped = true
m.logger.Info("audit: agent stopped", "name", name)
m.audit(AuditAgentStopped, name, auditFields(
"outcome", "success",
"backend", agent.effectiveBackend(),
"model", agent.effectiveModel(),
))
}

if agent.cancel != nil {
agent.cancel()
m.mu.Unlock()
if leaseID != "" {
m.evictSharedPoolSession(name, leaseID)
}

m.tmuxSendKeysForAgent(agent, "C-c", "")

agent.State = StateStopped
agent.OperatorStopped = true
m.logger.Info("audit: agent stopped", "name", name)
m.audit(AuditAgentStopped, name, auditFields(
"outcome", "success",
"backend", agent.effectiveBackend(),
"model", agent.effectiveModel(),
))

return nil
}

Expand Down
18 changes: 16 additions & 2 deletions src/crux/agent/manager_kick.go
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
package agent

import (
Expand Down Expand Up @@ -128,10 +128,24 @@
}

if agent.State != StateRunning {
if m.deferStartupKickLocked(agent, message) {
if m.sharedPool != nil && agent.State == StateStopped && !agent.Paused && !agent.OperatorStopped {
// Re-lock before every return: the deferred Unlock above must
// always pair with a held mutex.
m.mu.Unlock()
err := m.Start(context.Background(), name)
m.mu.Lock()
if err != nil {
return fmt.Errorf("auto-start stopped agent for kick: %w", err)
}
agent, ok = m.agents[name]
if !ok || agent.State != StateRunning {
return fmt.Errorf("agent %s failed to enter running state", name)
}
} else if m.deferStartupKickLocked(agent, message) {
return nil
} else {
return fmt.Errorf("agent %s cannot be kicked: %s", name, notRunningReason(agent))
}
return fmt.Errorf("agent %s cannot be kicked: %s", name, notRunningReason(agent))
}
if m.deferStartupKickLocked(agent, message) {
return nil
Expand Down
10 changes: 9 additions & 1 deletion src/crux/agent/manager_pause.go
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
package agent

import (
Expand Down Expand Up @@ -227,17 +227,25 @@
m.mu.Unlock()
m.mintAgentTokenUnlocked(ctx, agent)
m.bobPreflightUnlocked(agent)
// The relaunch needs a live-session lease exactly like Start.
leaseID, freshLease, leaseErr := m.acquireSharedPoolLeaseUnlocked(ctx, agent, name)
m.mu.Lock()
if leaseErr != nil {
m.mu.Unlock()
return leaseErr
}
if cur, ok := m.agents[name]; !ok || cur != agent {
m.abandonFreshSharedPoolLeaseLocked(name, leaseID, freshLease)
m.mu.Unlock()
return fmt.Errorf("agent %s removed during resume", name)
}
if agent.State == StateRunning {
// Another path won the launch race while we were unlocked.
m.abandonFreshSharedPoolLeaseLocked(name, leaseID, freshLease)
m.mu.Unlock()
return nil
}
err := m.launchInTmux(ctx, agent)
err := m.launchInTmuxLeasedLocked(ctx, agent, name, leaseID, freshLease)
m.mu.Unlock()
return err
}
Expand Down
Loading
Loading