From 6b7d8f5573ec793577c268c0f39e9fcc0a744332 Mon Sep 17 00:00:00 2001 From: James Reilly Date: Tue, 27 Oct 2026 06:38:51 +0000 Subject: [PATCH 1/8] feat(hub,agent): Hub-coordinated live CLI session pool for multi-tenant runner capacity Introduce an opt-in Hub-coordinated session lease pool so multiple Hive spokes sharing constrained runner hosts can serialize heavyweight CLI processes without sharing filesystem trees, tmux sessions, or credentials. - Add /api/shared-agent-pool/leases on Hub backed by per-Hive HMAC auth, lease persistence across restarts, and a single-waiter FIFO queue. - Add WithSharedSessionPool runtime and Manager option to acquire a lease before launch and evict idle sessions back to stopped state post-turn. - Add HIVE_SHARED_AGENT_POOL_ENABLED and HIVE_SHARED_AGENT_POOL_MAX_SESSIONS environment controls. - Add comprehensive regression and HTTP integration tests across hub and agent. Signed-off-by: James Reilly --- .../added-shared-agent-session-pool.md | 1 + src/cmd/hive/main.go | 27 +- src/cmd/hive/shared_pool.go | 15 ++ src/cmd/hive/shared_pool_test.go | 27 ++ src/crux/agent/kick_outcome.go | 20 +- src/crux/agent/manager.go | 98 +++++-- src/crux/agent/shared_pool.go | 239 +++++++++++++++++ src/crux/agent/shared_pool_test.go | 149 +++++++++++ src/crux/runtime/runtime.go | 14 + src/crux/runtime/runtime_test.go | 27 ++ src/docs/env-vars.md | 3 + src/pkg/hub/hub_testing_helpers_test.go | 1 + src/pkg/hub/server.go | 38 +-- src/pkg/hub/shared_agent_pool.go | 243 ++++++++++++++++++ src/pkg/hub/shared_agent_pool_test.go | 116 +++++++++ 15 files changed, 970 insertions(+), 48 deletions(-) create mode 100644 changelog.d/added-shared-agent-session-pool.md create mode 100644 src/cmd/hive/shared_pool.go create mode 100644 src/cmd/hive/shared_pool_test.go create mode 100644 src/crux/agent/shared_pool.go create mode 100644 src/crux/agent/shared_pool_test.go create mode 100644 src/pkg/hub/shared_agent_pool.go create mode 100644 src/pkg/hub/shared_agent_pool_test.go diff --git a/changelog.d/added-shared-agent-session-pool.md b/changelog.d/added-shared-agent-session-pool.md new file mode 100644 index 0000000000..30b6049e64 --- /dev/null +++ b/changelog.d/added-shared-agent-session-pool.md @@ -0,0 +1 @@ +- Add an opt-in Hub-coordinated CLI session pool for spokes. Participating Hives keep separate workspaces, UIDs, credentials, and tmux sessions; the Hub leases a global live-session slot and idle sessions are terminated after a turn. Configure `HIVE_SHARED_AGENT_POOL_ENABLED` on spokes and `HIVE_SHARED_AGENT_POOL_MAX_SESSIONS` on the Hub. diff --git a/src/cmd/hive/main.go b/src/cmd/hive/main.go index 7248b15a9f..7ef08fe682 100644 --- a/src/cmd/hive/main.go +++ b/src/cmd/hive/main.go @@ -27,6 +27,7 @@ import ( gh "github.com/google/go-github/v72/github" "github.com/hivecommons/hive/crux" + cruxagent "github.com/hivecommons/hive/crux/agent" "github.com/hivecommons/hive/crux/inference" "github.com/hivecommons/hive/crux/logscrub" "github.com/hivecommons/hive/crux/proxy" @@ -2192,10 +2193,25 @@ func (b *boot) bootAgentsWith(deps bootAgentsDeps) { b.logger.Warn("cleared invalid inference launch_cmd; inference and gateway backends launch through claude", "agent", name) } rtCfg := cruxconfig.RuntimeConfigFromEnv() - b.agentMgr, err = deps.newRuntime(rtCfg, + runtimeOptions := []cruxruntime.Option{ cruxruntime.WithDefinitions(b.cfg.EnabledAgents()), cruxruntime.WithLogger(b.logger), - cruxruntime.WithProject(b.projectCtx)) + cruxruntime.WithProject(b.projectCtx), + } + if sharedAgentPoolEnabled() { + hubURL := strings.TrimSpace(os.Getenv("HIVE_HUB_URL")) + if hubURL == "" { + hubURL = b.cfg.Hub.URL + } + pool, poolErr := cruxagent.NewSharedSessionPoolClient(hubURL, b.cfg.HiveID, hub.SpokeHeartbeatKey()) + if poolErr != nil { + b.logger.Error("shared agent pool is enabled but cannot be configured", "error", poolErr) + deps.exit(1) + return + } + runtimeOptions = append(runtimeOptions, cruxruntime.WithSharedSessionPool(pool)) + } + b.agentMgr, err = deps.newRuntime(rtCfg, runtimeOptions...) if err != nil { b.logger.Error("crux runtime unavailable", "mode", rtCfg.Mode, "error", err) deps.exit(1) @@ -4638,7 +4654,12 @@ func (b *boot) bootLaunchWith(deps bootLaunchDeps) { // shutdown the ctx-aware stagger returns immediately instead of leaking a // goroutine parked in a bare time.Sleep. enabledAgents := b.cfg.EnabledAgents() - startupAgents := startupLaunchNames(enabledAgents, b.onDemandFromPack) + startupAgents := []string{} + if sharedAgentPoolEnabled() { + b.logger.Info("shared agent pool enabled; persistent agents will start on demand") + } else { + startupAgents = startupLaunchNames(enabledAgents, b.onDemandFromPack) + } b.agentMgr.MarkStartupLaunchQueued(startupAgents) deps.spawn("agent-launch", func() { agentIndex := 0 diff --git a/src/cmd/hive/shared_pool.go b/src/cmd/hive/shared_pool.go new file mode 100644 index 0000000000..2b2e1a77be --- /dev/null +++ b/src/cmd/hive/shared_pool.go @@ -0,0 +1,15 @@ +package main + +import ( + "os" + "strings" +) + +func sharedAgentPoolEnabled() bool { + switch strings.ToLower(strings.TrimSpace(os.Getenv("HIVE_SHARED_AGENT_POOL_ENABLED"))) { + case "1", "true", "yes", "on": + return true + default: + return false + } +} diff --git a/src/cmd/hive/shared_pool_test.go b/src/cmd/hive/shared_pool_test.go new file mode 100644 index 0000000000..1a5991b786 --- /dev/null +++ b/src/cmd/hive/shared_pool_test.go @@ -0,0 +1,27 @@ +package main + +import ( + "testing" +) + +func TestSharedAgentPoolEnabledEnv(t *testing.T) { + for _, tc := range []struct { + val string + want bool + }{ + {"", false}, + {"0", false}, + {"false", false}, + {"no", false}, + {"1", true}, + {"true", true}, + {"TRUE", true}, + {"yes", true}, + {"on", true}, + } { + t.Setenv("HIVE_SHARED_AGENT_POOL_ENABLED", tc.val) + if got := sharedAgentPoolEnabled(); got != tc.want { + t.Errorf("sharedAgentPoolEnabled() for %q = %v, want %v", tc.val, got, tc.want) + } + } +} diff --git a/src/crux/agent/kick_outcome.go b/src/crux/agent/kick_outcome.go index 3a60327be6..493c901592 100644 --- a/src/crux/agent/kick_outcome.go +++ b/src/crux/agent/kick_outcome.go @@ -241,9 +241,21 @@ func (m *Manager) SetKickOutcomeObserver(fn func(agentName string, outcome KickO } func (m *Manager) notifyKickOutcome(agentName string, outcome KickOutcome) { - fn := m.kickOutcomeObserver.Load() - if fn == nil { - return + if fn := m.kickOutcomeObserver.Load(); fn != nil { + go (*fn)(agentName, outcome) + } + if m.sharedPool != nil { + go func() { + m.mu.RLock() + agent := m.agents[agentName] + leaseID := "" + if agent != nil { + leaseID = agent.poolLeaseID + } + m.mu.RUnlock() + if leaseID != "" { + m.evictSharedPoolSessionAfterTurn(agentName, leaseID, outcome) + } + }() } - go (*fn)(agentName, outcome) } diff --git a/src/crux/agent/manager.go b/src/crux/agent/manager.go index bb317bf684..189fefe183 100644 --- a/src/crux/agent/manager.go +++ b/src/crux/agent/manager.go @@ -258,6 +258,8 @@ type AgentProcess struct { KickRefusalReason string LaunchedMode AgentMode HasLaunched bool + poolLeaseID string + poolLeaseCancel context.CancelFunc baseConfig config.AgentConfig agentSpecPrompt string tmuxSession string @@ -575,6 +577,7 @@ type Manager struct { copilotAuthTokenRejected bool claudeAuthToken string uidMap *UIDMap + sharedPool SharedSessionPool uidMapPath string appAuth AppTokenMinter appNotInstalledWarned sync.Map // agent name -> struct{}: not-installed mint failure already logged @@ -789,6 +792,7 @@ type ManagerOption func(*managerOptions) type managerOptions struct { uidMapPath string uidMapPathSet bool + sharedPool SharedSessionPool } // WithUIDMapPath points a manager at a specific persisted UID map. Production @@ -800,6 +804,13 @@ func WithUIDMapPath(path string) ManagerOption { } } +// WithSharedSessionPool opts a manager into a Hub-coordinated live CLI slot. +// Every Hive retains separate sessions and tenant data; the lease only limits +// how many CLI sessions across the participating Hives remain live. +func WithSharedSessionPool(pool SharedSessionPool) ManagerOption { + return func(opts *managerOptions) { opts.sharedPool = pool } +} + func (m *Manager) effectiveUIDMapPath() string { if m.uidMapPath != "" { return m.uidMapPath @@ -889,6 +900,7 @@ func NewManagerWithOptions(agents map[string]config.AgentConfig, logger *slog.Lo copilotAuthTokenSource: copilotTokenSource, claudeAuthToken: claudeToken, uidMap: uidMap, + sharedPool: opts.sharedPool, uidMapPath: selectedUIDMapPath, kickLogDir: kickLogDir, kickLogRetention: kickLogRetention, @@ -1148,22 +1160,58 @@ func (m *Manager) Start(ctx context.Context, name string) error { // // The remaining subprocess sends below are the actual tmux launch boundary; // Bob's shared-home/file probes have already run lock-free above. + m.mu.RLock() + leaseID := agent.poolLeaseID + hasLeaseWatcher := agent.poolLeaseCancel != nil + m.mu.RUnlock() + if m.sharedPool != nil && leaseID == "" { + var err error + leaseID, err = m.sharedPool.Acquire(ctx, name) + if err != nil { + return fmt.Errorf("acquire shared CLI session slot: %w", err) + } + } + launchSucceeded := false + defer func() { + if leaseID != "" && !launchSucceeded { + m.releaseSharedPoolLease(name, leaseID) + } + }() + m.mu.Lock() defer m.mu.Unlock() // Re-verify under the re-acquired lock: while m.mu was released for Phase 2, // a concurrent Stop/Remove could have deleted this agent from the map or a - // racing path could have started it. The launching guard prevents a second - // concurrent Start of THIS agent, but not a Stop/delete, so re-check both - // before mutating launch state. (agent still points at the same struct; the - // map re-lookup is what detects a delete.) + // racing path could have started it. if cur, ok := m.agents[name]; !ok || cur != agent { return fmt.Errorf("agent %s removed during launch", name) } if agent.State == StateRunning { - // Another path won the launch race while we were unlocked; nothing to do. + if leaseID != "" { + agent.poolLeaseID = leaseID + launchSucceeded = true + if !hasLeaseWatcher { + leaseCtx, cancel := context.WithCancel(context.Background()) + agent.poolLeaseCancel = cancel + go m.watchSharedPoolLease(leaseCtx, name, leaseID) + } + } return nil } - return m.launchInTmux(ctx, agent) + if agent.OperatorStopped { + return fmt.Errorf("agent %s was stopped during launch", name) + } + agent.poolLeaseID = leaseID + err := m.launchInTmux(ctx, agent) + if err == nil && leaseID != "" { + launchSucceeded = true + if !hasLeaseWatcher { + leaseCtx, cancel := context.WithCancel(context.Background()) + agent.poolLeaseCancel = cancel + go m.watchSharedPoolLease(leaseCtx, name, leaseID) + } + } + return err } // kickOutcomePollEvery throttles the post-kick turn-ended check to one visible @@ -1262,35 +1310,33 @@ func (m *Manager) waitForInputPromptForAgentReason(agent *AgentProcess, abort fu func (m *Manager) Stop(name string) error { m.mu.Lock() - defer m.mu.Unlock() - agent, ok := m.agents[name] if !ok { + m.mu.Unlock() return fmt.Errorf("agent %s not found", name) } - if agent.State == StateStopped { agent.OperatorStopped = true } - if agent.State != StateRunning { - return nil + leaseID := agent.poolLeaseID + if agent.State == StateRunning { + if agent.cancel != nil { + agent.cancel() + } + m.tmuxSendKeysForAgent(agent, "C-c", "") + agent.State = StateStopped + agent.OperatorStopped = true + m.logger.Info("audit: agent stopped", "name", name) + m.audit(AuditAgentStopped, name, auditFields( + "outcome", "success", + "backend", agent.effectiveBackend(), + "model", agent.effectiveModel(), + )) } - - if agent.cancel != nil { - agent.cancel() + m.mu.Unlock() + if leaseID != "" { + m.evictSharedPoolSession(name, leaseID) } - - m.tmuxSendKeysForAgent(agent, "C-c", "") - - agent.State = StateStopped - agent.OperatorStopped = true - m.logger.Info("audit: agent stopped", "name", name) - m.audit(AuditAgentStopped, name, auditFields( - "outcome", "success", - "backend", agent.effectiveBackend(), - "model", agent.effectiveModel(), - )) - return nil } diff --git a/src/crux/agent/shared_pool.go b/src/crux/agent/shared_pool.go new file mode 100644 index 0000000000..a8093944d9 --- /dev/null +++ b/src/crux/agent/shared_pool.go @@ -0,0 +1,239 @@ +package agent + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "time" +) + +const sharedSessionPoolPath = "/api/shared-agent-pool" +const sharedPoolIdleEvictionDelay = 5 * time.Minute + +var errSharedSessionPoolBusy = fmt.Errorf("shared session pool is busy") + +// SharedSessionPool coordinates heavyweight CLI sessions across Hive tenants. +// Each tenant keeps its own tmux session, UID, credentials, and workspace; only +// the right to have a live CLI session is shared. +type SharedSessionPool interface { + Acquire(ctx context.Context, agentName string) (string, error) + Renew(ctx context.Context, leaseID string) error + Release(ctx context.Context, leaseID string) error +} + +type sharedSessionPoolClient struct { + baseURL string + hiveID string + token string + client *http.Client +} + +// NewSharedSessionPoolClient creates an authenticated client for the Hub's +// tenant-scoped session-slot API. The token must be the per-Hive heartbeat key, +// never the fleet master secret. +func NewSharedSessionPoolClient(baseURL, hiveID, token string) (SharedSessionPool, error) { + baseURL = strings.TrimRight(strings.TrimSpace(baseURL), "/") + if baseURL == "" || hiveID == "" || token == "" { + return nil, fmt.Errorf("shared session pool requires Hub URL, Hive ID, and per-Hive token") + } + return &sharedSessionPoolClient{ + baseURL: baseURL, + hiveID: hiveID, + token: token, + client: &http.Client{Timeout: 10 * time.Second}, + }, nil +} + +type sharedPoolRequest struct { + HiveID string `json:"hive_id"` + AgentName string `json:"agent_name,omitempty"` + LeaseID string `json:"lease_id,omitempty"` + CancelWait bool `json:"cancel_wait,omitempty"` +} + +type sharedPoolResponse struct { + LeaseID string `json:"lease_id"` +} + +func (c *sharedSessionPoolClient) request(ctx context.Context, method, path string, body sharedPoolRequest) (sharedPoolResponse, error) { + payload, err := json.Marshal(body) + if err != nil { + return sharedPoolResponse{}, err + } + req, err := http.NewRequestWithContext(ctx, method, c.baseURL+sharedSessionPoolPath+path, bytes.NewReader(payload)) + if err != nil { + return sharedPoolResponse{}, err + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Bearer "+c.token) + resp, err := c.client.Do(req) + if err != nil { + return sharedPoolResponse{}, err + } + defer resp.Body.Close() + if resp.StatusCode == http.StatusConflict { + return sharedPoolResponse{}, errSharedSessionPoolBusy + } + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + message, _ := io.ReadAll(io.LimitReader(resp.Body, 2048)) + return sharedPoolResponse{}, fmt.Errorf("shared session pool returned %s: %s", resp.Status, strings.TrimSpace(string(message))) + } + if resp.ContentLength == 0 { + return sharedPoolResponse{}, nil + } + var result sharedPoolResponse + if err := json.NewDecoder(io.LimitReader(resp.Body, 4096)).Decode(&result); err != nil && err != io.EOF { + return sharedPoolResponse{}, err + } + return result, nil +} + +func (c *sharedSessionPoolClient) Acquire(ctx context.Context, agentName string) (string, error) { + for { + result, err := c.request(ctx, http.MethodPost, "/leases", sharedPoolRequest{HiveID: c.hiveID, AgentName: agentName}) + if err == nil { + if result.LeaseID == "" { + return "", fmt.Errorf("shared session pool returned an empty lease ID") + } + return result.LeaseID, nil + } + if err != errSharedSessionPoolBusy { + return "", err + } + timer := time.NewTimer(3 * time.Second) + select { + case <-ctx.Done(): + timer.Stop() + cancelCtx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + _, _ = c.request(cancelCtx, http.MethodDelete, "/leases", sharedPoolRequest{HiveID: c.hiveID, CancelWait: true}) + cancel() + return "", ctx.Err() + case <-timer.C: + } + } +} + +func (c *sharedSessionPoolClient) Renew(ctx context.Context, leaseID string) error { + _, err := c.request(ctx, http.MethodPut, "/leases", sharedPoolRequest{HiveID: c.hiveID, LeaseID: leaseID}) + return err +} + +func (c *sharedSessionPoolClient) Release(ctx context.Context, leaseID string) error { + _, err := c.request(ctx, http.MethodDelete, "/leases", sharedPoolRequest{HiveID: c.hiveID, LeaseID: leaseID}) + return err +} + +func (m *Manager) watchSharedPoolLease(ctx context.Context, name, leaseID string) { + ticker := time.NewTicker(20 * time.Second) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + } + m.mu.RLock() + a := m.agents[name] + if a == nil || a.poolLeaseID != leaseID { + m.mu.RUnlock() + return + } + state := a.State + idle := a.LastKick == nil || a.KickOutcome.Settled(a.LastKick) + delivering := a.kickDelivering.Load() + idleSince := time.Time{} + if a.StartedAt != nil { + idleSince = *a.StartedAt + } + if a.KickOutcome.At.After(idleSince) { + idleSince = a.KickOutcome.At + } + m.mu.RUnlock() + if state == StateRunning && idle && !delivering && + time.Since(idleSince) >= sharedPoolIdleEvictionDelay && + paneShowsInputPrompt(m.captureVisiblePaneForAgent(a)) { + m.evictSharedPoolSession(name, leaseID) + return + } + ctx, cancel := context.WithTimeout(context.Background(), 8*time.Second) + err := m.sharedPool.Renew(ctx, leaseID) + cancel() + if err != nil { + m.logger.Warn("shared agent pool lease renewal failed; evicting CLI session", "agent", name, "error", err) + m.evictSharedPoolSession(name, leaseID) + return + } + } +} + +func (m *Manager) evictSharedPoolSession(name, leaseID string) { + m.evictSharedPoolSessionIf(name, leaseID, nil) +} + +func (m *Manager) evictSharedPoolSessionAfterTurn(name, leaseID string, outcome KickOutcome) { + m.evictSharedPoolSessionIf(name, leaseID, &outcome) +} + +func (m *Manager) evictSharedPoolSessionIf(name, leaseID string, expected *KickOutcome) { + m.mu.Lock() + a := m.agents[name] + if a == nil || a.poolLeaseID != leaseID { + m.mu.Unlock() + return + } + if expected != nil && + (a.KickOutcome.At != expected.At || + a.KickOutcome.KickAt != expected.KickAt || + a.KickOutcome.Kind != expected.Kind || + a.KickOutcome.Reason != expected.Reason || + !a.KickOutcome.Settled(a.LastKick) || + a.kickDelivering.Load()) { + m.mu.Unlock() + return + } + if a.cancel != nil { + a.cancel() + } + m.archiveKickLogLocked(a, "shared-pool-eviction") + _ = m.tmuxCmd(a, "kill-session", "-t", a.tmuxSession).Run() + _ = m.reapAgentCLI(a) + a.State = StateStopped + a.poolLeaseID = "" + cancel := a.poolLeaseCancel + a.poolLeaseCancel = nil + m.mu.Unlock() + if cancel != nil { + cancel() + } + ctx, done := context.WithTimeout(context.Background(), 8*time.Second) + if err := m.sharedPool.Release(ctx, leaseID); err != nil { + m.logger.Warn("shared agent pool lease release failed", "agent", name, "error", err) + } + done() + m.logger.Info("shared agent pool evicted idle CLI session", "agent", name) +} + +func (m *Manager) releaseSharedPoolLease(name, leaseID string) { + m.mu.Lock() + a := m.agents[name] + if a != nil && a.poolLeaseID == leaseID { + a.poolLeaseID = "" + cancel := a.poolLeaseCancel + a.poolLeaseCancel = nil + m.mu.Unlock() + if cancel != nil { + cancel() + } + } else { + m.mu.Unlock() + } + ctx, done := context.WithTimeout(context.Background(), 8*time.Second) + defer done() + if err := m.sharedPool.Release(ctx, leaseID); err != nil { + m.logger.Warn("shared agent pool lease release failed", "agent", name, "error", err) + } +} diff --git a/src/crux/agent/shared_pool_test.go b/src/crux/agent/shared_pool_test.go new file mode 100644 index 0000000000..2e063edb4c --- /dev/null +++ b/src/crux/agent/shared_pool_test.go @@ -0,0 +1,149 @@ +package agent + +import ( + "context" + "sync" + "testing" + "time" + + "github.com/hivecommons/hive/crux/config" +) + +type recordingSharedSessionPool struct { + mu sync.Mutex + acquired []string + released []string + renewed []string + acquireFn func(ctx context.Context, agent string) (string, error) +} + +func (p *recordingSharedSessionPool) Acquire(ctx context.Context, agent string) (string, error) { + p.mu.Lock() + p.acquired = append(p.acquired, agent) + fn := p.acquireFn + p.mu.Unlock() + if fn != nil { + return fn(ctx, agent) + } + return "lease-" + agent, nil +} + +func (p *recordingSharedSessionPool) Renew(_ context.Context, id string) error { + p.mu.Lock() + p.renewed = append(p.renewed, id) + p.mu.Unlock() + return nil +} + +func (p *recordingSharedSessionPool) Release(_ context.Context, id string) error { + p.mu.Lock() + p.released = append(p.released, id) + p.mu.Unlock() + return nil +} + +func TestSharedPoolEvictionStopsCLIAndReleasesLease(t *testing.T) { + pool := &recordingSharedSessionPool{} + m := NewManager(map[string]config.AgentConfig{"scanner": {Backend: "claude"}}, discardLogger(), ProjectContext{}) + m.sharedPool = pool + a := m.agents["scanner"] + a.State = StateRunning + a.poolLeaseID = "lease-1" + + m.evictSharedPoolSession("scanner", "lease-1") + + if a.State != StateStopped { + t.Fatalf("agent state = %q, want stopped", a.State) + } + if a.poolLeaseID != "" { + t.Fatalf("agent retained lease %q", a.poolLeaseID) + } + pool.mu.Lock() + defer pool.mu.Unlock() + if len(pool.released) != 1 || pool.released[0] != "lease-1" { + t.Fatalf("released leases = %v, want [lease-1]", pool.released) + } +} + +func TestSharedPoolStopReleasesLease(t *testing.T) { + pool := &recordingSharedSessionPool{} + m := NewManager(map[string]config.AgentConfig{"scanner": {Backend: "claude"}}, discardLogger(), ProjectContext{}) + m.sharedPool = pool + a := m.agents["scanner"] + a.State = StateRunning + a.poolLeaseID = "lease-stop" + + if err := m.Stop("scanner"); err != nil { + t.Fatalf("Stop failed: %v", err) + } + + if a.State != StateStopped { + t.Fatalf("agent state = %q, want stopped", a.State) + } + if a.poolLeaseID != "" { + t.Fatalf("agent retained lease %q", a.poolLeaseID) + } + pool.mu.Lock() + defer pool.mu.Unlock() + if len(pool.released) != 1 || pool.released[0] != "lease-stop" { + t.Fatalf("released leases = %v, want [lease-stop]", pool.released) + } +} + +func TestSharedPoolKickOutcomeEvictsSession(t *testing.T) { + pool := &recordingSharedSessionPool{} + m := NewManager(map[string]config.AgentConfig{"scanner": {Backend: "claude"}}, discardLogger(), ProjectContext{}) + m.sharedPool = pool + a := m.agents["scanner"] + now := time.Now() + a.State = StateRunning + a.StartedAt = &now + a.LastKick = &now + a.poolLeaseID = "lease-outcome" + + outcome := KickOutcome{ + Kind: KickOutcomeEnded, + At: now, + KickAt: now, + } + a.KickOutcome = outcome + + m.notifyKickOutcome("scanner", outcome) + + // Wait briefly for the asynchronous eviction goroutine to complete. + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + m.mu.RLock() + st := a.State + lease := a.poolLeaseID + m.mu.RUnlock() + if st == StateStopped && lease == "" { + break + } + time.Sleep(10 * time.Millisecond) + } + + m.mu.RLock() + if a.State != StateStopped || a.poolLeaseID != "" { + t.Fatalf("after outcome: state=%q lease=%q, want stopped/empty", a.State, a.poolLeaseID) + } + m.mu.RUnlock() + + pool.mu.Lock() + defer pool.mu.Unlock() + if len(pool.released) != 1 || pool.released[0] != "lease-outcome" { + t.Fatalf("released leases = %v, want [lease-outcome]", pool.released) + } +} + +func TestWithSharedSessionPoolOption(t *testing.T) { + pool := &recordingSharedSessionPool{} + m := NewManagerWithOptions(map[string]config.AgentConfig{"scanner": {Backend: "claude"}}, + discardLogger(), + ProjectContext{}, + WithSharedSessionPool(pool), + ) + if m.sharedPool != pool { + t.Fatal("sharedPool option was not applied") + } +} diff --git a/src/crux/runtime/runtime.go b/src/crux/runtime/runtime.go index cee764ad00..bad1b89709 100644 --- a/src/crux/runtime/runtime.go +++ b/src/crux/runtime/runtime.go @@ -55,6 +55,7 @@ type options struct { logger *slog.Logger project crux.ProjectContext uidMapPath *string + sharedPool agent.SharedSessionPool } // WithDefinitions supplies the agent definitions the runtime manages. The @@ -81,6 +82,13 @@ func WithUIDMapPath(path string) Option { return func(o *options) { o.uidMapPath = &path } } +// WithSharedSessionPool coordinates heavyweight CLI sessions through the Hub. +// It affects only the in-process Manager; sidecar mode must implement the same +// lease contract before it can opt in. +func WithSharedSessionPool(pool agent.SharedSessionPool) Option { + return func(o *options) { o.sharedPool = pool } +} + // New validates cfg and returns the runtime its Mode selects. Options are // applied in order; a nil option is ignored. func New(cfg config.RuntimeConfig, opts ...Option) (crux.Runtime, error) { @@ -93,6 +101,9 @@ func New(cfg config.RuntimeConfig, opts ...Option) (crux.Runtime, error) { opt(&o) } } + if o.sharedPool != nil && cfg.Mode != config.ModeInProcess { + return nil, fmt.Errorf("shared session pool is not supported by runtime mode %q", cfg.Mode) + } switch cfg.Mode { case config.ModeInProcess: return newInProcess(o), nil @@ -122,6 +133,9 @@ func newInProcess(o options) crux.Runtime { if o.uidMapPath != nil { mopts = append(mopts, agent.WithUIDMapPath(*o.uidMapPath)) } + if o.sharedPool != nil { + mopts = append(mopts, agent.WithSharedSessionPool(o.sharedPool)) + } return agent.AsRuntime(agent.NewManagerWithOptions(o.defs, o.logger, o.project, mopts...)) } diff --git a/src/crux/runtime/runtime_test.go b/src/crux/runtime/runtime_test.go index 6703ce48ce..d06985fe10 100644 --- a/src/crux/runtime/runtime_test.go +++ b/src/crux/runtime/runtime_test.go @@ -239,3 +239,30 @@ func TestRunAgyTurnHelp(t *testing.T) { t.Fatalf("RunAgyTurn(-h) = %d, stderr %q", code, stderr.String()) } } + +type stubSharedSessionPool struct{} + +func (stubSharedSessionPool) Acquire(context.Context, string) (string, error) { return "lease", nil } +func (stubSharedSessionPool) Renew(context.Context, string) error { return nil } +func (stubSharedSessionPool) Release(context.Context, string) error { return nil } + +func TestWithSharedSessionPool(t *testing.T) { + t.Setenv("HIVE_WORK_DIR", t.TempDir()) + pool := stubSharedSessionPool{} + rt, err := cruxruntime.New(config.RuntimeConfig{Mode: config.ModeInProcess}, + cruxruntime.WithSharedSessionPool(pool), + ) + if err != nil { + t.Fatalf("in-process with shared pool: %v", err) + } + if rt == nil { + t.Fatal("expected non-nil runtime") + } + + _, err = cruxruntime.New(config.RuntimeConfig{Mode: config.ModeSidecar, Server: "http://127.0.0.1:9", Token: "t"}, + cruxruntime.WithSharedSessionPool(pool), + ) + if err == nil || !strings.Contains(err.Error(), "shared session pool is not supported") { + t.Fatalf("sidecar with shared pool err = %v, want unsupported error", err) + } +} diff --git a/src/docs/env-vars.md b/src/docs/env-vars.md index add0f06384..3c68477e2c 100644 --- a/src/docs/env-vars.md +++ b/src/docs/env-vars.md @@ -28,6 +28,9 @@ This reference is compiled by hand from the Go source under `src/`, the deployme | `HIVE_ID` | No | config or generated id | Stable hive/spoke identifier override; passed through to launched agents. | | `HIVE_CLUSTER_ID` | No | config or hub-provisioned value | Hosted cluster identifier override. | | `HIVE_HUB_URL` | No | `hub.url` from config | Hub URL override for spoke heartbeats/registration. On the hub it is also the last environment variable consulted in the hub public-origin chain (see `HIVE_HUB_PUBLIC_URL`). | +| `HIVE_SHARED_AGENT_POOL_ENABLED` | No | `false` | Spoke opt-in for Hub-coordinated CLI session sharing. Requires an in-process runtime and a Hub link; each Hive keeps its own workspace, UID, credentials, and tmux session. Idle CLI sessions are terminated after turns to return the global slot. | +| `HIVE_SHARED_AGENT_POOL_MAX_SESSIONS` | No | `0` (disabled) | Hub-side maximum number of live CLI session leases. Set to `1` to serialize agent sessions across participating Hives. | +| `HIVE_SHARED_AGENT_POOL_LEASE_SECONDS` | No | `90` | Hub-side lease expiry in seconds. Values below 30 or invalid values use the 90-second default. Spokes renew active leases every 20 seconds. | | `HIVE_NPS_ENABLED` | No | `hub.nps_enabled`; when both are unset, on for hosted spokes (`hub.hive_type: hosted`) and off otherwise | Turns the dashboard NPS feedback prompt on or off. Accepts `1/true/yes/on` and `0/false/no/off`; other values are ignored. Responses are forwarded over the spoke's hub link or, for a standalone hive with a relay configured, to the NPS relay, never to anyone else. See [NPS feedback prompt](nps.md). | | `HIVE_NPS_RELAY_URL` | No | `hub.nps_relay_url`; empty (relay disabled) | Base URL of the NPS relay for standalone hives (the hivecommons relay is `https://docs.hivecommons.dev/api/nps`). A spoke with NPS enabled and no hub link self-registers a generated Ed25519 key at `/register` and POSTs signed responses here, with no token to configure; a hub pulls from `/pending` and acks at `/ack`. Must be `https` (plain `http` only to a loopback host). See [NPS feedback prompt](nps.md#standalone-hives-the-nps-relay). | | `HIVE_NPS_RELAY_PULL_SECRET` | No | `hub.nps_relay_pull_secret`; empty | Secret, hub only. Authenticates the hub's periodic pull from the NPS relay. Without it (or without a relay URL) the hub does not pull. Never logged. | diff --git a/src/pkg/hub/hub_testing_helpers_test.go b/src/pkg/hub/hub_testing_helpers_test.go index d0c4efa9bc..21dc2b21e6 100644 --- a/src/pkg/hub/hub_testing_helpers_test.go +++ b/src/pkg/hub/hub_testing_helpers_test.go @@ -83,6 +83,7 @@ var hubTestPathVars = []hubTestPathVar{ {ptr: ®istryPath, prod: registryPath, rel: "hub-registry.json", always: true}, {ptr: &hubSecretPath, prod: hubSecretPath, rel: filepath.Join("saas", "hub-secret.key"), always: true}, {ptr: &hubBannersPath, prod: hubBannersPath, rel: filepath.Join("saas", "hub-banners.json")}, + {ptr: &sharedAgentPoolPath, prod: sharedAgentPoolPath, rel: filepath.Join("saas", "shared-agent-pool.json")}, {ptr: &stablePromotionPath, prod: stablePromotionPath, rel: filepath.Join("saas", "stable-promotion.json")}, {ptr: &hubGenerationsPath, prod: hubGenerationsPath, rel: filepath.Join("saas", "hub-generations.json")}, {ptr: &clustersConfigPath, prod: clustersConfigPath, rel: filepath.Join("saas", "clusters.json")}, diff --git a/src/pkg/hub/server.go b/src/pkg/hub/server.go index 43296e89de..d1060e1b35 100644 --- a/src/pkg/hub/server.go +++ b/src/pkg/hub/server.go @@ -1032,12 +1032,13 @@ type HubServer struct { // goroutine — which outlives test servers, since nothing closes saveCh — // only ever sees its own path and cannot race a test redirecting the // global for the next server (the TestLoadRegistry -race failure). - registryPath string - hubGitHash string - hubGitBranch string - hubSecret string - myHivesCacheMu sync.Mutex - myHivesCache map[string]myHivesSnapshot + registryPath string + hubGitHash string + hubGitBranch string + sharedAgentPool *sharedAgentPool + hubSecret string + myHivesCacheMu sync.Mutex + myHivesCache map[string]myHivesSnapshot // keyGenerations is the ordered set of master generations this hub accepts // (hub_generations.go). Until a rotation happens it holds exactly ONE // generation whose secret IS hubSecret, so every derived key is @@ -1548,15 +1549,16 @@ func NewHubServer(port int, logger *slog.Logger, gitHash, gitBranch string) *Hub logger.Info("generated hub secret", "path", hubSecretPath) } s := &HubServer{ - mux: http.NewServeMux(), - logger: logger, - saveCh: make(chan struct{}, 1), - saveLoopStop: make(chan struct{}), - saveLoopDone: make(chan struct{}), - registryPath: registryPath, - hubGitHash: gitHash, - hubGitBranch: gitBranch, - hubSecret: secret, + mux: http.NewServeMux(), + logger: logger, + saveCh: make(chan struct{}, 1), + saveLoopStop: make(chan struct{}), + saveLoopDone: make(chan struct{}), + registryPath: registryPath, + hubGitHash: gitHash, + hubGitBranch: gitBranch, + sharedAgentPool: newSharedAgentPool(), + hubSecret: secret, // Provisional single generation holding the existing master. Replaced // immediately below by loadGenerations, which reads any persisted // rotation off the PVC. It is set here too so the field is never nil @@ -1699,6 +1701,12 @@ func NewHubServer(port int, logger *slog.Logger, gitHash, gitBranch string) *Hub // a hub that has just rolled never honours a session revoked before it. s.loadRevokedSessions() + handleSharedPoolLeases := func(w http.ResponseWriter, r *http.Request) { + s.sharedAgentPool.handle(w, r, s) + } + s.mux.HandleFunc("POST /api/shared-agent-pool/leases", handleSharedPoolLeases) + s.mux.HandleFunc("PUT /api/shared-agent-pool/leases", handleSharedPoolLeases) + s.mux.HandleFunc("DELETE /api/shared-agent-pool/leases", handleSharedPoolLeases) s.mux.HandleFunc("POST /api/heartbeat", s.handleHeartbeat) s.mux.HandleFunc("POST /api/task-status", s.handleTaskStatus) // NPS feedback forwarded by spokes (#9610): per-hive heartbeat bearer, the diff --git a/src/pkg/hub/shared_agent_pool.go b/src/pkg/hub/shared_agent_pool.go new file mode 100644 index 0000000000..2dea511f51 --- /dev/null +++ b/src/pkg/hub/shared_agent_pool.go @@ -0,0 +1,243 @@ +package hub + +import ( + "crypto/rand" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "net/http" + "os" + "path/filepath" + "strconv" + "strings" + "sync" + "time" +) + +var sharedAgentPoolPath = "/data/saas/shared-agent-pool.json" + +const defaultSharedAgentLeaseTTL = 90 * time.Second + +type sharedAgentLease struct { + ID string `json:"id"` + HiveID string `json:"hive_id"` + AgentName string `json:"agent_name"` + ExpiresAt time.Time `json:"expires_at"` +} + +type sharedAgentPoolState struct { + Version int `json:"version"` + Leases []sharedAgentLease `json:"leases"` +} + +type sharedAgentPool struct { + mu sync.Mutex + path string + max int + ttl time.Duration + leases map[string]sharedAgentLease + waitingHives map[string]struct{} + loadError error +} + +func newSharedAgentPool() *sharedAgentPool { + max, _ := strconv.Atoi(strings.TrimSpace(os.Getenv("HIVE_SHARED_AGENT_POOL_MAX_SESSIONS"))) + ttl := defaultSharedAgentLeaseTTL + if seconds, err := strconv.Atoi(strings.TrimSpace(os.Getenv("HIVE_SHARED_AGENT_POOL_LEASE_SECONDS"))); err == nil && seconds >= 30 { + ttl = time.Duration(seconds) * time.Second + } + p := &sharedAgentPool{ + path: sharedAgentPoolPath, + max: max, + ttl: ttl, + leases: make(map[string]sharedAgentLease), + waitingHives: make(map[string]struct{}), + } + p.load() + return p +} + +func sharedPoolOwnerKey(hiveID, agentName string) string { return hiveID + "\x00" + agentName } + +func (p *sharedAgentPool) load() { + data, err := os.ReadFile(p.path) + if errors.Is(err, os.ErrNotExist) { + return + } + if err != nil { + p.loadError = err + return + } + var state sharedAgentPoolState + if err := json.Unmarshal(data, &state); err != nil || state.Version != 1 { + p.loadError = fmt.Errorf("invalid shared agent pool state") + return + } + now := time.Now() + for _, lease := range state.Leases { + if lease.ID == "" || lease.HiveID == "" || lease.AgentName == "" || !lease.ExpiresAt.After(now) { + continue + } + p.leases[sharedPoolOwnerKey(lease.HiveID, lease.AgentName)] = lease + } +} + +func (p *sharedAgentPool) saveLocked() error { + state := sharedAgentPoolState{Version: 1, Leases: make([]sharedAgentLease, 0, len(p.leases))} + for _, lease := range p.leases { + state.Leases = append(state.Leases, lease) + } + data, err := json.Marshal(state) + if err != nil { + return err + } + if err := os.MkdirAll(filepath.Dir(p.path), 0o755); err != nil { + return err + } + tmpPath := p.path + ".tmp" + if err := os.WriteFile(tmpPath, data, 0o600); err != nil { + return err + } + if err := os.Rename(tmpPath, p.path); err != nil { + _ = os.Remove(tmpPath) + return err + } + return nil +} + +func (p *sharedAgentPool) hasOtherWaitingHive(hiveID string) bool { + for waitingHive := range p.waitingHives { + if waitingHive != hiveID { + return true + } + } + return false +} + +func (p *sharedAgentPool) handle(w http.ResponseWriter, r *http.Request, s *HubServer) { + if p == nil || p.max < 1 { + http.Error(w, "shared agent pool is disabled", http.StatusServiceUnavailable) + return + } + if p.loadError != nil { + s.logger.Error("shared agent pool state is untrusted; refusing leases", "error", p.loadError) + http.Error(w, "shared agent pool state unavailable", http.StatusServiceUnavailable) + return + } + if r.Method != http.MethodPost && r.Method != http.MethodPut && r.Method != http.MethodDelete { + w.Header().Set("Allow", "POST, PUT, DELETE") + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + return + } + auth := r.Header.Get("Authorization") + if !strings.HasPrefix(auth, "Bearer ") { + http.Error(w, "unauthorized", http.StatusUnauthorized) + return + } + var req struct { + HiveID string `json:"hive_id"` + AgentName string `json:"agent_name"` + LeaseID string `json:"lease_id"` + CancelWait bool `json:"cancel_wait"` + } + if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 4096)).Decode(&req); err != nil || req.HiveID == "" { + http.Error(w, "invalid request", http.StatusBadRequest) + return + } + if !s.verifyHeartbeatBearer(strings.TrimPrefix(auth, "Bearer "), req.HiveID) { + http.Error(w, "unauthorized", http.StatusUnauthorized) + return + } + p.mu.Lock() + defer p.mu.Unlock() + now := time.Now() + for key, lease := range p.leases { + if !lease.ExpiresAt.After(now) { + delete(p.leases, key) + } + } + switch r.Method { + case http.MethodPost: + if req.AgentName == "" || len(req.AgentName) > 128 { + http.Error(w, "invalid agent name", http.StatusBadRequest) + return + } + key := sharedPoolOwnerKey(req.HiveID, req.AgentName) + if lease, ok := p.leases[key]; ok { + lease.ExpiresAt = now.Add(p.ttl) + p.leases[key] = lease + delete(p.waitingHives, req.HiveID) + if err := p.saveLocked(); err != nil { + http.Error(w, "shared agent pool persistence failed", http.StatusServiceUnavailable) + return + } + writeSharedPoolLease(w, lease.ID) + return + } + if len(p.leases) >= p.max || p.hasOtherWaitingHive(req.HiveID) { + if p.waitingHives == nil { + p.waitingHives = make(map[string]struct{}) + } + p.waitingHives[req.HiveID] = struct{}{} + w.Header().Set("Retry-After", "3") + http.Error(w, "shared agent pool is busy", http.StatusConflict) + return + } + var raw [16]byte + if _, err := rand.Read(raw[:]); err != nil { + http.Error(w, "lease generation failed", http.StatusInternalServerError) + return + } + lease := sharedAgentLease{ID: hex.EncodeToString(raw[:]), HiveID: req.HiveID, AgentName: req.AgentName, ExpiresAt: now.Add(p.ttl)} + p.leases[key] = lease + delete(p.waitingHives, req.HiveID) + if err := p.saveLocked(); err != nil { + delete(p.leases, key) + http.Error(w, "shared agent pool persistence failed", http.StatusServiceUnavailable) + return + } + writeSharedPoolLease(w, lease.ID) + case http.MethodPut, http.MethodDelete: + if r.Method == http.MethodDelete && req.CancelWait { + delete(p.waitingHives, req.HiveID) + w.WriteHeader(http.StatusNoContent) + return + } + if req.LeaseID == "" { + http.Error(w, "lease ID required", http.StatusBadRequest) + return + } + var key string + for candidate, lease := range p.leases { + if lease.ID == req.LeaseID && lease.HiveID == req.HiveID { + key = candidate + break + } + } + if key == "" { + http.Error(w, "lease not found", http.StatusNotFound) + return + } + if r.Method == http.MethodPut { + lease := p.leases[key] + lease.ExpiresAt = now.Add(p.ttl) + p.leases[key] = lease + } else { + delete(p.leases, key) + } + if err := p.saveLocked(); err != nil { + http.Error(w, "shared agent pool persistence failed", http.StatusServiceUnavailable) + return + } + w.WriteHeader(http.StatusNoContent) + default: + w.Header().Set("Allow", "POST, PUT, DELETE") + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + } +} + +func writeSharedPoolLease(w http.ResponseWriter, leaseID string) { + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]string{"lease_id": leaseID}) +} diff --git a/src/pkg/hub/shared_agent_pool_test.go b/src/pkg/hub/shared_agent_pool_test.go new file mode 100644 index 0000000000..03b73c9f3f --- /dev/null +++ b/src/pkg/hub/shared_agent_pool_test.go @@ -0,0 +1,116 @@ +package hub + +import ( + "bytes" + "context" + "encoding/json" + "log/slog" + "net/http" + "net/http/httptest" + "path/filepath" + "testing" + "time" + + cruxagent "github.com/hivecommons/hive/crux/agent" +) + +func serveSharedPoolRequest(t *testing.T, pool *sharedAgentPool, s *HubServer, method, bearerHive string, body map[string]string) *httptest.ResponseRecorder { + t.Helper() + payload, err := json.Marshal(body) + if err != nil { + t.Fatal(err) + } + req := httptest.NewRequest(method, "/api/shared-agent-pool/leases", bytes.NewReader(payload)) + req.Header.Set("Authorization", "Bearer "+s.heartbeatKeyFor(bearerHive)) + rec := httptest.NewRecorder() + pool.handle(rec, req, s) + return rec +} + +func TestSharedAgentPoolTenantAuthCapacityAndPersistence(t *testing.T) { + path := filepath.Join(t.TempDir(), "shared-agent-pool.json") + pool := &sharedAgentPool{path: path, max: 1, ttl: time.Minute, leases: make(map[string]sharedAgentLease)} + s := &HubServer{logger: slog.Default()} + s.setHubSecret("test-shared-agent-pool-master") + + school := map[string]string{"hive_id": "school", "agent_name": "scanner"} + firstRec := serveSharedPoolRequest(t, pool, s, http.MethodPost, "school", school) + if firstRec.Code != http.StatusOK { + t.Fatalf("first acquire status = %d, body %q", firstRec.Code, firstRec.Body.String()) + } + var first struct { + LeaseID string `json:"lease_id"` + } + if err := json.Unmarshal(firstRec.Body.Bytes(), &first); err != nil || first.LeaseID == "" { + t.Fatalf("first lease response = %q, err %v", firstRec.Body.String(), err) + } + + spoof := map[string]string{"hive_id": "reef", "agent_name": "scanner"} + spoofRec := serveSharedPoolRequest(t, pool, s, http.MethodPost, "school", spoof) + if spoofRec.Code != http.StatusUnauthorized { + t.Fatalf("cross-tenant acquire status = %d, want 401", spoofRec.Code) + } + + reef := map[string]string{"hive_id": "reef", "agent_name": "reviewer"} + busyRec := serveSharedPoolRequest(t, pool, s, http.MethodPost, "reef", reef) + if busyRec.Code != http.StatusConflict { + t.Fatalf("second tenant acquire status = %d, want 409", busyRec.Code) + } + + restored := &sharedAgentPool{path: path, max: 1, ttl: time.Minute, leases: make(map[string]sharedAgentLease)} + restored.load() + if restored.loadError != nil || len(restored.leases) != 1 { + t.Fatalf("restored pool: leases=%d loadError=%v", len(restored.leases), restored.loadError) + } + lease := restored.leases[sharedPoolOwnerKey("school", "scanner")] + if lease.ID != first.LeaseID { + t.Fatalf("restored lease ID = %q, want %q", lease.ID, first.LeaseID) + } + + release := map[string]string{"hive_id": "school", "lease_id": first.LeaseID} + releaseRec := serveSharedPoolRequest(t, restored, s, http.MethodDelete, "school", release) + if releaseRec.Code != http.StatusNoContent { + t.Fatalf("release status = %d, want 204 (%s)", releaseRec.Code, releaseRec.Body.String()) + } + + acquireRec := serveSharedPoolRequest(t, restored, s, http.MethodPost, "reef", reef) + if acquireRec.Code != http.StatusOK { + t.Fatalf("acquire after release status = %d, want 200 (%s)", acquireRec.Code, acquireRec.Body.String()) + } +} + +func TestSharedAgentPoolHTTPSerializesTwoHives(t *testing.T) { + t.Setenv("HIVE_SHARED_AGENT_POOL_MAX_SESSIONS", "1") + srv := newHubServerForTest(t) + ts := httptest.NewServer(srv.mux) + t.Cleanup(ts.Close) + + school, err := cruxagent.NewSharedSessionPoolClient(ts.URL, "school", srv.heartbeatKeyFor("school")) + if err != nil { + t.Fatal(err) + } + reef, err := cruxagent.NewSharedSessionPoolClient(ts.URL, "reef", srv.heartbeatKeyFor("reef")) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + lease, err := school.Acquire(ctx, "scanner") + if err != nil { + t.Fatalf("school acquire: %v", err) + } + + busyCtx, busyCancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer busyCancel() + if _, err := reef.Acquire(busyCtx, "reviewer"); err == nil { + t.Fatal("reef acquired a second live CLI slot while school held the only slot") + } + if err := school.Release(ctx, lease); err != nil { + t.Fatalf("school release: %v", err) + } + reefCtx, reefCancel := context.WithTimeout(context.Background(), time.Second) + defer reefCancel() + if _, err := reef.Acquire(reefCtx, "reviewer"); err != nil { + t.Fatalf("reef acquire after school release: %v", err) + } +} From 25089b0349e5e1961dbdbed785bcd0991cc54170 Mon Sep 17 00:00:00 2001 From: James Reilly Date: Tue, 27 Oct 2026 07:09:08 +0000 Subject: [PATCH 2/8] fix(agent): auto-start stopped agents on kick when shared pool is enabled Signed-off-by: James Reilly --- src/crux/agent/kick_async.go | 25 ++++++++++++++++++++----- src/crux/agent/manager_kick.go | 15 +++++++++++++-- src/crux/agent/shared_pool.go | 4 ++-- 3 files changed, 35 insertions(+), 9 deletions(-) diff --git a/src/crux/agent/kick_async.go b/src/crux/agent/kick_async.go index 6305b19a9b..edfc47e90c 100644 --- a/src/crux/agent/kick_async.go +++ b/src/crux/agent/kick_async.go @@ -223,7 +223,9 @@ func (m *Manager) SendKickAsync(name string, message string) (started bool, err func (m *Manager) validatePaneKickLocked(agent *AgentProcess, name string) error { if agent.State != StateRunning { - return fmt.Errorf("agent %s cannot be kicked: %s", name, notRunningReason(agent)) + if m.sharedPool == nil || agent.State != StateStopped || agent.Paused || agent.OperatorStopped { + return fmt.Errorf("agent %s cannot be kicked: %s", name, notRunningReason(agent)) + } } if remaining := m.providerErrorBackoffRemainingLocked(agent, time.Now()); remaining > 0 { class, line := agent.ProviderErrorClass, agent.ProviderErrorLine @@ -233,7 +235,7 @@ func (m *Manager) validatePaneKickLocked(agent *AgentProcess, name string) error if err := m.restartKickHoldErrLocked(agent, time.Now()); err != nil { return err } - if !m.tmuxSessionExistsForAgent(agent) { + if agent.State == StateRunning && !m.tmuxSessionExistsForAgent(agent) { return fmt.Errorf("tmux session %s not found", agent.tmuxSession) } return nil @@ -255,9 +257,22 @@ func (m *Manager) deliverKickAsync(name, message string) error { return fmt.Errorf("agent %s not found", name) } if agent.State != StateRunning { - reason := notRunningReason(agent) - m.mu.Unlock() - return fmt.Errorf("agent %s cannot be kicked: %s", name, reason) + if m.sharedPool != nil && agent.State == StateStopped && !agent.Paused && !agent.OperatorStopped { + m.mu.Unlock() + if err := m.Start(context.Background(), name); err != nil { + return fmt.Errorf("auto-start stopped agent for kick: %w", err) + } + m.mu.Lock() + agent, ok = m.agents[name] + if !ok || agent.State != StateRunning { + m.mu.Unlock() + return fmt.Errorf("agent %s failed to enter running state", name) + } + } else { + reason := notRunningReason(agent) + m.mu.Unlock() + return fmt.Errorf("agent %s cannot be kicked: %s", name, reason) + } } if remaining := m.providerErrorBackoffRemainingLocked(agent, time.Now()); remaining > 0 { class, line := agent.ProviderErrorClass, agent.ProviderErrorLine diff --git a/src/crux/agent/manager_kick.go b/src/crux/agent/manager_kick.go index 122f0a7c2d..7a97a44e3c 100644 --- a/src/crux/agent/manager_kick.go +++ b/src/crux/agent/manager_kick.go @@ -128,10 +128,21 @@ func (m *Manager) SendKick(name string, message string) (err error) { } if agent.State != StateRunning { - if m.deferStartupKickLocked(agent, message) { + if m.sharedPool != nil && agent.State == StateStopped && !agent.Paused && !agent.OperatorStopped { + m.mu.Unlock() + if err := m.Start(context.Background(), name); err != nil { + return fmt.Errorf("auto-start stopped agent for kick: %w", err) + } + m.mu.Lock() + agent, ok = m.agents[name] + if !ok || agent.State != StateRunning { + return fmt.Errorf("agent %s failed to enter running state", name) + } + } else if m.deferStartupKickLocked(agent, message) { return nil + } else { + return fmt.Errorf("agent %s cannot be kicked: %s", name, notRunningReason(agent)) } - return fmt.Errorf("agent %s cannot be kicked: %s", name, notRunningReason(agent)) } if m.deferStartupKickLocked(agent, message) { return nil diff --git a/src/crux/agent/shared_pool.go b/src/crux/agent/shared_pool.go index a8093944d9..fbe6bb800e 100644 --- a/src/crux/agent/shared_pool.go +++ b/src/crux/agent/shared_pool.go @@ -186,8 +186,8 @@ func (m *Manager) evictSharedPoolSessionIf(name, leaseID string, expected *KickO return } if expected != nil && - (a.KickOutcome.At != expected.At || - a.KickOutcome.KickAt != expected.KickAt || + (!a.KickOutcome.At.Equal(expected.At) || + !a.KickOutcome.KickAt.Equal(expected.KickAt) || a.KickOutcome.Kind != expected.Kind || a.KickOutcome.Reason != expected.Reason || !a.KickOutcome.Settled(a.LastKick) || From 7f827565b3c84c219ff85acaad1dc2956601f90f Mon Sep 17 00:00:00 2001 From: James Reilly Date: Tue, 27 Oct 2026 12:59:45 +0000 Subject: [PATCH 3/8] fix(hub,agent): address review feedback on waiter livelock, sleep ratchet, and api-reference citations Signed-off-by: James Reilly --- src/crux/agent/manager.go | 16 ++++-- src/crux/agent/shared_pool_test.go | 26 ++------- src/docs/api-reference.md | 55 +++++++++--------- src/pkg/hub/shared_agent_pool.go | 57 +++++++++++++------ src/pkg/hub/shared_agent_pool_test.go | 80 +++++++++++++++++++++++++++ 5 files changed, 166 insertions(+), 68 deletions(-) diff --git a/src/crux/agent/manager.go b/src/crux/agent/manager.go index 189fefe183..44de27a861 100644 --- a/src/crux/agent/manager.go +++ b/src/crux/agent/manager.go @@ -1191,9 +1191,7 @@ func (m *Manager) Start(ctx context.Context, name string) error { agent.poolLeaseID = leaseID launchSucceeded = true if !hasLeaseWatcher { - leaseCtx, cancel := context.WithCancel(context.Background()) - agent.poolLeaseCancel = cancel - go m.watchSharedPoolLease(leaseCtx, name, leaseID) + m.startSharedPoolLeaseWatcherLocked(agent, name, leaseID) } } return nil @@ -1206,14 +1204,20 @@ func (m *Manager) Start(ctx context.Context, name string) error { if err == nil && leaseID != "" { launchSucceeded = true if !hasLeaseWatcher { - leaseCtx, cancel := context.WithCancel(context.Background()) - agent.poolLeaseCancel = cancel - go m.watchSharedPoolLease(leaseCtx, name, leaseID) + m.startSharedPoolLeaseWatcherLocked(agent, name, leaseID) } } return err } +func (m *Manager) startSharedPoolLeaseWatcherLocked(agent *AgentProcess, name, leaseID string) { + if agent.poolLeaseCancel == nil { + leaseCtx, cancel := context.WithCancel(context.Background()) + agent.poolLeaseCancel = cancel + go m.watchSharedPoolLease(leaseCtx, name, leaseID) + } +} + // kickOutcomePollEvery throttles the post-kick turn-ended check to one visible // pane capture per this many 3s poll ticks (15s), so a long turn does not // cost an extra tmux exec every tick. diff --git a/src/crux/agent/shared_pool_test.go b/src/crux/agent/shared_pool_test.go index 2e063edb4c..d0d5702923 100644 --- a/src/crux/agent/shared_pool_test.go +++ b/src/crux/agent/shared_pool_test.go @@ -2,11 +2,11 @@ package agent import ( "context" + "github.com/hivecommons/hive/crux/config" + "github.com/hivecommons/hive/crux/internal/testutil" "sync" "testing" "time" - - "github.com/hivecommons/hive/crux/config" ) type recordingSharedSessionPool struct { @@ -110,25 +110,11 @@ func TestSharedPoolKickOutcomeEvictsSession(t *testing.T) { m.notifyKickOutcome("scanner", outcome) - // Wait briefly for the asynchronous eviction goroutine to complete. - deadline := time.Now().Add(2 * time.Second) - for time.Now().Before(deadline) { + testutil.EventuallyEvery(t, 2*time.Second, 10*time.Millisecond, func() bool { m.mu.RLock() - st := a.State - lease := a.poolLeaseID - m.mu.RUnlock() - if st == StateStopped && lease == "" { - break - } - time.Sleep(10 * time.Millisecond) - } - - m.mu.RLock() - if a.State != StateStopped || a.poolLeaseID != "" { - t.Fatalf("after outcome: state=%q lease=%q, want stopped/empty", a.State, a.poolLeaseID) - } - m.mu.RUnlock() - + defer m.mu.RUnlock() + return a.State == StateStopped && a.poolLeaseID == "" + }, "session was not evicted after kick outcome") pool.mu.Lock() defer pool.mu.Unlock() if len(pool.released) != 1 || pool.released[0] != "lease-outcome" { diff --git a/src/docs/api-reference.md b/src/docs/api-reference.md index 83c4fe36d4..763ca9ff5e 100644 --- a/src/docs/api-reference.md +++ b/src/docs/api-reference.md @@ -725,31 +725,34 @@ always resolved server-side from the validated token. | `GET` | `/api/hub/clusters` | Hub auth | List Clusters | `pkg/hub/saas.go:528` | | `GET` | `/api/hub/image-pulls` | Hub auth | Per-Release Image Pull Series | `pkg/hub/saas.go:377` | | `GET` | `/api/reach` | Hub admin | PR Reach Report (?pr=NNN or ?recent=K) | `pkg/hub/saas.go:519` | -| `GET` | `/fleet` | Hub handler-specific | My-Hives Fleet Page (static; data via `/api/saas/my-hives`) | `pkg/hub/server.go:1761` | -| `GET` | `/my-hives` | Hub handler-specific | 301 redirect to `/fleet` (query preserved) | `pkg/hub/server.go:1762` | -| `POST` | `/api/heartbeat` | Hub handler-specific | Heartbeat | `pkg/hub/server.go:1702` | -| `POST` | `/api/task-status` | Hub handler-specific | Task Status | `pkg/hub/server.go:1703` | -| `GET` | `/api/registry` | Hub handler-specific | Registry | `pkg/hub/server.go:1711` | -| `GET` | `/api/hub/leaderboard` | Hub handler-specific | Leaderboard | `pkg/hub/server.go:1712` | -| `GET` | `/api/hub/stats` | Hub handler-specific | Stats | `pkg/hub/server.go:1713` | -| `GET` | `/api/fleet-stats` | Hub handler-specific | Fleet Stats | `pkg/hub/server.go:1714` | +| `GET` | `/fleet` | Hub handler-specific | My-Hives Fleet Page (static; data via `/api/saas/my-hives`) | `pkg/hub/server.go:1769` | +| `GET` | `/my-hives` | Hub handler-specific | 301 redirect to `/fleet` (query preserved) | `pkg/hub/server.go:1770` | +| `POST` | `/api/shared-agent-pool/leases` | Hub handler-specific | Acquire or renew a shared CLI session lease | `pkg/hub/server.go:1707` | +| `PUT` | `/api/shared-agent-pool/leases` | Hub handler-specific | Renew an active shared CLI session lease | `pkg/hub/server.go:1708` | +| `DELETE` | `/api/shared-agent-pool/leases` | Hub handler-specific | Release a shared CLI session lease | `pkg/hub/server.go:1709` | +| `POST` | `/api/heartbeat` | Hub handler-specific | Heartbeat | `pkg/hub/server.go:1710` | +| `POST` | `/api/task-status` | Hub handler-specific | Task Status | `pkg/hub/server.go:1711` | +| `GET` | `/api/registry` | Hub handler-specific | Registry | `pkg/hub/server.go:1719` | +| `GET` | `/api/hub/leaderboard` | Hub handler-specific | Leaderboard | `pkg/hub/server.go:1720` | +| `GET` | `/api/hub/stats` | Hub handler-specific | Stats | `pkg/hub/server.go:1721` | +| `GET` | `/api/fleet-stats` | Hub handler-specific | Fleet Stats | `pkg/hub/server.go:1722` | | `GET` | `/api/hub/evidence` | Hub admin | Read-only cross-hive evidence drill-down from each spoke's stored heartbeat `evidence` summary ([#12116](https://github.com/hivecommons/hive/issues/12116)); non-admins get `403`. Returns `spokes`, `reporting`, pooled `calibration` (bucket 0–5 → `{n, bad, rate, lo, hi}`; the hub view folds the six buckets into four reviewer-confidence ranges), `findings[]` (the plain-language *What this says* sentences, generated from the same fields; [#12456](https://github.com/hivecommons/hive/issues/12456)), `by_acmm_level` (`level`, `hives`, `merged_90d`, `revert_rate`, and `time_to_merge` — `median_of_hive_medians_s`, `hives`, `n` — the median of the level's per-hive median time-to-merge, omitted when no hive at the level reports one; [#12464](https://github.com/hivecommons/hive/issues/12464)), `family_false_approve_rate` (`same`/`cross`/`mixed`, hives classed by their reviewer vs author family sets), fleet `record_coverage` (merged PRs with an evidence record over `prs_merged_90d`), fleet panels ([#12370](https://github.com/hivecommons/hive/issues/12370)) `outcomes_by_author_kind[]` (`kind`, `hives`, `n`, `merged`, `reverted`, `merge_rate`, `revert_rate`), `write_counters[]` (`tier`, `hives`, `attempted`, `blocked`, `block_rate`) and `shadow_gate` (`review_gate` / `intent` → `held` / `passed` outcomes with `revert_rate`), each omitted while no spoke reports it and repeated per `by_acmm_level[]` row, and per hive `hives[]` with `acmm_level`, `prs_merged_90d`, `record_coverage`, `family_mix`, false-approve / false-block / revert rates, families, a calibration series the raw `counts` behind them (including `by_author_kind`) so the drill-down can re-pool a hive / ACMM-level selection client-side, and `criteria[]`: the per-axis criteria table derived hub-side from the quadrant criteria (`axis`, `label`, `scored`, `criteria[]` with `name`, `label`, `status` `met`/`unmet`/`not_reported`, `display`/`score` when measured, `reason` otherwise; offline, evidence-less or unscored hives are `not_reported`, never `unmet`; [#12463](https://github.com/hivecommons/hive/issues/12463)). Every rate is `{n, rate, lo, hi}` with a 95% Wilson interval; `rate`/`lo`/`hi` are null when `n` < 10. Window selection ([RFC #12062](https://github.com/hivecommons/hive/issues/12062)): `?hours=24|168|720` pools each hive's matching heartbeat `windows` block instead of its 90-day block (any other or absent `hours` → 90 days); `?from=YYYY-MM-DD|RFC3339[&to=…]` (date-only `to` is inclusive, `to` defaults to now, `to` ≥ `from`, span ≤ 366 days, else `400`) is served from the smallest reported window that reaches back to `from` and flagged `window.approximate`. The response carries `window` (`hours`, `key`, `label`, `from`, `to`, `approximate`, `coverage_available` — `record_coverage` is only computed for the 90-day window because `prs_merged_90d` is a 90-day count), `window_unreported` and per-hive `window_unreported` for hives whose spoke predates per-window reporting. `?hive=[,…]` (repeatable) and `?level=` restrict the registry server-side to those hives / that ACMM level before pooling. Hub admins open it from the Fleet quadrant card or its **🔬 Evidence** button ([#12117](https://github.com/hivecommons/hive/issues/12117)), pick a window (24h / 7d / 30d / 90d / custom), filter it by hive and ACMM level and read each section's ⓘ explainer; the per-hive author-kind revert counts behind the *Revert rate by author kind* panel are `hives[].counts.by_author_kind` ([#12326](https://github.com/hivecommons/hive/issues/12326)). Never consulted by merge decisions | `pkg/hub/evidence.go` | | `GET` | `/api/hub/evidence/report` | Hub admin | Plain-language report of the hub evidence view ([#12456](https://github.com/hivecommons/hive/issues/12456)), rendered server-side from the same `EvidenceView` as `/api/hub/evidence` so every figure reconciles with the modal; non-admins get `403`. Same `hours=` / `from=` / `to=` window and `hive=` / `level=` filter parameters. `format=md` (default, `text/markdown`) or `format=html` (standalone page, inline CSS, no scripts or external assets, hive names escaped); any other value → `400`. `download=1` sets `Content-Disposition: attachment; filename="hive-evidence---.md|html"`, otherwise inline. Sections: Summary (the generated findings), What we measured and why, The numbers, How to read these, What is not reported yet and why, Grade per section, footer link to `review-evidence.md`. `Cache-Control: no-store`. Read-only; never consulted by merge decisions | `pkg/hub/evidence_report.go` | -| `GET` | `/api/hub/version` | Hub handler-specific | Hub Version | `pkg/hub/server.go:1719` | -| `DELETE` | `/api/hub/registry/{id}` | Hub handler-specific | Registry Delete | `pkg/hub/server.go:1729` | -| `POST` | `/api/contribute/register` | Hub handler-specific | Contribute Proxy | `pkg/hub/server.go:1730` | -| `GET` | `/api/contribute/status` | Hub handler-specific | Contribute Status | `pkg/hub/server.go:1731` | -| `GET` | `/api/contribute/ws` | Hub handler-specific | Contribute WSProxy | `pkg/hub/server.go:1732` | -| `POST` | `/api/github/webhook` | Hub handler-specific | GitHub Webhook | `pkg/hub/server.go:1733` | -| `GET` | `/gh-setup` | Hub handler-specific | GitHub App Setup Router | `pkg/hub/server.go:1734` | -| `GET` | `/learn` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1735` | -| `GET` | `/get-started` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1736` | -| `GET` | `/api/docs` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1739` | -| `GET` | `/api/reading-list` | Hub handler-specific | Reading List | `pkg/hub/server.go:1742` | -| `GET` | `/reading` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1743` | -| `GET` | `/cncf-reference-architecture` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1750` | -| `GET` | `/cncf-reference-architecture/console` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1751` | -| `GET` | `/cncf-reference-architecture/bluefin` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1752` | -| `GET` | `/{$}` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1769` | -| `GET` | `/og-card.png` | Hub handler-specific | OGCard | `pkg/hub/server.go:1774` | -| `GET` | `/` | Public | Static asset fallback (`http.FileServerFS` over the embedded `static/` tree) for any path no other route claims | `pkg/hub/server.go:1776` | +| `GET` | `/api/hub/version` | Hub handler-specific | Hub Version | `pkg/hub/server.go:1727` | +| `DELETE` | `/api/hub/registry/{id}` | Hub handler-specific | Registry Delete | `pkg/hub/server.go:1737` | +| `POST` | `/api/contribute/register` | Hub handler-specific | Contribute Proxy | `pkg/hub/server.go:1738` | +| `GET` | `/api/contribute/status` | Hub handler-specific | Contribute Status | `pkg/hub/server.go:1739` | +| `GET` | `/api/contribute/ws` | Hub handler-specific | Contribute WSProxy | `pkg/hub/server.go:1740` | +| `POST` | `/api/github/webhook` | Hub handler-specific | GitHub Webhook | `pkg/hub/server.go:1741` | +| `GET` | `/gh-setup` | Hub handler-specific | GitHub App Setup Router | `pkg/hub/server.go:1742` | +| `GET` | `/learn` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1743` | +| `GET` | `/get-started` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1744` | +| `GET` | `/api/docs` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1747` | +| `GET` | `/api/reading-list` | Hub handler-specific | Reading List | `pkg/hub/server.go:1750` | +| `GET` | `/reading` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1751` | +| `GET` | `/cncf-reference-architecture` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1758` | +| `GET` | `/cncf-reference-architecture/console` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1759` | +| `GET` | `/cncf-reference-architecture/bluefin` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1760` | +| `GET` | `/{$}` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1777` | +| `GET` | `/og-card.png` | Hub handler-specific | OGCard | `pkg/hub/server.go:1782` | +| `GET` | `/` | Public | Static asset fallback (`http.FileServerFS` over the embedded `static/` tree) for any path no other route claims | `pkg/hub/server.go:1784` | diff --git a/src/pkg/hub/shared_agent_pool.go b/src/pkg/hub/shared_agent_pool.go index 2dea511f51..10d649d73e 100644 --- a/src/pkg/hub/shared_agent_pool.go +++ b/src/pkg/hub/shared_agent_pool.go @@ -19,6 +19,10 @@ var sharedAgentPoolPath = "/data/saas/shared-agent-pool.json" const defaultSharedAgentLeaseTTL = 90 * time.Second +// A waiter that has not retried within this window is treated as gone so a +// crashed spoke cannot starve the fleet. +const sharedPoolWaiterStaleAfter = 15 * time.Second + type sharedAgentLease struct { ID string `json:"id"` HiveID string `json:"hive_id"` @@ -37,7 +41,8 @@ type sharedAgentPool struct { max int ttl time.Duration leases map[string]sharedAgentLease - waitingHives map[string]struct{} + waitingHives map[string]time.Time // hive_id -> first 409 (FIFO position) + lastSeen map[string]time.Time // hive_id -> last retry loadError error } @@ -52,7 +57,8 @@ func newSharedAgentPool() *sharedAgentPool { max: max, ttl: ttl, leases: make(map[string]sharedAgentLease), - waitingHives: make(map[string]struct{}), + waitingHives: make(map[string]time.Time), + lastSeen: make(map[string]time.Time), } p.load() return p @@ -106,9 +112,27 @@ func (p *sharedAgentPool) saveLocked() error { return nil } -func (p *sharedAgentPool) hasOtherWaitingHive(hiveID string) bool { - for waitingHive := range p.waitingHives { - if waitingHive != hiveID { +// hasEarlierWaitingHive reports whether another hive has been waiting longer +// than hiveID (which may not be waiting yet). Stale waiters are pruned first. +func (p *sharedAgentPool) hasEarlierWaitingHive(hiveID string, now time.Time) bool { + if p.waitingHives == nil { + p.waitingHives = make(map[string]time.Time) + } + if p.lastSeen == nil { + p.lastSeen = make(map[string]time.Time) + } + for h := range p.waitingHives { + if now.Sub(p.lastSeen[h]) > sharedPoolWaiterStaleAfter { + delete(p.waitingHives, h) + delete(p.lastSeen, h) + } + } + mySince, waiting := p.waitingHives[hiveID] + if !waiting { + mySince = now + } + for h, since := range p.waitingHives { + if h != hiveID && since.Before(mySince) { return true } } @@ -125,11 +149,6 @@ func (p *sharedAgentPool) handle(w http.ResponseWriter, r *http.Request, s *HubS http.Error(w, "shared agent pool state unavailable", http.StatusServiceUnavailable) return } - if r.Method != http.MethodPost && r.Method != http.MethodPut && r.Method != http.MethodDelete { - w.Header().Set("Allow", "POST, PUT, DELETE") - http.Error(w, "method not allowed", http.StatusMethodNotAllowed) - return - } auth := r.Header.Get("Authorization") if !strings.HasPrefix(auth, "Bearer ") { http.Error(w, "unauthorized", http.StatusUnauthorized) @@ -168,6 +187,7 @@ func (p *sharedAgentPool) handle(w http.ResponseWriter, r *http.Request, s *HubS lease.ExpiresAt = now.Add(p.ttl) p.leases[key] = lease delete(p.waitingHives, req.HiveID) + delete(p.lastSeen, req.HiveID) if err := p.saveLocked(); err != nil { http.Error(w, "shared agent pool persistence failed", http.StatusServiceUnavailable) return @@ -175,11 +195,17 @@ func (p *sharedAgentPool) handle(w http.ResponseWriter, r *http.Request, s *HubS writeSharedPoolLease(w, lease.ID) return } - if len(p.leases) >= p.max || p.hasOtherWaitingHive(req.HiveID) { + if len(p.leases) >= p.max || p.hasEarlierWaitingHive(req.HiveID, now) { if p.waitingHives == nil { - p.waitingHives = make(map[string]struct{}) + p.waitingHives = make(map[string]time.Time) + } + if p.lastSeen == nil { + p.lastSeen = make(map[string]time.Time) + } + if _, ok := p.waitingHives[req.HiveID]; !ok { + p.waitingHives[req.HiveID] = now } - p.waitingHives[req.HiveID] = struct{}{} + p.lastSeen[req.HiveID] = now w.Header().Set("Retry-After", "3") http.Error(w, "shared agent pool is busy", http.StatusConflict) return @@ -192,6 +218,7 @@ func (p *sharedAgentPool) handle(w http.ResponseWriter, r *http.Request, s *HubS lease := sharedAgentLease{ID: hex.EncodeToString(raw[:]), HiveID: req.HiveID, AgentName: req.AgentName, ExpiresAt: now.Add(p.ttl)} p.leases[key] = lease delete(p.waitingHives, req.HiveID) + delete(p.lastSeen, req.HiveID) if err := p.saveLocked(); err != nil { delete(p.leases, key) http.Error(w, "shared agent pool persistence failed", http.StatusServiceUnavailable) @@ -201,6 +228,7 @@ func (p *sharedAgentPool) handle(w http.ResponseWriter, r *http.Request, s *HubS case http.MethodPut, http.MethodDelete: if r.Method == http.MethodDelete && req.CancelWait { delete(p.waitingHives, req.HiveID) + delete(p.lastSeen, req.HiveID) w.WriteHeader(http.StatusNoContent) return } @@ -231,9 +259,6 @@ func (p *sharedAgentPool) handle(w http.ResponseWriter, r *http.Request, s *HubS return } w.WriteHeader(http.StatusNoContent) - default: - w.Header().Set("Allow", "POST, PUT, DELETE") - http.Error(w, "method not allowed", http.StatusMethodNotAllowed) } } diff --git a/src/pkg/hub/shared_agent_pool_test.go b/src/pkg/hub/shared_agent_pool_test.go index 03b73c9f3f..745f9db6bd 100644 --- a/src/pkg/hub/shared_agent_pool_test.go +++ b/src/pkg/hub/shared_agent_pool_test.go @@ -114,3 +114,83 @@ func TestSharedAgentPoolHTTPSerializesTwoHives(t *testing.T) { t.Fatalf("reef acquire after school release: %v", err) } } + +func TestSharedAgentPoolFIFOWaitersAndStaleExpiry(t *testing.T) { + path := filepath.Join(t.TempDir(), "shared-agent-pool.json") + pool := &sharedAgentPool{ + path: path, + max: 1, + ttl: time.Minute, + leases: make(map[string]sharedAgentLease), + waitingHives: make(map[string]time.Time), + lastSeen: make(map[string]time.Time), + } + s := &HubServer{logger: slog.Default()} + s.setHubSecret("test-fifo-waiters-master") + + h0 := map[string]string{"hive_id": "h0", "agent_name": "a"} + h1 := map[string]string{"hive_id": "h1", "agent_name": "b"} + h2 := map[string]string{"hive_id": "h2", "agent_name": "c"} + + // h0 acquires the slot. + r0 := serveSharedPoolRequest(t, pool, s, http.MethodPost, "h0", h0) + if r0.Code != http.StatusOK { + t.Fatalf("h0 acquire status = %d", r0.Code) + } + var lease0 struct { + LeaseID string `json:"lease_id"` + } + _ = json.Unmarshal(r0.Body.Bytes(), &lease0) + + // h1 tries and gets 409, becomes first waiter. + r1 := serveSharedPoolRequest(t, pool, s, http.MethodPost, "h1", h1) + if r1.Code != http.StatusConflict { + t.Fatalf("h1 status = %d, want 409", r1.Code) + } + time.Sleep(10 * time.Millisecond) + + // h2 tries and gets 409, becomes second waiter. + r2 := serveSharedPoolRequest(t, pool, s, http.MethodPost, "h2", h2) + if r2.Code != http.StatusConflict { + t.Fatalf("h2 status = %d, want 409", r2.Code) + } + + // h0 releases. + rel := map[string]string{"hive_id": "h0", "lease_id": lease0.LeaseID} + rRel := serveSharedPoolRequest(t, pool, s, http.MethodDelete, "h0", rel) + if rRel.Code != http.StatusNoContent { + t.Fatalf("h0 release status = %d", rRel.Code) + } + + // h2 tries to acquire before h1, but h1 is earlier in queue -> 409. + r2Blocked := serveSharedPoolRequest(t, pool, s, http.MethodPost, "h2", h2) + if r2Blocked.Code != http.StatusConflict { + t.Fatalf("h2 status = %d, want 409 (h1 was earlier waiter)", r2Blocked.Code) + } + + // h1 acquires -> 200 OK. + r1Acq := serveSharedPoolRequest(t, pool, s, http.MethodPost, "h1", h1) + if r1Acq.Code != http.StatusOK { + t.Fatalf("h1 acquire status = %d, want 200", r1Acq.Code) + } + var lease1 struct { + LeaseID string `json:"lease_id"` + } + _ = json.Unmarshal(r1Acq.Body.Bytes(), &lease1) + + // h1 releases. + rel1 := map[string]string{"hive_id": "h1", "lease_id": lease1.LeaseID} + serveSharedPoolRequest(t, pool, s, http.MethodDelete, "h1", rel1) + + // Now make h2 stale (simulate h2 crashing and not retrying for > 15s). + pool.mu.Lock() + pool.lastSeen["h2"] = time.Now().Add(-20 * time.Second) + pool.mu.Unlock() + + // Fresh hive h3 can acquire because h2 is stale and pruned! + h3 := map[string]string{"hive_id": "h3", "agent_name": "d"} + r3 := serveSharedPoolRequest(t, pool, s, http.MethodPost, "h3", h3) + if r3.Code != http.StatusOK { + t.Fatalf("h3 acquire status = %d, want 200 after stale waiter pruned", r3.Code) + } +} From f4e222e634f65e5ebcea811d3dc42c81a4f9e298 Mon Sep 17 00:00:00 2001 From: James Reilly Date: Tue, 27 Oct 2026 13:08:35 +0000 Subject: [PATCH 4/8] fix(agent): bound acquire timeout to prevent boot deadlock when pool is full Signed-off-by: James Reilly --- src/cmd/hive/eval_cycle_seams.go | 4 +++- src/crux/agent/shared_pool.go | 14 ++++++++++---- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/src/cmd/hive/eval_cycle_seams.go b/src/cmd/hive/eval_cycle_seams.go index fa5d42b806..8ece747bb6 100644 --- a/src/cmd/hive/eval_cycle_seams.go +++ b/src/cmd/hive/eval_cycle_seams.go @@ -644,7 +644,9 @@ func stoppedAgentStartErrorIsBenign(err error) bool { return true } msg := err.Error() - return strings.Contains(msg, "already running") || strings.Contains(msg, "launch already in progress") + return strings.Contains(msg, "already running") || + strings.Contains(msg, "launch already in progress") || + strings.Contains(msg, "shared session pool is busy") } // advisoryIngestDeps are the side effects of recording an ioscan canary leak diff --git a/src/crux/agent/shared_pool.go b/src/crux/agent/shared_pool.go index fbe6bb800e..2137b7881c 100644 --- a/src/crux/agent/shared_pool.go +++ b/src/crux/agent/shared_pool.go @@ -93,8 +93,14 @@ func (c *sharedSessionPoolClient) request(ctx context.Context, method, path stri } func (c *sharedSessionPoolClient) Acquire(ctx context.Context, agentName string) (string, error) { + deadlineCtx := ctx + if _, ok := ctx.Deadline(); !ok { + var cancel context.CancelFunc + deadlineCtx, cancel = context.WithTimeout(ctx, 5*time.Second) + defer cancel() + } for { - result, err := c.request(ctx, http.MethodPost, "/leases", sharedPoolRequest{HiveID: c.hiveID, AgentName: agentName}) + result, err := c.request(deadlineCtx, http.MethodPost, "/leases", sharedPoolRequest{HiveID: c.hiveID, AgentName: agentName}) if err == nil { if result.LeaseID == "" { return "", fmt.Errorf("shared session pool returned an empty lease ID") @@ -104,14 +110,14 @@ func (c *sharedSessionPoolClient) Acquire(ctx context.Context, agentName string) if err != errSharedSessionPoolBusy { return "", err } - timer := time.NewTimer(3 * time.Second) + timer := time.NewTimer(1 * time.Second) select { - case <-ctx.Done(): + case <-deadlineCtx.Done(): timer.Stop() cancelCtx, cancel := context.WithTimeout(context.Background(), 2*time.Second) _, _ = c.request(cancelCtx, http.MethodDelete, "/leases", sharedPoolRequest{HiveID: c.hiveID, CancelWait: true}) cancel() - return "", ctx.Err() + return "", fmt.Errorf("shared session pool is busy: %w", deadlineCtx.Err()) case <-timer.C: } } From 45934ccfc66fb15968c156f289444545c58404ce Mon Sep 17 00:00:00 2001 From: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Wed, 28 Oct 2026 14:55:48 -0400 Subject: [PATCH 5/8] fix(ratchets): route shared pool client through crux/runtime; drop test sleep cmd/hive named crux/agent directly for NewSharedSessionPoolClient, which TestCruxBoundaryRatchet counts as new consumer coupling. Re-export the SharedSessionPool contract and constructor from crux/runtime so the host configures the pool through the runtime seam. The hub FIFO-waiter test slept 10ms to order two 409s; back-date the first waiter's recorded timestamp under the pool mutex instead, keeping TestSleepRatchet at its baseline. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- src/cmd/hive/main.go | 3 +-- src/crux/runtime/runtime.go | 14 +++++++++++++- src/crux/runtime/runtime_test.go | 4 ++-- src/pkg/hub/shared_agent_pool_test.go | 6 +++++- 4 files changed, 21 insertions(+), 6 deletions(-) diff --git a/src/cmd/hive/main.go b/src/cmd/hive/main.go index 7ef08fe682..784c626aba 100644 --- a/src/cmd/hive/main.go +++ b/src/cmd/hive/main.go @@ -27,7 +27,6 @@ import ( gh "github.com/google/go-github/v72/github" "github.com/hivecommons/hive/crux" - cruxagent "github.com/hivecommons/hive/crux/agent" "github.com/hivecommons/hive/crux/inference" "github.com/hivecommons/hive/crux/logscrub" "github.com/hivecommons/hive/crux/proxy" @@ -2203,7 +2202,7 @@ func (b *boot) bootAgentsWith(deps bootAgentsDeps) { if hubURL == "" { hubURL = b.cfg.Hub.URL } - pool, poolErr := cruxagent.NewSharedSessionPoolClient(hubURL, b.cfg.HiveID, hub.SpokeHeartbeatKey()) + pool, poolErr := cruxruntime.NewSharedSessionPoolClient(hubURL, b.cfg.HiveID, hub.SpokeHeartbeatKey()) if poolErr != nil { b.logger.Error("shared agent pool is enabled but cannot be configured", "error", poolErr) deps.exit(1) diff --git a/src/crux/runtime/runtime.go b/src/crux/runtime/runtime.go index bad1b89709..5f3070e61e 100644 --- a/src/crux/runtime/runtime.go +++ b/src/crux/runtime/runtime.go @@ -82,10 +82,22 @@ func WithUIDMapPath(path string) Option { return func(o *options) { o.uidMapPath = &path } } +// SharedSessionPool is the lease contract a Hub-coordinated CLI session pool +// satisfies. It is re-exported here so consumers configure the pool through +// the runtime seam without naming crux/agent. +type SharedSessionPool = agent.SharedSessionPool + +// NewSharedSessionPoolClient creates an authenticated client for the Hub's +// tenant-scoped session-slot API. The token must be the per-Hive heartbeat +// key, never the fleet master secret. +func NewSharedSessionPoolClient(baseURL, hiveID, token string) (SharedSessionPool, error) { + return agent.NewSharedSessionPoolClient(baseURL, hiveID, token) +} + // WithSharedSessionPool coordinates heavyweight CLI sessions through the Hub. // It affects only the in-process Manager; sidecar mode must implement the same // lease contract before it can opt in. -func WithSharedSessionPool(pool agent.SharedSessionPool) Option { +func WithSharedSessionPool(pool SharedSessionPool) Option { return func(o *options) { o.sharedPool = pool } } diff --git a/src/crux/runtime/runtime_test.go b/src/crux/runtime/runtime_test.go index d06985fe10..78db1bb0c7 100644 --- a/src/crux/runtime/runtime_test.go +++ b/src/crux/runtime/runtime_test.go @@ -243,8 +243,8 @@ func TestRunAgyTurnHelp(t *testing.T) { type stubSharedSessionPool struct{} func (stubSharedSessionPool) Acquire(context.Context, string) (string, error) { return "lease", nil } -func (stubSharedSessionPool) Renew(context.Context, string) error { return nil } -func (stubSharedSessionPool) Release(context.Context, string) error { return nil } +func (stubSharedSessionPool) Renew(context.Context, string) error { return nil } +func (stubSharedSessionPool) Release(context.Context, string) error { return nil } func TestWithSharedSessionPool(t *testing.T) { t.Setenv("HIVE_WORK_DIR", t.TempDir()) diff --git a/src/pkg/hub/shared_agent_pool_test.go b/src/pkg/hub/shared_agent_pool_test.go index 745f9db6bd..f2ece72dee 100644 --- a/src/pkg/hub/shared_agent_pool_test.go +++ b/src/pkg/hub/shared_agent_pool_test.go @@ -147,7 +147,11 @@ func TestSharedAgentPoolFIFOWaitersAndStaleExpiry(t *testing.T) { if r1.Code != http.StatusConflict { t.Fatalf("h1 status = %d, want 409", r1.Code) } - time.Sleep(10 * time.Millisecond) + // Back-date h1's FIFO position so h2's 409 below is strictly later even + // when both requests land in the same clock tick. + pool.mu.Lock() + pool.waitingHives["h1"] = pool.waitingHives["h1"].Add(-time.Second) + pool.mu.Unlock() // h2 tries and gets 409, becomes second waiter. r2 := serveSharedPoolRequest(t, pool, s, http.MethodPost, "h2", h2) From 71d0138a788d2c71c5a181f7f392c7a9287cb912 Mon Sep 17 00:00:00 2001 From: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Wed, 28 Oct 2026 14:56:22 -0400 Subject: [PATCH 6/8] fix(hub): construct shared pool client through crux/runtime in test pkg/hub's pool test named crux/agent directly; the boundary ratchet counts test references too. Use the crux/runtime re-export instead. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- src/pkg/hub/shared_agent_pool_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pkg/hub/shared_agent_pool_test.go b/src/pkg/hub/shared_agent_pool_test.go index f2ece72dee..585c3fe599 100644 --- a/src/pkg/hub/shared_agent_pool_test.go +++ b/src/pkg/hub/shared_agent_pool_test.go @@ -11,7 +11,7 @@ import ( "testing" "time" - cruxagent "github.com/hivecommons/hive/crux/agent" + cruxruntime "github.com/hivecommons/hive/crux/runtime" ) func serveSharedPoolRequest(t *testing.T, pool *sharedAgentPool, s *HubServer, method, bearerHive string, body map[string]string) *httptest.ResponseRecorder { @@ -85,11 +85,11 @@ func TestSharedAgentPoolHTTPSerializesTwoHives(t *testing.T) { ts := httptest.NewServer(srv.mux) t.Cleanup(ts.Close) - school, err := cruxagent.NewSharedSessionPoolClient(ts.URL, "school", srv.heartbeatKeyFor("school")) + school, err := cruxruntime.NewSharedSessionPoolClient(ts.URL, "school", srv.heartbeatKeyFor("school")) if err != nil { t.Fatal(err) } - reef, err := cruxagent.NewSharedSessionPoolClient(ts.URL, "reef", srv.heartbeatKeyFor("reef")) + reef, err := cruxruntime.NewSharedSessionPoolClient(ts.URL, "reef", srv.heartbeatKeyFor("reef")) if err != nil { t.Fatal(err) } From 80a8f6568d7828877d61aaa521f56e1c2e3dc366 Mon Sep 17 00:00:00 2001 From: ci-maintainer <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Wed, 28 Oct 2026 15:06:47 -0400 Subject: [PATCH 7/8] fix(agent,hub): balance SendKick lock, lease restarts, tolerate renew blips, cap per-hive leases Review fixes for the shared CLI session pool: - SendKick's auto-start branch unlocked m.mu, called Start, and returned on error without re-locking, so the deferred Unlock hit an unlocked mutex (runtime fatal) whenever Acquire timed out on a busy pool. Re-lock before every return. - restartWithReason, RestartMissingSessionWithReason, RestartWithBootstrap and Resume relaunched via launchInTmux without a pool lease, so an evicted agent restarted from the dashboard or recovery ran unleased and uncapped. Acquire in the existing unlocked mint window, bind the lease and start its watcher on success, and return a fresh lease to the Hub when the launch does not happen. - watchSharedPoolLease evicted a live turn on the first Renew error. Tolerate transient failures for 60s after the last successful renewal (three missed ticks, still inside the 90s Hub TTL); a 404 from the Hub (lease reclaimed) still evicts immediately. - Add HIVE_SHARED_AGENT_POOL_MAX_PER_HIVE so a single tenant cannot hold every slot and renew it forever; a Hive at its cap gets 409 without taking a FIFO waiter position that would block other tenants. Hive-Fix-Attempt: 1/6 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: ci-maintainer <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- .../added-shared-agent-session-pool.md | 2 +- src/crux/agent/manager_kick.go | 7 +- src/crux/agent/manager_pause.go | 10 +- src/crux/agent/manager_restart.go | 29 +++- src/crux/agent/shared_pool.go | 97 ++++++++++++- src/crux/agent/shared_pool_test.go | 134 +++++++++++++++++- src/docs/env-vars.md | 3 +- src/pkg/hub/shared_agent_pool.go | 37 ++++- src/pkg/hub/shared_agent_pool_test.go | 50 +++++++ 9 files changed, 355 insertions(+), 14 deletions(-) diff --git a/changelog.d/added-shared-agent-session-pool.md b/changelog.d/added-shared-agent-session-pool.md index 30b6049e64..5c53e72569 100644 --- a/changelog.d/added-shared-agent-session-pool.md +++ b/changelog.d/added-shared-agent-session-pool.md @@ -1 +1 @@ -- Add an opt-in Hub-coordinated CLI session pool for spokes. Participating Hives keep separate workspaces, UIDs, credentials, and tmux sessions; the Hub leases a global live-session slot and idle sessions are terminated after a turn. Configure `HIVE_SHARED_AGENT_POOL_ENABLED` on spokes and `HIVE_SHARED_AGENT_POOL_MAX_SESSIONS` on the Hub. +- Add an opt-in Hub-coordinated CLI session pool for spokes. Participating Hives keep separate workspaces, UIDs, credentials, and tmux sessions; the Hub leases a global live-session slot and idle sessions are terminated after a turn. Configure `HIVE_SHARED_AGENT_POOL_ENABLED` on spokes and `HIVE_SHARED_AGENT_POOL_MAX_SESSIONS` (optionally `HIVE_SHARED_AGENT_POOL_MAX_PER_HIVE` for a per-tenant cap) on the Hub. diff --git a/src/crux/agent/manager_kick.go b/src/crux/agent/manager_kick.go index 7a97a44e3c..5405e6ec03 100644 --- a/src/crux/agent/manager_kick.go +++ b/src/crux/agent/manager_kick.go @@ -129,11 +129,14 @@ func (m *Manager) SendKick(name string, message string) (err error) { if agent.State != StateRunning { if m.sharedPool != nil && agent.State == StateStopped && !agent.Paused && !agent.OperatorStopped { + // Re-lock before every return: the deferred Unlock above must + // always pair with a held mutex. m.mu.Unlock() - if err := m.Start(context.Background(), name); err != nil { + err := m.Start(context.Background(), name) + m.mu.Lock() + if err != nil { return fmt.Errorf("auto-start stopped agent for kick: %w", err) } - m.mu.Lock() agent, ok = m.agents[name] if !ok || agent.State != StateRunning { return fmt.Errorf("agent %s failed to enter running state", name) diff --git a/src/crux/agent/manager_pause.go b/src/crux/agent/manager_pause.go index c118f2a959..2e45ac1dd5 100644 --- a/src/crux/agent/manager_pause.go +++ b/src/crux/agent/manager_pause.go @@ -227,17 +227,25 @@ func (m *Manager) Resume(ctx context.Context, name, trigger, reason string) erro m.mu.Unlock() m.mintAgentTokenUnlocked(ctx, agent) m.bobPreflightUnlocked(agent) + // The relaunch needs a live-session lease exactly like Start. + leaseID, freshLease, leaseErr := m.acquireSharedPoolLeaseUnlocked(ctx, agent, name) m.mu.Lock() + if leaseErr != nil { + m.mu.Unlock() + return leaseErr + } if cur, ok := m.agents[name]; !ok || cur != agent { + m.abandonFreshSharedPoolLeaseLocked(name, leaseID, freshLease) m.mu.Unlock() return fmt.Errorf("agent %s removed during resume", name) } if agent.State == StateRunning { // Another path won the launch race while we were unlocked. + m.abandonFreshSharedPoolLeaseLocked(name, leaseID, freshLease) m.mu.Unlock() return nil } - err := m.launchInTmux(ctx, agent) + err := m.launchInTmuxLeasedLocked(ctx, agent, name, leaseID, freshLease) m.mu.Unlock() return err } diff --git a/src/crux/agent/manager_restart.go b/src/crux/agent/manager_restart.go index 94eba1c472..2fb358bcfd 100644 --- a/src/crux/agent/manager_restart.go +++ b/src/crux/agent/manager_restart.go @@ -495,6 +495,11 @@ func (m *Manager) RestartWithBootstrap(ctx context.Context, name, prompt string) // never got) a token — see mintAgentTokenUnlocked. m.mintAgentTokenUnlocked(ctx, agent) m.bobPreflightUnlocked(agent) + // The relaunch needs a live-session lease exactly like Start. + leaseID, freshLease, leaseErr := m.acquireSharedPoolLeaseUnlocked(ctx, agent, name) + if leaseErr != nil { + return leaseErr + } // Wait for the new shell to initialize before sending the launch command. // Without this, $(cat /tmp/.hive-bootstrap-*.txt) can fail because the @@ -508,9 +513,10 @@ func (m *Manager) RestartWithBootstrap(ctx context.Context, name, prompt string) // the unlocked window could have deleted this agent from the map (same // guard Start applies after its unlocked phase). if cur, ok := m.agents[name]; !ok || cur != agent { + m.abandonFreshSharedPoolLeaseLocked(name, leaseID, freshLease) return fmt.Errorf("agent %s removed during restart", name) } - return m.launchInTmux(ctx, agent) + return m.launchInTmuxLeasedLocked(ctx, agent, name, leaseID, freshLease) } // RestartThenSendKick restarts the agent with a clean slate (no bootstrap @@ -783,15 +789,23 @@ func (m *Manager) RestartMissingSessionWithReason(ctx context.Context, name, rea m.mu.Unlock() m.mintAgentTokenUnlocked(ctx, agent) m.bobPreflightUnlocked(agent) + // The relaunch needs a live-session lease exactly like Start: an evicted + // agent recovered here must not run unleased and uncapped. + leaseID, freshLease, leaseErr := m.acquireSharedPoolLeaseUnlocked(ctx, agent, name) m.mu.Lock() + if leaseErr != nil { + return false, leaseErr + } if cur, ok := m.agents[name]; !ok || cur != agent { + m.abandonFreshSharedPoolLeaseLocked(name, leaseID, freshLease) return false, fmt.Errorf("agent %s removed during missing-session recovery", name) } if agent.launchGen != genBeforeMint { + m.abandonFreshSharedPoolLeaseLocked(name, leaseID, freshLease) return false, nil } - return true, m.launchInTmux(ctx, agent) + return true, m.launchInTmuxLeasedLocked(ctx, agent, name, leaseID, freshLease) } // restartForKick is the restart a kick delivery performs on its OWN behalf @@ -933,16 +947,25 @@ func (m *Manager) restartWithReason(ctx context.Context, name, reason string, ki m.mu.Unlock() m.mintAgentTokenUnlocked(ctx, agent) m.bobPreflightUnlocked(agent) + // The relaunch needs a live-session lease exactly like Start: an evicted + // agent restarted from the dashboard or recovery must not run unleased + // and uncapped. + leaseID, freshLease, leaseErr := m.acquireSharedPoolLeaseUnlocked(ctx, agent, name) m.mu.Lock() + if leaseErr != nil { + return leaseErr + } if cur, ok := m.agents[name]; !ok || cur != agent { + m.abandonFreshSharedPoolLeaseLocked(name, leaseID, freshLease) return fmt.Errorf("agent %s removed during restart", name) } if agent.launchGen != genBeforeMint { // Another path won the launch race while we were unlocked. + m.abandonFreshSharedPoolLeaseLocked(name, leaseID, freshLease) return nil } - return m.launchInTmux(ctx, agent) + return m.launchInTmuxLeasedLocked(ctx, agent, name, leaseID, freshLease) } func isTmuxDuplicateSessionError(err error) bool { diff --git a/src/crux/agent/shared_pool.go b/src/crux/agent/shared_pool.go index 2137b7881c..a844f02e23 100644 --- a/src/crux/agent/shared_pool.go +++ b/src/crux/agent/shared_pool.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/json" + "errors" "fmt" "io" "net/http" @@ -14,8 +15,19 @@ import ( const sharedSessionPoolPath = "/api/shared-agent-pool" const sharedPoolIdleEvictionDelay = 5 * time.Minute +// Lease renewals tick every 20s against the Hub's default 90s TTL. A single +// failed renewal must not kill an in-flight CLI turn: the watcher tolerates +// transient failures until this much time has passed since the last successful +// renewal (three missed ticks), which still leaves a margin before the Hub +// expires the lease. A definitive "lease not found" from the Hub evicts at once. +const sharedPoolRenewGrace = 60 * time.Second + var errSharedSessionPoolBusy = fmt.Errorf("shared session pool is busy") +// ErrSharedSessionLeaseNotFound reports that the Hub no longer recognises the +// lease: the slot has been reclaimed, so the spoke must stop using it. +var ErrSharedSessionLeaseNotFound = fmt.Errorf("shared session pool lease not found") + // SharedSessionPool coordinates heavyweight CLI sessions across Hive tenants. // Each tenant keeps its own tmux session, UID, credentials, and workspace; only // the right to have a live CLI session is shared. @@ -78,6 +90,9 @@ func (c *sharedSessionPoolClient) request(ctx context.Context, method, path stri if resp.StatusCode == http.StatusConflict { return sharedPoolResponse{}, errSharedSessionPoolBusy } + if resp.StatusCode == http.StatusNotFound && body.LeaseID != "" { + return sharedPoolResponse{}, ErrSharedSessionLeaseNotFound + } if resp.StatusCode < 200 || resp.StatusCode >= 300 { message, _ := io.ReadAll(io.LimitReader(resp.Body, 2048)) return sharedPoolResponse{}, fmt.Errorf("shared session pool returned %s: %s", resp.Status, strings.TrimSpace(string(message))) @@ -136,6 +151,7 @@ func (c *sharedSessionPoolClient) Release(ctx context.Context, leaseID string) e func (m *Manager) watchSharedPoolLease(ctx context.Context, name, leaseID string) { ticker := time.NewTicker(20 * time.Second) defer ticker.Stop() + lastRenewed := time.Now() for { select { case <-ctx.Done(): @@ -168,12 +184,91 @@ func (m *Manager) watchSharedPoolLease(ctx context.Context, name, leaseID string ctx, cancel := context.WithTimeout(context.Background(), 8*time.Second) err := m.sharedPool.Renew(ctx, leaseID) cancel() - if err != nil { + if err == nil { + lastRenewed = time.Now() + continue + } + if m.sharedPoolRenewFailureEvicts(err, lastRenewed, time.Now()) { m.logger.Warn("shared agent pool lease renewal failed; evicting CLI session", "agent", name, "error", err) m.evictSharedPoolSession(name, leaseID) return } + m.logger.Warn("shared agent pool lease renewal failed; retrying before eviction", + "agent", name, "error", err, "since_last_renewal", time.Since(lastRenewed).Round(time.Second)) + } +} + +// sharedPoolRenewFailureEvicts decides whether a failed renewal ends the CLI +// session now. A Hub that no longer knows the lease is definitive; any other +// error (Hub restart, network blip, timeout) is tolerated until +// sharedPoolRenewGrace has elapsed since the last successful renewal. +func (m *Manager) sharedPoolRenewFailureEvicts(err error, lastRenewed, now time.Time) bool { + if errors.Is(err, ErrSharedSessionLeaseNotFound) { + return true + } + return now.Sub(lastRenewed) >= sharedPoolRenewGrace +} + +// acquireSharedPoolLeaseUnlocked obtains the live-session lease a relaunch +// outside Start needs (restart, missing-session recovery, resume). It MUST be +// called with m.mu released: Acquire may block on a busy pool. A lease the +// agent already holds is reused (fresh=false); otherwise a new one is taken +// (fresh=true) and the caller owns releasing it if the launch fails. With no +// shared pool configured it returns an empty lease. +func (m *Manager) acquireSharedPoolLeaseUnlocked(ctx context.Context, agent *AgentProcess, name string) (leaseID string, fresh bool, err error) { + if m.sharedPool == nil { + return "", false, nil + } + m.mu.RLock() + leaseID = agent.poolLeaseID + m.mu.RUnlock() + if leaseID != "" { + return leaseID, false, nil + } + leaseID, err = m.sharedPool.Acquire(ctx, name) + if err != nil { + return "", false, fmt.Errorf("acquire shared CLI session slot: %w", err) } + return leaseID, true, nil +} + +// abandonFreshSharedPoolLeaseLocked returns a lease acquired for a relaunch +// that will not happen (agent removed, or another path won the launch race). +// Caller holds m.mu; the Hub call runs with the lock dropped and re-acquired. +func (m *Manager) abandonFreshSharedPoolLeaseLocked(name, leaseID string, fresh bool) { + if leaseID == "" || !fresh { + return + } + m.mu.Unlock() + m.releaseSharedPoolLease(name, leaseID) + m.mu.Lock() +} + +// launchInTmuxLeasedLocked is launchInTmux for the relaunch paths that do not +// go through Start. Caller holds m.mu and passes the lease obtained from +// acquireSharedPoolLeaseUnlocked. On success the lease is bound to the agent +// and its renewal watcher started; on failure a freshly acquired lease is +// returned to the Hub (m.mu is dropped around that network call and +// re-acquired before returning, so the caller's deferred Unlock stays +// balanced). +func (m *Manager) launchInTmuxLeasedLocked(ctx context.Context, agent *AgentProcess, name, leaseID string, fresh bool) error { + if leaseID != "" { + agent.poolLeaseID = leaseID + } + err := m.launchInTmux(ctx, agent) + if leaseID == "" { + return err + } + if err == nil { + m.startSharedPoolLeaseWatcherLocked(agent, name, leaseID) + return nil + } + if fresh { + m.mu.Unlock() + m.releaseSharedPoolLease(name, leaseID) + m.mu.Lock() + } + return err } func (m *Manager) evictSharedPoolSession(name, leaseID string) { diff --git a/src/crux/agent/shared_pool_test.go b/src/crux/agent/shared_pool_test.go index d0d5702923..9af8555fae 100644 --- a/src/crux/agent/shared_pool_test.go +++ b/src/crux/agent/shared_pool_test.go @@ -2,11 +2,16 @@ package agent import ( "context" - "github.com/hivecommons/hive/crux/config" - "github.com/hivecommons/hive/crux/internal/testutil" + "errors" + "fmt" + "net/http" + "net/http/httptest" "sync" "testing" "time" + + "github.com/hivecommons/hive/crux/config" + "github.com/hivecommons/hive/crux/internal/testutil" ) type recordingSharedSessionPool struct { @@ -133,3 +138,128 @@ func TestWithSharedSessionPoolOption(t *testing.T) { t.Fatal("sharedPool option was not applied") } } + +func TestSharedPoolRenewFailureTolerance(t *testing.T) { + m := NewManager(map[string]config.AgentConfig{"scanner": {Backend: "claude"}}, discardLogger(), ProjectContext{}) + now := time.Now() + transient := errors.New("dial tcp: connection refused") + if m.sharedPoolRenewFailureEvicts(transient, now.Add(-20*time.Second), now) { + t.Fatal("a single transient renewal failure must not evict a live CLI session") + } + if m.sharedPoolRenewFailureEvicts(transient, now.Add(-40*time.Second), now) { + t.Fatal("two missed renewals must not evict a live CLI session") + } + if !m.sharedPoolRenewFailureEvicts(transient, now.Add(-sharedPoolRenewGrace), now) { + t.Fatal("renewal failures past the grace window must evict") + } + if !m.sharedPoolRenewFailureEvicts(fmt.Errorf("renew: %w", ErrSharedSessionLeaseNotFound), now.Add(-time.Second), now) { + t.Fatal("a Hub that no longer knows the lease must evict immediately") + } +} + +func TestSharedPoolClientRenewReportsLeaseNotFound(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Error(w, "lease not found", http.StatusNotFound) + })) + defer srv.Close() + pool, err := NewSharedSessionPoolClient(srv.URL, "school", "token") + if err != nil { + t.Fatal(err) + } + err = pool.Renew(context.Background(), "lease-gone") + if !errors.Is(err, ErrSharedSessionLeaseNotFound) { + t.Fatalf("Renew error = %v, want ErrSharedSessionLeaseNotFound", err) + } +} + +func TestSharedPoolSendKickAutoStartFailureKeepsLockBalanced(t *testing.T) { + if !tmuxAvailable() { + t.Skip("tmux not available") + } + t.Setenv("HIVE_WORK_DIR", t.TempDir()) + pool := &recordingSharedSessionPool{acquireFn: func(context.Context, string) (string, error) { + return "", errors.New("shared session pool is busy") + }} + m := NewManager(map[string]config.AgentConfig{"scanner": makeAgentConfig("claude", "sonnet")}, discardLogger(), ProjectContext{}) + m.sharedPool = pool + defer cleanupAgent(t, m, "scanner") + m.mu.Lock() + m.agents["scanner"].State = StateStopped + m.mu.Unlock() + + // Before the fix the error return left m.mu unlocked for the deferred + // Unlock, which is a non-recoverable runtime fatal. + if err := m.SendKick("scanner", "hello"); err == nil { + t.Fatal("SendKick succeeded although the pool refused a slot") + } + m.mu.Lock() + m.mu.Unlock() +} + +func TestSharedPoolRestartAcquiresLeaseForEvictedAgent(t *testing.T) { + if !tmuxAvailable() { + t.Skip("tmux not available") + } + t.Setenv("HIVE_WORK_DIR", t.TempDir()) + pool := &recordingSharedSessionPool{} + m := NewManager(map[string]config.AgentConfig{"worker": makeAgentConfig("claude", "sonnet")}, discardLogger(), ProjectContext{}) + m.sharedPool = pool + defer cleanupAgent(t, m, "worker") + + if err := m.Start(context.Background(), "worker"); err != nil { + t.Fatalf("Start: %v", err) + } + m.evictSharedPoolSession("worker", "lease-worker") + m.mu.RLock() + state, lease := m.agents["worker"].State, m.agents["worker"].poolLeaseID + m.mu.RUnlock() + if state != StateStopped || lease != "" { + t.Fatalf("after eviction state=%q lease=%q", state, lease) + } + + if err := m.Restart(context.Background(), "worker"); err != nil { + t.Fatalf("Restart: %v", err) + } + m.mu.RLock() + lease = m.agents["worker"].poolLeaseID + watcher := m.agents["worker"].poolLeaseCancel != nil + m.mu.RUnlock() + if lease != "lease-worker" || !watcher { + t.Fatalf("restarted agent lease=%q watcher=%v, want leased with renewal watcher", lease, watcher) + } + pool.mu.Lock() + acquired := append([]string(nil), pool.acquired...) + pool.mu.Unlock() + if len(acquired) != 2 { + t.Fatalf("acquired = %v, want Start and Restart each to take a lease", acquired) + } +} + +func TestSharedPoolRestartRefusedWhenPoolBusy(t *testing.T) { + if !tmuxAvailable() { + t.Skip("tmux not available") + } + t.Setenv("HIVE_WORK_DIR", t.TempDir()) + pool := &recordingSharedSessionPool{} + m := NewManager(map[string]config.AgentConfig{"worker": makeAgentConfig("claude", "sonnet")}, discardLogger(), ProjectContext{}) + m.sharedPool = pool + defer cleanupAgent(t, m, "worker") + + if err := m.Start(context.Background(), "worker"); err != nil { + t.Fatalf("Start: %v", err) + } + m.evictSharedPoolSession("worker", "lease-worker") + pool.mu.Lock() + pool.acquireFn = func(context.Context, string) (string, error) { return "", errSharedSessionPoolBusy } + pool.mu.Unlock() + + if err := m.Restart(context.Background(), "worker"); err == nil { + t.Fatal("Restart relaunched an evicted agent without a shared pool lease") + } + m.mu.RLock() + state, lease := m.agents["worker"].State, m.agents["worker"].poolLeaseID + m.mu.RUnlock() + if state == StateRunning || lease != "" { + t.Fatalf("after refused restart state=%q lease=%q, want not running and unleased", state, lease) + } +} diff --git a/src/docs/env-vars.md b/src/docs/env-vars.md index 3c68477e2c..3a7454ada9 100644 --- a/src/docs/env-vars.md +++ b/src/docs/env-vars.md @@ -30,7 +30,8 @@ This reference is compiled by hand from the Go source under `src/`, the deployme | `HIVE_HUB_URL` | No | `hub.url` from config | Hub URL override for spoke heartbeats/registration. On the hub it is also the last environment variable consulted in the hub public-origin chain (see `HIVE_HUB_PUBLIC_URL`). | | `HIVE_SHARED_AGENT_POOL_ENABLED` | No | `false` | Spoke opt-in for Hub-coordinated CLI session sharing. Requires an in-process runtime and a Hub link; each Hive keeps its own workspace, UID, credentials, and tmux session. Idle CLI sessions are terminated after turns to return the global slot. | | `HIVE_SHARED_AGENT_POOL_MAX_SESSIONS` | No | `0` (disabled) | Hub-side maximum number of live CLI session leases. Set to `1` to serialize agent sessions across participating Hives. | -| `HIVE_SHARED_AGENT_POOL_LEASE_SECONDS` | No | `90` | Hub-side lease expiry in seconds. Values below 30 or invalid values use the 90-second default. Spokes renew active leases every 20 seconds. | +| `HIVE_SHARED_AGENT_POOL_MAX_PER_HIVE` | No | `0` (no per-tenant cap) | Hub-side maximum number of live CLI session leases a single Hive may hold. Set below `HIVE_SHARED_AGENT_POOL_MAX_SESSIONS` on multi-tenant hosts so one tenant cannot take every slot and renew them indefinitely. A Hive at its cap gets `409` without taking a FIFO position. | +| `HIVE_SHARED_AGENT_POOL_LEASE_SECONDS` | No | `90` | Hub-side lease expiry in seconds. Values below 30 or invalid values use the 90-second default. Spokes renew active leases every 20 seconds and tolerate transient renewal failures for 60 seconds after the last successful renewal before terminating the CLI session; a `404` (lease reclaimed by the Hub) terminates it immediately. | | `HIVE_NPS_ENABLED` | No | `hub.nps_enabled`; when both are unset, on for hosted spokes (`hub.hive_type: hosted`) and off otherwise | Turns the dashboard NPS feedback prompt on or off. Accepts `1/true/yes/on` and `0/false/no/off`; other values are ignored. Responses are forwarded over the spoke's hub link or, for a standalone hive with a relay configured, to the NPS relay, never to anyone else. See [NPS feedback prompt](nps.md). | | `HIVE_NPS_RELAY_URL` | No | `hub.nps_relay_url`; empty (relay disabled) | Base URL of the NPS relay for standalone hives (the hivecommons relay is `https://docs.hivecommons.dev/api/nps`). A spoke with NPS enabled and no hub link self-registers a generated Ed25519 key at `/register` and POSTs signed responses here, with no token to configure; a hub pulls from `/pending` and acks at `/ack`. Must be `https` (plain `http` only to a loopback host). See [NPS feedback prompt](nps.md#standalone-hives-the-nps-relay). | | `HIVE_NPS_RELAY_PULL_SECRET` | No | `hub.nps_relay_pull_secret`; empty | Secret, hub only. Authenticates the hub's periodic pull from the NPS relay. Without it (or without a relay URL) the hub does not pull. Never logged. | diff --git a/src/pkg/hub/shared_agent_pool.go b/src/pkg/hub/shared_agent_pool.go index 10d649d73e..d47b573fa4 100644 --- a/src/pkg/hub/shared_agent_pool.go +++ b/src/pkg/hub/shared_agent_pool.go @@ -36,9 +36,12 @@ type sharedAgentPoolState struct { } type sharedAgentPool struct { - mu sync.Mutex - path string - max int + mu sync.Mutex + path string + max int + // maxPerHive caps the leases a single tenant may hold so one Hive cannot + // take every slot and renew them indefinitely. 0 means no cap below max. + maxPerHive int ttl time.Duration leases map[string]sharedAgentLease waitingHives map[string]time.Time // hive_id -> first 409 (FIFO position) @@ -48,6 +51,10 @@ type sharedAgentPool struct { func newSharedAgentPool() *sharedAgentPool { max, _ := strconv.Atoi(strings.TrimSpace(os.Getenv("HIVE_SHARED_AGENT_POOL_MAX_SESSIONS"))) + maxPerHive, _ := strconv.Atoi(strings.TrimSpace(os.Getenv("HIVE_SHARED_AGENT_POOL_MAX_PER_HIVE"))) + if maxPerHive < 0 { + maxPerHive = 0 + } ttl := defaultSharedAgentLeaseTTL if seconds, err := strconv.Atoi(strings.TrimSpace(os.Getenv("HIVE_SHARED_AGENT_POOL_LEASE_SECONDS"))); err == nil && seconds >= 30 { ttl = time.Duration(seconds) * time.Second @@ -55,6 +62,7 @@ func newSharedAgentPool() *sharedAgentPool { p := &sharedAgentPool{ path: sharedAgentPoolPath, max: max, + maxPerHive: maxPerHive, ttl: ttl, leases: make(map[string]sharedAgentLease), waitingHives: make(map[string]time.Time), @@ -112,6 +120,20 @@ func (p *sharedAgentPool) saveLocked() error { return nil } +// hiveAtLeaseCap reports whether hiveID already holds its per-tenant share. +func (p *sharedAgentPool) hiveAtLeaseCap(hiveID string) bool { + if p.maxPerHive < 1 { + return false + } + held := 0 + for _, lease := range p.leases { + if lease.HiveID == hiveID { + held++ + } + } + return held >= p.maxPerHive +} + // hasEarlierWaitingHive reports whether another hive has been waiting longer // than hiveID (which may not be waiting yet). Stale waiters are pruned first. func (p *sharedAgentPool) hasEarlierWaitingHive(hiveID string, now time.Time) bool { @@ -195,6 +217,15 @@ func (p *sharedAgentPool) handle(w http.ResponseWriter, r *http.Request, s *HubS writeSharedPoolLease(w, lease.ID) return } + if p.hiveAtLeaseCap(req.HiveID) { + // A tenant at its own cap is not waiting on fleet capacity, so it + // must not hold a FIFO position that blocks other Hives. + delete(p.waitingHives, req.HiveID) + delete(p.lastSeen, req.HiveID) + w.Header().Set("Retry-After", "3") + http.Error(w, "shared agent pool per-hive limit reached", http.StatusConflict) + return + } if len(p.leases) >= p.max || p.hasEarlierWaitingHive(req.HiveID, now) { if p.waitingHives == nil { p.waitingHives = make(map[string]time.Time) diff --git a/src/pkg/hub/shared_agent_pool_test.go b/src/pkg/hub/shared_agent_pool_test.go index 585c3fe599..97343abc1e 100644 --- a/src/pkg/hub/shared_agent_pool_test.go +++ b/src/pkg/hub/shared_agent_pool_test.go @@ -198,3 +198,53 @@ func TestSharedAgentPoolFIFOWaitersAndStaleExpiry(t *testing.T) { t.Fatalf("h3 acquire status = %d, want 200 after stale waiter pruned", r3.Code) } } + +func TestSharedAgentPoolPerHiveCapDoesNotStarveOtherTenants(t *testing.T) { + pool := &sharedAgentPool{ + path: filepath.Join(t.TempDir(), "shared-agent-pool.json"), + max: 3, + maxPerHive: 1, + ttl: time.Minute, + leases: make(map[string]sharedAgentLease), + waitingHives: make(map[string]time.Time), + lastSeen: make(map[string]time.Time), + } + s := &HubServer{logger: slog.Default()} + s.setHubSecret("test-per-hive-cap-master") + + if r := serveSharedPoolRequest(t, pool, s, http.MethodPost, "h0", map[string]string{"hive_id": "h0", "agent_name": "a"}); r.Code != http.StatusOK { + t.Fatalf("h0 first acquire status = %d, want 200", r.Code) + } + // Second agent on the same tenant exceeds its cap even though the fleet + // still has two free slots. + r := serveSharedPoolRequest(t, pool, s, http.MethodPost, "h0", map[string]string{"hive_id": "h0", "agent_name": "b"}) + if r.Code != http.StatusConflict { + t.Fatalf("h0 second acquire status = %d, want 409", r.Code) + } + pool.mu.Lock() + _, waiting := pool.waitingHives["h0"] + pool.mu.Unlock() + if waiting { + t.Fatal("tenant at its per-hive cap must not take a FIFO waiter position") + } + // Another tenant is not blocked by h0's capped request. + if r := serveSharedPoolRequest(t, pool, s, http.MethodPost, "h1", map[string]string{"hive_id": "h1", "agent_name": "c"}); r.Code != http.StatusOK { + t.Fatalf("h1 acquire status = %d, want 200", r.Code) + } + // Re-acquiring an existing lease (same hive+agent) is a renewal, not a new slot. + if r := serveSharedPoolRequest(t, pool, s, http.MethodPost, "h0", map[string]string{"hive_id": "h0", "agent_name": "a"}); r.Code != http.StatusOK { + t.Fatalf("h0 renew-by-acquire status = %d, want 200", r.Code) + } +} + +func TestNewSharedAgentPoolReadsPerHiveCap(t *testing.T) { + t.Setenv("HIVE_SHARED_AGENT_POOL_MAX_SESSIONS", "4") + t.Setenv("HIVE_SHARED_AGENT_POOL_MAX_PER_HIVE", "2") + if got := newSharedAgentPool().maxPerHive; got != 2 { + t.Fatalf("maxPerHive = %d, want 2", got) + } + t.Setenv("HIVE_SHARED_AGENT_POOL_MAX_PER_HIVE", "-1") + if got := newSharedAgentPool().maxPerHive; got != 0 { + t.Fatalf("maxPerHive = %d, want 0 for negative input", got) + } +} From 5a9b26305eb0e267b47e9d86676e0fef42980936 Mon Sep 17 00:00:00 2001 From: ci-maintainer <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Wed, 28 Oct 2026 16:09:23 -0400 Subject: [PATCH 8/8] test(agent): cover shared pool lease renew/evict paths; TryLock lock-balance probe Replace the empty critical section (SA2001) in the SendKick lock-balance test with a TryLock assertion so the test fails instead of hanging when the lock leaks, and golangci-lint passes. Make the lease watcher tick interval a package var (same pattern as agyTurnStatusInterval) so tests can drive watchSharedPoolLease: successful renewals mid-turn, transient renewal failures tolerated within the grace window, immediate eviction on ErrSharedSessionLeaseNotFound, exit when the lease is superseded or the agent is gone, and idle agents without a visible prompt. Also cover the eviction outcome guards, releaseSharedPoolLease, acquireSharedPoolLeaseUnlocked/abandonFreshSharedPoolLeaseLocked, and the Hub client's Acquire busy-retry, deadline cancel_wait, error, Renew and Release branches. Hive-Fix-Attempt: 2/6 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: ci-maintainer <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- src/crux/agent/shared_pool.go | 5 +- src/crux/agent/shared_pool_test.go | 458 ++++++++++++++++++++++++++++- 2 files changed, 459 insertions(+), 4 deletions(-) diff --git a/src/crux/agent/shared_pool.go b/src/crux/agent/shared_pool.go index a844f02e23..56ee459d0d 100644 --- a/src/crux/agent/shared_pool.go +++ b/src/crux/agent/shared_pool.go @@ -22,6 +22,9 @@ const sharedPoolIdleEvictionDelay = 5 * time.Minute // expires the lease. A definitive "lease not found" from the Hub evicts at once. const sharedPoolRenewGrace = 60 * time.Second +// sharedPoolRenewInterval is a var so tests can tick the lease watcher fast. +var sharedPoolRenewInterval = 20 * time.Second + var errSharedSessionPoolBusy = fmt.Errorf("shared session pool is busy") // ErrSharedSessionLeaseNotFound reports that the Hub no longer recognises the @@ -149,7 +152,7 @@ func (c *sharedSessionPoolClient) Release(ctx context.Context, leaseID string) e } func (m *Manager) watchSharedPoolLease(ctx context.Context, name, leaseID string) { - ticker := time.NewTicker(20 * time.Second) + ticker := time.NewTicker(sharedPoolRenewInterval) defer ticker.Stop() lastRenewed := time.Now() for { diff --git a/src/crux/agent/shared_pool_test.go b/src/crux/agent/shared_pool_test.go index 9af8555fae..6ea94cdf4e 100644 --- a/src/crux/agent/shared_pool_test.go +++ b/src/crux/agent/shared_pool_test.go @@ -2,10 +2,12 @@ package agent import ( "context" + "encoding/json" "errors" "fmt" "net/http" "net/http/httptest" + "strings" "sync" "testing" "time" @@ -20,6 +22,8 @@ type recordingSharedSessionPool struct { released []string renewed []string acquireFn func(ctx context.Context, agent string) (string, error) + renewFn func(ctx context.Context, id string) error + releaseFn func(ctx context.Context, id string) error } func (p *recordingSharedSessionPool) Acquire(ctx context.Context, agent string) (string, error) { @@ -33,20 +37,59 @@ func (p *recordingSharedSessionPool) Acquire(ctx context.Context, agent string) return "lease-" + agent, nil } -func (p *recordingSharedSessionPool) Renew(_ context.Context, id string) error { +func (p *recordingSharedSessionPool) Renew(ctx context.Context, id string) error { p.mu.Lock() p.renewed = append(p.renewed, id) + fn := p.renewFn p.mu.Unlock() + if fn != nil { + return fn(ctx, id) + } return nil } -func (p *recordingSharedSessionPool) Release(_ context.Context, id string) error { +func (p *recordingSharedSessionPool) Release(ctx context.Context, id string) error { p.mu.Lock() p.released = append(p.released, id) + fn := p.releaseFn p.mu.Unlock() + if fn != nil { + return fn(ctx, id) + } return nil } +func (p *recordingSharedSessionPool) snapshot() (acquired, renewed, released []string) { + p.mu.Lock() + defer p.mu.Unlock() + return append([]string(nil), p.acquired...), append([]string(nil), p.renewed...), append([]string(nil), p.released...) +} + +func (p *recordingSharedSessionPool) setRenewFn(fn func(ctx context.Context, id string) error) { + p.mu.Lock() + p.renewFn = fn + p.mu.Unlock() +} + +// fastSharedPoolRenew shrinks the lease watcher tick for the test's duration. +func fastSharedPoolRenew(t *testing.T) { + t.Helper() + prev := sharedPoolRenewInterval + sharedPoolRenewInterval = 5 * time.Millisecond + t.Cleanup(func() { sharedPoolRenewInterval = prev }) +} + +// newSharedPoolManager returns a manager with one "scanner" agent that is +// marked running (no tmux) and bound to the given pool. +func newSharedPoolManager(t *testing.T, pool SharedSessionPool) (*Manager, *AgentProcess) { + t.Helper() + m := NewManager(map[string]config.AgentConfig{"scanner": {Backend: "claude"}}, discardLogger(), ProjectContext{}) + m.sharedPool = pool + a := m.agents["scanner"] + a.State = StateRunning + return m, a +} + func TestSharedPoolEvictionStopsCLIAndReleasesLease(t *testing.T) { pool := &recordingSharedSessionPool{} m := NewManager(map[string]config.AgentConfig{"scanner": {Backend: "claude"}}, discardLogger(), ProjectContext{}) @@ -192,7 +235,9 @@ func TestSharedPoolSendKickAutoStartFailureKeepsLockBalanced(t *testing.T) { if err := m.SendKick("scanner", "hello"); err == nil { t.Fatal("SendKick succeeded although the pool refused a slot") } - m.mu.Lock() + if !m.mu.TryLock() { + t.Fatal("m.mu still held after SendKick returned an error") + } m.mu.Unlock() } @@ -263,3 +308,410 @@ func TestSharedPoolRestartRefusedWhenPoolBusy(t *testing.T) { t.Fatalf("after refused restart state=%q lease=%q, want not running and unleased", state, lease) } } + +func TestSharedPoolWatcherRenewsWhileTurnInFlight(t *testing.T) { + fastSharedPoolRenew(t) + pool := &recordingSharedSessionPool{} + m, a := newSharedPoolManager(t, pool) + now := time.Now() + a.StartedAt = &now + a.LastKick = &now // unsettled outcome: the agent is mid-turn, never idle + a.poolLeaseID = "lease-busy" + m.mu.Lock() + m.startSharedPoolLeaseWatcherLocked(a, "scanner", "lease-busy") + // A second call must not spawn a second watcher. + first := a.poolLeaseCancel + m.startSharedPoolLeaseWatcherLocked(a, "scanner", "lease-busy") + second := a.poolLeaseCancel + m.mu.Unlock() + if first == nil || fmt.Sprintf("%p", first) != fmt.Sprintf("%p", second) { + t.Fatal("watcher cancel was not installed exactly once") + } + + testutil.EventuallyEvery(t, 2*time.Second, time.Millisecond, func() bool { + _, renewed, _ := pool.snapshot() + return len(renewed) >= 3 + }, "watcher did not renew the lease") + m.mu.RLock() + state, lease := a.State, a.poolLeaseID + m.mu.RUnlock() + if state != StateRunning || lease != "lease-busy" { + t.Fatalf("state=%q lease=%q, want running with lease intact", state, lease) + } + + // Stopping releases the lease and cancels the watcher; renewals cease. + if err := m.Stop("scanner"); err != nil { + t.Fatalf("Stop: %v", err) + } + _, _, released := pool.snapshot() + if len(released) != 1 || released[0] != "lease-busy" { + t.Fatalf("released = %v, want [lease-busy]", released) + } + m.mu.RLock() + defer m.mu.RUnlock() + if a.poolLeaseID != "" || a.poolLeaseCancel != nil { + t.Fatalf("Stop left lease=%q watcher=%v", a.poolLeaseID, a.poolLeaseCancel != nil) + } +} + +func TestSharedPoolWatcherToleratesTransientRenewFailures(t *testing.T) { + fastSharedPoolRenew(t) + pool := &recordingSharedSessionPool{renewFn: func(context.Context, string) error { + return errors.New("dial tcp: connection refused") + }} + m, a := newSharedPoolManager(t, pool) + now := time.Now() + a.StartedAt = &now + a.LastKick = &now + a.poolLeaseID = "lease-blip" + m.mu.Lock() + m.startSharedPoolLeaseWatcherLocked(a, "scanner", "lease-blip") + m.mu.Unlock() + + testutil.EventuallyEvery(t, 2*time.Second, time.Millisecond, func() bool { + _, renewed, _ := pool.snapshot() + return len(renewed) >= 4 + }, "watcher did not keep retrying renewals") + m.mu.RLock() + state, lease := a.State, a.poolLeaseID + m.mu.RUnlock() + if state != StateRunning || lease != "lease-blip" { + t.Fatalf("transient failures evicted: state=%q lease=%q", state, lease) + } + + // A definitive "lease not found" evicts on the next tick. + pool.setRenewFn(func(context.Context, string) error { + return fmt.Errorf("renew: %w", ErrSharedSessionLeaseNotFound) + }) + testutil.EventuallyEvery(t, 2*time.Second, time.Millisecond, func() bool { + m.mu.RLock() + defer m.mu.RUnlock() + return a.State == StateStopped && a.poolLeaseID == "" && a.poolLeaseCancel == nil + }, "lease-not-found did not evict the session") + _, _, released := pool.snapshot() + if len(released) != 1 || released[0] != "lease-blip" { + t.Fatalf("released = %v, want [lease-blip]", released) + } +} + +func TestSharedPoolWatcherExitsWhenLeaseChanges(t *testing.T) { + fastSharedPoolRenew(t) + pool := &recordingSharedSessionPool{} + m, a := newSharedPoolManager(t, pool) + now := time.Now() + a.LastKick = &now + a.poolLeaseID = "lease-new" + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + done := make(chan struct{}) + go func() { + defer close(done) + m.watchSharedPoolLease(ctx, "scanner", "lease-old") + }() + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("watcher for a superseded lease did not exit") + } + if _, renewed, _ := pool.snapshot(); len(renewed) != 0 { + t.Fatalf("superseded watcher renewed %v", renewed) + } + + // An unknown agent exits the same way; a cancelled context exits before + // the first tick. + done = make(chan struct{}) + go func() { + defer close(done) + m.watchSharedPoolLease(ctx, "ghost", "lease-x") + }() + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("watcher for an unknown agent did not exit") + } + cancelled, cancelNow := context.WithCancel(context.Background()) + cancelNow() + m.watchSharedPoolLease(cancelled, "scanner", "lease-new") +} + +func TestSharedPoolWatcherIdleWithoutPromptKeepsRenewing(t *testing.T) { + fastSharedPoolRenew(t) + pool := &recordingSharedSessionPool{} + m, a := newSharedPoolManager(t, pool) + started := time.Now().Add(-2 * sharedPoolIdleEvictionDelay) + a.StartedAt = &started + a.poolLeaseID = "lease-idle" + // Idle past the delay, but with no tmux pane showing an input prompt the + // watcher must not evict; it falls through to a renewal. + m.mu.Lock() + m.startSharedPoolLeaseWatcherLocked(a, "scanner", "lease-idle") + m.mu.Unlock() + testutil.EventuallyEvery(t, 2*time.Second, time.Millisecond, func() bool { + _, renewed, _ := pool.snapshot() + return len(renewed) >= 2 + }, "idle agent without a prompt was not renewed") + m.mu.RLock() + state := a.State + cancel := a.poolLeaseCancel + m.mu.RUnlock() + if state != StateRunning { + t.Fatalf("state = %q, want running", state) + } + cancel() +} + +func TestSharedPoolEvictionGuards(t *testing.T) { + pool := &recordingSharedSessionPool{} + m, a := newSharedPoolManager(t, pool) + now := time.Now() + a.StartedAt = &now + a.LastKick = &now + a.poolLeaseID = "lease-guard" + a.KickOutcome = KickOutcome{Kind: KickOutcomeEnded, At: now, KickAt: now} + + m.evictSharedPoolSession("ghost", "lease-guard") + m.evictSharedPoolSession("scanner", "lease-other") + stale := KickOutcome{Kind: KickOutcomeEnded, At: now.Add(-time.Minute), KickAt: now.Add(-time.Minute)} + m.evictSharedPoolSessionAfterTurn("scanner", "lease-guard", stale) + a.kickDelivering.Store(true) + m.evictSharedPoolSessionAfterTurn("scanner", "lease-guard", a.KickOutcome) + a.kickDelivering.Store(false) + if a.State != StateRunning || a.poolLeaseID != "lease-guard" { + t.Fatalf("guarded eviction fired: state=%q lease=%q", a.State, a.poolLeaseID) + } + if _, _, released := pool.snapshot(); len(released) != 0 { + t.Fatalf("released = %v, want none", released) + } + + pool.releaseFn = func(context.Context, string) error { return errors.New("hub down") } + m.evictSharedPoolSessionAfterTurn("scanner", "lease-guard", a.KickOutcome) + if a.State != StateStopped || a.poolLeaseID != "" { + t.Fatalf("matching outcome did not evict: state=%q lease=%q", a.State, a.poolLeaseID) + } + if _, _, released := pool.snapshot(); len(released) != 1 { + t.Fatalf("released = %v, want one attempt despite Hub error", released) + } +} + +func TestSharedPoolReleaseLeasePaths(t *testing.T) { + pool := &recordingSharedSessionPool{} + m, a := newSharedPoolManager(t, pool) + a.poolLeaseID = "lease-held" + watcherCtx, watcherCancel := context.WithCancel(context.Background()) + a.poolLeaseCancel = watcherCancel + + // Mismatched lease: returned to the Hub, agent binding untouched. + m.releaseSharedPoolLease("scanner", "lease-stray") + if a.poolLeaseID != "lease-held" || a.poolLeaseCancel == nil { + t.Fatal("releasing a stray lease disturbed the agent's own lease") + } + // Matching lease: binding cleared and watcher cancelled. + pool.releaseFn = func(context.Context, string) error { return errors.New("hub down") } + m.releaseSharedPoolLease("scanner", "lease-held") + if a.poolLeaseID != "" || a.poolLeaseCancel != nil { + t.Fatalf("lease binding not cleared: lease=%q cancel=%v", a.poolLeaseID, a.poolLeaseCancel != nil) + } + select { + case <-watcherCtx.Done(): + default: + t.Fatal("watcher context was not cancelled") + } + _, _, released := pool.snapshot() + if len(released) != 2 || released[0] != "lease-stray" || released[1] != "lease-held" { + t.Fatalf("released = %v", released) + } +} + +func TestSharedPoolAcquireLeaseUnlockedPaths(t *testing.T) { + m, a := newSharedPoolManager(t, nil) + ctx := context.Background() + if lease, fresh, err := m.acquireSharedPoolLeaseUnlocked(ctx, a, "scanner"); lease != "" || fresh || err != nil { + t.Fatalf("no pool: lease=%q fresh=%v err=%v", lease, fresh, err) + } + + pool := &recordingSharedSessionPool{} + m.sharedPool = pool + a.poolLeaseID = "lease-kept" + if lease, fresh, err := m.acquireSharedPoolLeaseUnlocked(ctx, a, "scanner"); lease != "lease-kept" || fresh || err != nil { + t.Fatalf("existing lease: lease=%q fresh=%v err=%v", lease, fresh, err) + } + + a.poolLeaseID = "" + lease, fresh, err := m.acquireSharedPoolLeaseUnlocked(ctx, a, "scanner") + if lease != "lease-scanner" || !fresh || err != nil { + t.Fatalf("fresh lease: lease=%q fresh=%v err=%v", lease, fresh, err) + } + // A fresh lease for a launch that will not happen goes back to the Hub. + m.mu.Lock() + m.abandonFreshSharedPoolLeaseLocked("scanner", "", true) + m.abandonFreshSharedPoolLeaseLocked("scanner", lease, false) + m.abandonFreshSharedPoolLeaseLocked("scanner", lease, true) + m.mu.Unlock() + if _, _, released := pool.snapshot(); len(released) != 1 || released[0] != lease { + t.Fatalf("released = %v, want [%s]", released, lease) + } + + pool.acquireFn = func(context.Context, string) (string, error) { return "", errSharedSessionPoolBusy } + if _, _, err := m.acquireSharedPoolLeaseUnlocked(ctx, a, "scanner"); !errors.Is(err, errSharedSessionPoolBusy) { + t.Fatalf("busy pool err = %v, want errSharedSessionPoolBusy", err) + } +} + +func TestNewSharedSessionPoolClientValidation(t *testing.T) { + for _, tc := range [][3]string{{"", "h", "t"}, {"http://x", "", "t"}, {"http://x", "h", ""}} { + if _, err := NewSharedSessionPoolClient(tc[0], tc[1], tc[2]); err == nil { + t.Fatalf("NewSharedSessionPoolClient(%q,%q,%q) succeeded", tc[0], tc[1], tc[2]) + } + } + pool, err := NewSharedSessionPoolClient(" http://hub.example/ ", "school", "token") + if err != nil { + t.Fatal(err) + } + if got := pool.(*sharedSessionPoolClient).baseURL; got != "http://hub.example" { + t.Fatalf("baseURL = %q, want trimmed", got) + } +} + +type sharedPoolHubStub struct { + mu sync.Mutex + requests []sharedPoolRequest + methods []string + handle func(n int, w http.ResponseWriter, r *http.Request, body sharedPoolRequest) +} + +func (s *sharedPoolHubStub) ServeHTTP(w http.ResponseWriter, r *http.Request) { + var body sharedPoolRequest + _ = json.NewDecoder(r.Body).Decode(&body) + s.mu.Lock() + s.requests = append(s.requests, body) + s.methods = append(s.methods, r.Method) + n := len(s.requests) + s.mu.Unlock() + if r.URL.Path != sharedSessionPoolPath+"/leases" || r.Header.Get("Authorization") != "Bearer token" { + http.Error(w, "bad request", http.StatusBadRequest) + return + } + s.handle(n, w, r, body) +} + +func newSharedPoolClientForStub(t *testing.T, stub *sharedPoolHubStub) SharedSessionPool { + t.Helper() + srv := httptest.NewServer(stub) + t.Cleanup(srv.Close) + pool, err := NewSharedSessionPoolClient(srv.URL, "school", "token") + if err != nil { + t.Fatal(err) + } + return pool +} + +func TestSharedPoolClientAcquireRetriesWhileBusy(t *testing.T) { + stub := &sharedPoolHubStub{handle: func(n int, w http.ResponseWriter, _ *http.Request, body sharedPoolRequest) { + if body.HiveID != "school" || body.AgentName != "scanner" { + http.Error(w, "wrong tenant", http.StatusBadRequest) + return + } + if n == 1 { + http.Error(w, "busy", http.StatusConflict) + return + } + _ = json.NewEncoder(w).Encode(sharedPoolResponse{LeaseID: "lease-42"}) + }} + pool := newSharedPoolClientForStub(t, stub) + lease, err := pool.Acquire(context.Background(), "scanner") + if err != nil || lease != "lease-42" { + t.Fatalf("Acquire = %q, %v; want lease-42", lease, err) + } + stub.mu.Lock() + defer stub.mu.Unlock() + if len(stub.requests) != 2 { + t.Fatalf("requests = %d, want one busy retry then success", len(stub.requests)) + } +} + +func TestSharedPoolClientAcquireGivesUpAtDeadlineAndCancelsWait(t *testing.T) { + stub := &sharedPoolHubStub{handle: func(_ int, w http.ResponseWriter, r *http.Request, body sharedPoolRequest) { + if r.Method == http.MethodDelete { + w.WriteHeader(http.StatusNoContent) + return + } + http.Error(w, "busy", http.StatusConflict) + }} + pool := newSharedPoolClientForStub(t, stub) + ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancel() + if _, err := pool.Acquire(ctx, "scanner"); err == nil || !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("Acquire err = %v, want deadline exceeded", err) + } + stub.mu.Lock() + defer stub.mu.Unlock() + last := stub.requests[len(stub.requests)-1] + if stub.methods[len(stub.methods)-1] != http.MethodDelete || !last.CancelWait || last.HiveID != "school" { + t.Fatalf("final request = %s %+v, want DELETE with cancel_wait", stub.methods[len(stub.methods)-1], last) + } +} + +func TestSharedPoolClientAcquireErrors(t *testing.T) { + status := http.StatusOK + stub := &sharedPoolHubStub{handle: func(_ int, w http.ResponseWriter, _ *http.Request, _ sharedPoolRequest) { + if status != http.StatusOK { + http.Error(w, "nope", status) + return + } + _ = json.NewEncoder(w).Encode(sharedPoolResponse{}) + }} + pool := newSharedPoolClientForStub(t, stub) + if _, err := pool.Acquire(context.Background(), "scanner"); err == nil || !strings.Contains(err.Error(), "empty lease ID") { + t.Fatalf("empty lease err = %v", err) + } + status = http.StatusForbidden + if _, err := pool.Acquire(context.Background(), "scanner"); err == nil || !strings.Contains(err.Error(), "403") || !strings.Contains(err.Error(), "nope") { + t.Fatalf("403 err = %v, want status and body", err) + } + // A 404 on Acquire (no lease ID in the body) is a plain HTTP error, not + // ErrSharedSessionLeaseNotFound. + status = http.StatusNotFound + if _, err := pool.Acquire(context.Background(), "scanner"); err == nil || errors.Is(err, ErrSharedSessionLeaseNotFound) { + t.Fatalf("404 acquire err = %v, want generic error", err) + } + down, err := NewSharedSessionPoolClient("http://127.0.0.1:1", "school", "token") + if err != nil { + t.Fatal(err) + } + if _, err := down.Acquire(context.Background(), "scanner"); err == nil { + t.Fatal("Acquire against a closed port succeeded") + } +} + +func TestSharedPoolClientRenewAndRelease(t *testing.T) { + stub := &sharedPoolHubStub{handle: func(_ int, w http.ResponseWriter, r *http.Request, body sharedPoolRequest) { + if body.LeaseID != "lease-7" || body.HiveID != "school" { + http.Error(w, "wrong lease", http.StatusBadRequest) + return + } + switch r.Method { + case http.MethodPut: + w.WriteHeader(http.StatusNoContent) // empty body path + case http.MethodDelete: + _, _ = w.Write([]byte("{}")) + default: + http.Error(w, "method", http.StatusMethodNotAllowed) + } + }} + pool := newSharedPoolClientForStub(t, stub) + if err := pool.Renew(context.Background(), "lease-7"); err != nil { + t.Fatalf("Renew: %v", err) + } + if err := pool.Release(context.Background(), "lease-7"); err != nil { + t.Fatalf("Release: %v", err) + } + if err := pool.Release(context.Background(), "lease-other"); err == nil { + t.Fatal("Release with the wrong lease succeeded") + } + stub.mu.Lock() + defer stub.mu.Unlock() + if len(stub.methods) != 3 || stub.methods[0] != http.MethodPut || stub.methods[1] != http.MethodDelete { + t.Fatalf("methods = %v", stub.methods) + } +}