diff --git a/changelog.d/added-shared-agent-session-pool.md b/changelog.d/added-shared-agent-session-pool.md new file mode 100644 index 0000000000..30b6049e64 --- /dev/null +++ b/changelog.d/added-shared-agent-session-pool.md @@ -0,0 +1 @@ +- Add an opt-in Hub-coordinated CLI session pool for spokes. Participating Hives keep separate workspaces, UIDs, credentials, and tmux sessions; the Hub leases a global live-session slot and idle sessions are terminated after a turn. Configure `HIVE_SHARED_AGENT_POOL_ENABLED` on spokes and `HIVE_SHARED_AGENT_POOL_MAX_SESSIONS` on the Hub. diff --git a/src/cmd/hive/eval_cycle_seams.go b/src/cmd/hive/eval_cycle_seams.go index d5406916d9..7f68004aba 100644 --- a/src/cmd/hive/eval_cycle_seams.go +++ b/src/cmd/hive/eval_cycle_seams.go @@ -616,7 +616,9 @@ func stoppedAgentStartErrorIsBenign(err error) bool { return true } msg := err.Error() - return strings.Contains(msg, "already running") || strings.Contains(msg, "launch already in progress") + return strings.Contains(msg, "already running") || + strings.Contains(msg, "launch already in progress") || + strings.Contains(msg, "shared session pool is busy") } // advisoryIngestDeps are the side effects of recording an ioscan canary leak diff --git a/src/cmd/hive/main.go b/src/cmd/hive/main.go index 895ae88a87..09f1f94f1b 100644 --- a/src/cmd/hive/main.go +++ b/src/cmd/hive/main.go @@ -26,6 +26,7 @@ import ( gh "github.com/google/go-github/v72/github" "github.com/hivecommons/hive/crux" + cruxagent "github.com/hivecommons/hive/crux/agent" "github.com/hivecommons/hive/crux/inference" "github.com/hivecommons/hive/crux/logscrub" "github.com/hivecommons/hive/crux/proxy" @@ -2169,10 +2170,25 @@ func (b *boot) bootAgentsWith(deps bootAgentsDeps) { b.logger.Warn("cleared invalid inference launch_cmd; inference and gateway backends launch through claude", "agent", name) } rtCfg := cruxconfig.RuntimeConfigFromEnv() - b.agentMgr, err = deps.newRuntime(rtCfg, + runtimeOptions := []cruxruntime.Option{ cruxruntime.WithDefinitions(b.cfg.EnabledAgents()), cruxruntime.WithLogger(b.logger), - cruxruntime.WithProject(b.projectCtx)) + cruxruntime.WithProject(b.projectCtx), + } + if sharedAgentPoolEnabled() { + hubURL := strings.TrimSpace(os.Getenv("HIVE_HUB_URL")) + if hubURL == "" { + hubURL = b.cfg.Hub.URL + } + pool, poolErr := cruxagent.NewSharedSessionPoolClient(hubURL, b.cfg.HiveID, hub.SpokeHeartbeatKey()) + if poolErr != nil { + b.logger.Error("shared agent pool is enabled but cannot be configured", "error", poolErr) + deps.exit(1) + return + } + runtimeOptions = append(runtimeOptions, cruxruntime.WithSharedSessionPool(pool)) + } + b.agentMgr, err = deps.newRuntime(rtCfg, runtimeOptions...) if err != nil { b.logger.Error("crux runtime unavailable", "mode", rtCfg.Mode, "error", err) deps.exit(1) @@ -4603,7 +4619,12 @@ func (b *boot) bootLaunchWith(deps bootLaunchDeps) { // shutdown the ctx-aware stagger returns immediately instead of leaking a // goroutine parked in a bare time.Sleep. enabledAgents := b.cfg.EnabledAgents() - startupAgents := startupLaunchNames(enabledAgents, b.onDemandFromPack) + startupAgents := []string{} + if sharedAgentPoolEnabled() { + b.logger.Info("shared agent pool enabled; persistent agents will start on demand") + } else { + startupAgents = startupLaunchNames(enabledAgents, b.onDemandFromPack) + } b.agentMgr.MarkStartupLaunchQueued(startupAgents) deps.spawn("agent-launch", func() { agentIndex := 0 diff --git a/src/cmd/hive/shared_pool.go b/src/cmd/hive/shared_pool.go new file mode 100644 index 0000000000..2b2e1a77be --- /dev/null +++ b/src/cmd/hive/shared_pool.go @@ -0,0 +1,15 @@ +package main + +import ( + "os" + "strings" +) + +func sharedAgentPoolEnabled() bool { + switch strings.ToLower(strings.TrimSpace(os.Getenv("HIVE_SHARED_AGENT_POOL_ENABLED"))) { + case "1", "true", "yes", "on": + return true + default: + return false + } +} diff --git a/src/cmd/hive/shared_pool_test.go b/src/cmd/hive/shared_pool_test.go new file mode 100644 index 0000000000..1a5991b786 --- /dev/null +++ b/src/cmd/hive/shared_pool_test.go @@ -0,0 +1,27 @@ +package main + +import ( + "testing" +) + +func TestSharedAgentPoolEnabledEnv(t *testing.T) { + for _, tc := range []struct { + val string + want bool + }{ + {"", false}, + {"0", false}, + {"false", false}, + {"no", false}, + {"1", true}, + {"true", true}, + {"TRUE", true}, + {"yes", true}, + {"on", true}, + } { + t.Setenv("HIVE_SHARED_AGENT_POOL_ENABLED", tc.val) + if got := sharedAgentPoolEnabled(); got != tc.want { + t.Errorf("sharedAgentPoolEnabled() for %q = %v, want %v", tc.val, got, tc.want) + } + } +} diff --git a/src/crux/agent/kick_async.go b/src/crux/agent/kick_async.go index 6305b19a9b..edfc47e90c 100644 --- a/src/crux/agent/kick_async.go +++ b/src/crux/agent/kick_async.go @@ -223,7 +223,9 @@ func (m *Manager) SendKickAsync(name string, message string) (started bool, err func (m *Manager) validatePaneKickLocked(agent *AgentProcess, name string) error { if agent.State != StateRunning { - return fmt.Errorf("agent %s cannot be kicked: %s", name, notRunningReason(agent)) + if m.sharedPool == nil || agent.State != StateStopped || agent.Paused || agent.OperatorStopped { + return fmt.Errorf("agent %s cannot be kicked: %s", name, notRunningReason(agent)) + } } if remaining := m.providerErrorBackoffRemainingLocked(agent, time.Now()); remaining > 0 { class, line := agent.ProviderErrorClass, agent.ProviderErrorLine @@ -233,7 +235,7 @@ func (m *Manager) validatePaneKickLocked(agent *AgentProcess, name string) error if err := m.restartKickHoldErrLocked(agent, time.Now()); err != nil { return err } - if !m.tmuxSessionExistsForAgent(agent) { + if agent.State == StateRunning && !m.tmuxSessionExistsForAgent(agent) { return fmt.Errorf("tmux session %s not found", agent.tmuxSession) } return nil @@ -255,9 +257,22 @@ func (m *Manager) deliverKickAsync(name, message string) error { return fmt.Errorf("agent %s not found", name) } if agent.State != StateRunning { - reason := notRunningReason(agent) - m.mu.Unlock() - return fmt.Errorf("agent %s cannot be kicked: %s", name, reason) + if m.sharedPool != nil && agent.State == StateStopped && !agent.Paused && !agent.OperatorStopped { + m.mu.Unlock() + if err := m.Start(context.Background(), name); err != nil { + return fmt.Errorf("auto-start stopped agent for kick: %w", err) + } + m.mu.Lock() + agent, ok = m.agents[name] + if !ok || agent.State != StateRunning { + m.mu.Unlock() + return fmt.Errorf("agent %s failed to enter running state", name) + } + } else { + reason := notRunningReason(agent) + m.mu.Unlock() + return fmt.Errorf("agent %s cannot be kicked: %s", name, reason) + } } if remaining := m.providerErrorBackoffRemainingLocked(agent, time.Now()); remaining > 0 { class, line := agent.ProviderErrorClass, agent.ProviderErrorLine diff --git a/src/crux/agent/kick_outcome.go b/src/crux/agent/kick_outcome.go index 3a60327be6..493c901592 100644 --- a/src/crux/agent/kick_outcome.go +++ b/src/crux/agent/kick_outcome.go @@ -241,9 +241,21 @@ func (m *Manager) SetKickOutcomeObserver(fn func(agentName string, outcome KickO } func (m *Manager) notifyKickOutcome(agentName string, outcome KickOutcome) { - fn := m.kickOutcomeObserver.Load() - if fn == nil { - return + if fn := m.kickOutcomeObserver.Load(); fn != nil { + go (*fn)(agentName, outcome) + } + if m.sharedPool != nil { + go func() { + m.mu.RLock() + agent := m.agents[agentName] + leaseID := "" + if agent != nil { + leaseID = agent.poolLeaseID + } + m.mu.RUnlock() + if leaseID != "" { + m.evictSharedPoolSessionAfterTurn(agentName, leaseID, outcome) + } + }() } - go (*fn)(agentName, outcome) } diff --git a/src/crux/agent/manager.go b/src/crux/agent/manager.go index 4ef743a05f..132a6b5ee3 100644 --- a/src/crux/agent/manager.go +++ b/src/crux/agent/manager.go @@ -258,6 +258,8 @@ type AgentProcess struct { KickRefusalReason string LaunchedMode AgentMode HasLaunched bool + poolLeaseID string + poolLeaseCancel context.CancelFunc baseConfig config.AgentConfig agentSpecPrompt string tmuxSession string @@ -563,6 +565,7 @@ type Manager struct { copilotAuthTokenRejected bool claudeAuthToken string uidMap *UIDMap + sharedPool SharedSessionPool uidMapPath string appAuth AppTokenMinter appNotInstalledWarned sync.Map // agent name -> struct{}: not-installed mint failure already logged @@ -775,6 +778,7 @@ type ManagerOption func(*managerOptions) type managerOptions struct { uidMapPath string uidMapPathSet bool + sharedPool SharedSessionPool } // WithUIDMapPath points a manager at a specific persisted UID map. Production @@ -786,6 +790,13 @@ func WithUIDMapPath(path string) ManagerOption { } } +// WithSharedSessionPool opts a manager into a Hub-coordinated live CLI slot. +// Every Hive retains separate sessions and tenant data; the lease only limits +// how many CLI sessions across the participating Hives remain live. +func WithSharedSessionPool(pool SharedSessionPool) ManagerOption { + return func(opts *managerOptions) { opts.sharedPool = pool } +} + func (m *Manager) effectiveUIDMapPath() string { if m.uidMapPath != "" { return m.uidMapPath @@ -876,6 +887,7 @@ func NewManagerWithOptions(agents map[string]config.AgentConfig, logger *slog.Lo copilotAuthTokenSource: copilotTokenSource, claudeAuthToken: claudeToken, uidMap: uidMap, + sharedPool: opts.sharedPool, uidMapPath: selectedUIDMapPath, kickLogDir: kickLogDir, kickLogRetention: kickLogRetention, @@ -1128,22 +1140,62 @@ func (m *Manager) Start(ctx context.Context, name string) error { // // The remaining subprocess sends below are the actual tmux launch boundary; // Bob's shared-home/file probes have already run lock-free above. + m.mu.RLock() + leaseID := agent.poolLeaseID + hasLeaseWatcher := agent.poolLeaseCancel != nil + m.mu.RUnlock() + if m.sharedPool != nil && leaseID == "" { + var err error + leaseID, err = m.sharedPool.Acquire(ctx, name) + if err != nil { + return fmt.Errorf("acquire shared CLI session slot: %w", err) + } + } + launchSucceeded := false + defer func() { + if leaseID != "" && !launchSucceeded { + m.releaseSharedPoolLease(name, leaseID) + } + }() + m.mu.Lock() defer m.mu.Unlock() // Re-verify under the re-acquired lock: while m.mu was released for Phase 2, // a concurrent Stop/Remove could have deleted this agent from the map or a - // racing path could have started it. The launching guard prevents a second - // concurrent Start of THIS agent, but not a Stop/delete, so re-check both - // before mutating launch state. (agent still points at the same struct; the - // map re-lookup is what detects a delete.) + // racing path could have started it. if cur, ok := m.agents[name]; !ok || cur != agent { return fmt.Errorf("agent %s removed during launch", name) } if agent.State == StateRunning { - // Another path won the launch race while we were unlocked; nothing to do. + if leaseID != "" { + agent.poolLeaseID = leaseID + launchSucceeded = true + if !hasLeaseWatcher { + m.startSharedPoolLeaseWatcherLocked(agent, name, leaseID) + } + } return nil } - return m.launchInTmux(ctx, agent) + if agent.OperatorStopped { + return fmt.Errorf("agent %s was stopped during launch", name) + } + agent.poolLeaseID = leaseID + err := m.launchInTmux(ctx, agent) + if err == nil && leaseID != "" { + launchSucceeded = true + if !hasLeaseWatcher { + m.startSharedPoolLeaseWatcherLocked(agent, name, leaseID) + } + } + return err +} + +func (m *Manager) startSharedPoolLeaseWatcherLocked(agent *AgentProcess, name, leaseID string) { + if agent.poolLeaseCancel == nil { + leaseCtx, cancel := context.WithCancel(context.Background()) + agent.poolLeaseCancel = cancel + go m.watchSharedPoolLease(leaseCtx, name, leaseID) + } } // kickOutcomePollEvery throttles the post-kick turn-ended check to one visible @@ -1242,35 +1294,33 @@ func (m *Manager) waitForInputPromptForAgentReason(agent *AgentProcess, abort fu func (m *Manager) Stop(name string) error { m.mu.Lock() - defer m.mu.Unlock() - agent, ok := m.agents[name] if !ok { + m.mu.Unlock() return fmt.Errorf("agent %s not found", name) } - if agent.State == StateStopped { agent.OperatorStopped = true } - if agent.State != StateRunning { - return nil + leaseID := agent.poolLeaseID + if agent.State == StateRunning { + if agent.cancel != nil { + agent.cancel() + } + m.tmuxSendKeysForAgent(agent, "C-c", "") + agent.State = StateStopped + agent.OperatorStopped = true + m.logger.Info("audit: agent stopped", "name", name) + m.audit(AuditAgentStopped, name, auditFields( + "outcome", "success", + "backend", agent.effectiveBackend(), + "model", agent.effectiveModel(), + )) } - - if agent.cancel != nil { - agent.cancel() + m.mu.Unlock() + if leaseID != "" { + m.evictSharedPoolSession(name, leaseID) } - - m.tmuxSendKeysForAgent(agent, "C-c", "") - - agent.State = StateStopped - agent.OperatorStopped = true - m.logger.Info("audit: agent stopped", "name", name) - m.audit(AuditAgentStopped, name, auditFields( - "outcome", "success", - "backend", agent.effectiveBackend(), - "model", agent.effectiveModel(), - )) - return nil } diff --git a/src/crux/agent/manager_kick.go b/src/crux/agent/manager_kick.go index 553fa1a02f..b1fefe0b0e 100644 --- a/src/crux/agent/manager_kick.go +++ b/src/crux/agent/manager_kick.go @@ -122,10 +122,21 @@ func (m *Manager) SendKick(name string, message string) error { } if agent.State != StateRunning { - if m.deferStartupKickLocked(agent, message) { + if m.sharedPool != nil && agent.State == StateStopped && !agent.Paused && !agent.OperatorStopped { + m.mu.Unlock() + if err := m.Start(context.Background(), name); err != nil { + return fmt.Errorf("auto-start stopped agent for kick: %w", err) + } + m.mu.Lock() + agent, ok = m.agents[name] + if !ok || agent.State != StateRunning { + return fmt.Errorf("agent %s failed to enter running state", name) + } + } else if m.deferStartupKickLocked(agent, message) { return nil + } else { + return fmt.Errorf("agent %s cannot be kicked: %s", name, notRunningReason(agent)) } - return fmt.Errorf("agent %s cannot be kicked: %s", name, notRunningReason(agent)) } if m.deferStartupKickLocked(agent, message) { return nil diff --git a/src/crux/agent/shared_pool.go b/src/crux/agent/shared_pool.go new file mode 100644 index 0000000000..2137b7881c --- /dev/null +++ b/src/crux/agent/shared_pool.go @@ -0,0 +1,245 @@ +package agent + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "time" +) + +const sharedSessionPoolPath = "/api/shared-agent-pool" +const sharedPoolIdleEvictionDelay = 5 * time.Minute + +var errSharedSessionPoolBusy = fmt.Errorf("shared session pool is busy") + +// SharedSessionPool coordinates heavyweight CLI sessions across Hive tenants. +// Each tenant keeps its own tmux session, UID, credentials, and workspace; only +// the right to have a live CLI session is shared. +type SharedSessionPool interface { + Acquire(ctx context.Context, agentName string) (string, error) + Renew(ctx context.Context, leaseID string) error + Release(ctx context.Context, leaseID string) error +} + +type sharedSessionPoolClient struct { + baseURL string + hiveID string + token string + client *http.Client +} + +// NewSharedSessionPoolClient creates an authenticated client for the Hub's +// tenant-scoped session-slot API. The token must be the per-Hive heartbeat key, +// never the fleet master secret. +func NewSharedSessionPoolClient(baseURL, hiveID, token string) (SharedSessionPool, error) { + baseURL = strings.TrimRight(strings.TrimSpace(baseURL), "/") + if baseURL == "" || hiveID == "" || token == "" { + return nil, fmt.Errorf("shared session pool requires Hub URL, Hive ID, and per-Hive token") + } + return &sharedSessionPoolClient{ + baseURL: baseURL, + hiveID: hiveID, + token: token, + client: &http.Client{Timeout: 10 * time.Second}, + }, nil +} + +type sharedPoolRequest struct { + HiveID string `json:"hive_id"` + AgentName string `json:"agent_name,omitempty"` + LeaseID string `json:"lease_id,omitempty"` + CancelWait bool `json:"cancel_wait,omitempty"` +} + +type sharedPoolResponse struct { + LeaseID string `json:"lease_id"` +} + +func (c *sharedSessionPoolClient) request(ctx context.Context, method, path string, body sharedPoolRequest) (sharedPoolResponse, error) { + payload, err := json.Marshal(body) + if err != nil { + return sharedPoolResponse{}, err + } + req, err := http.NewRequestWithContext(ctx, method, c.baseURL+sharedSessionPoolPath+path, bytes.NewReader(payload)) + if err != nil { + return sharedPoolResponse{}, err + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Bearer "+c.token) + resp, err := c.client.Do(req) + if err != nil { + return sharedPoolResponse{}, err + } + defer resp.Body.Close() + if resp.StatusCode == http.StatusConflict { + return sharedPoolResponse{}, errSharedSessionPoolBusy + } + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + message, _ := io.ReadAll(io.LimitReader(resp.Body, 2048)) + return sharedPoolResponse{}, fmt.Errorf("shared session pool returned %s: %s", resp.Status, strings.TrimSpace(string(message))) + } + if resp.ContentLength == 0 { + return sharedPoolResponse{}, nil + } + var result sharedPoolResponse + if err := json.NewDecoder(io.LimitReader(resp.Body, 4096)).Decode(&result); err != nil && err != io.EOF { + return sharedPoolResponse{}, err + } + return result, nil +} + +func (c *sharedSessionPoolClient) Acquire(ctx context.Context, agentName string) (string, error) { + deadlineCtx := ctx + if _, ok := ctx.Deadline(); !ok { + var cancel context.CancelFunc + deadlineCtx, cancel = context.WithTimeout(ctx, 5*time.Second) + defer cancel() + } + for { + result, err := c.request(deadlineCtx, http.MethodPost, "/leases", sharedPoolRequest{HiveID: c.hiveID, AgentName: agentName}) + if err == nil { + if result.LeaseID == "" { + return "", fmt.Errorf("shared session pool returned an empty lease ID") + } + return result.LeaseID, nil + } + if err != errSharedSessionPoolBusy { + return "", err + } + timer := time.NewTimer(1 * time.Second) + select { + case <-deadlineCtx.Done(): + timer.Stop() + cancelCtx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + _, _ = c.request(cancelCtx, http.MethodDelete, "/leases", sharedPoolRequest{HiveID: c.hiveID, CancelWait: true}) + cancel() + return "", fmt.Errorf("shared session pool is busy: %w", deadlineCtx.Err()) + case <-timer.C: + } + } +} + +func (c *sharedSessionPoolClient) Renew(ctx context.Context, leaseID string) error { + _, err := c.request(ctx, http.MethodPut, "/leases", sharedPoolRequest{HiveID: c.hiveID, LeaseID: leaseID}) + return err +} + +func (c *sharedSessionPoolClient) Release(ctx context.Context, leaseID string) error { + _, err := c.request(ctx, http.MethodDelete, "/leases", sharedPoolRequest{HiveID: c.hiveID, LeaseID: leaseID}) + return err +} + +func (m *Manager) watchSharedPoolLease(ctx context.Context, name, leaseID string) { + ticker := time.NewTicker(20 * time.Second) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + } + m.mu.RLock() + a := m.agents[name] + if a == nil || a.poolLeaseID != leaseID { + m.mu.RUnlock() + return + } + state := a.State + idle := a.LastKick == nil || a.KickOutcome.Settled(a.LastKick) + delivering := a.kickDelivering.Load() + idleSince := time.Time{} + if a.StartedAt != nil { + idleSince = *a.StartedAt + } + if a.KickOutcome.At.After(idleSince) { + idleSince = a.KickOutcome.At + } + m.mu.RUnlock() + if state == StateRunning && idle && !delivering && + time.Since(idleSince) >= sharedPoolIdleEvictionDelay && + paneShowsInputPrompt(m.captureVisiblePaneForAgent(a)) { + m.evictSharedPoolSession(name, leaseID) + return + } + ctx, cancel := context.WithTimeout(context.Background(), 8*time.Second) + err := m.sharedPool.Renew(ctx, leaseID) + cancel() + if err != nil { + m.logger.Warn("shared agent pool lease renewal failed; evicting CLI session", "agent", name, "error", err) + m.evictSharedPoolSession(name, leaseID) + return + } + } +} + +func (m *Manager) evictSharedPoolSession(name, leaseID string) { + m.evictSharedPoolSessionIf(name, leaseID, nil) +} + +func (m *Manager) evictSharedPoolSessionAfterTurn(name, leaseID string, outcome KickOutcome) { + m.evictSharedPoolSessionIf(name, leaseID, &outcome) +} + +func (m *Manager) evictSharedPoolSessionIf(name, leaseID string, expected *KickOutcome) { + m.mu.Lock() + a := m.agents[name] + if a == nil || a.poolLeaseID != leaseID { + m.mu.Unlock() + return + } + if expected != nil && + (!a.KickOutcome.At.Equal(expected.At) || + !a.KickOutcome.KickAt.Equal(expected.KickAt) || + a.KickOutcome.Kind != expected.Kind || + a.KickOutcome.Reason != expected.Reason || + !a.KickOutcome.Settled(a.LastKick) || + a.kickDelivering.Load()) { + m.mu.Unlock() + return + } + if a.cancel != nil { + a.cancel() + } + m.archiveKickLogLocked(a, "shared-pool-eviction") + _ = m.tmuxCmd(a, "kill-session", "-t", a.tmuxSession).Run() + _ = m.reapAgentCLI(a) + a.State = StateStopped + a.poolLeaseID = "" + cancel := a.poolLeaseCancel + a.poolLeaseCancel = nil + m.mu.Unlock() + if cancel != nil { + cancel() + } + ctx, done := context.WithTimeout(context.Background(), 8*time.Second) + if err := m.sharedPool.Release(ctx, leaseID); err != nil { + m.logger.Warn("shared agent pool lease release failed", "agent", name, "error", err) + } + done() + m.logger.Info("shared agent pool evicted idle CLI session", "agent", name) +} + +func (m *Manager) releaseSharedPoolLease(name, leaseID string) { + m.mu.Lock() + a := m.agents[name] + if a != nil && a.poolLeaseID == leaseID { + a.poolLeaseID = "" + cancel := a.poolLeaseCancel + a.poolLeaseCancel = nil + m.mu.Unlock() + if cancel != nil { + cancel() + } + } else { + m.mu.Unlock() + } + ctx, done := context.WithTimeout(context.Background(), 8*time.Second) + defer done() + if err := m.sharedPool.Release(ctx, leaseID); err != nil { + m.logger.Warn("shared agent pool lease release failed", "agent", name, "error", err) + } +} diff --git a/src/crux/agent/shared_pool_test.go b/src/crux/agent/shared_pool_test.go new file mode 100644 index 0000000000..d0d5702923 --- /dev/null +++ b/src/crux/agent/shared_pool_test.go @@ -0,0 +1,135 @@ +package agent + +import ( + "context" + "github.com/hivecommons/hive/crux/config" + "github.com/hivecommons/hive/crux/internal/testutil" + "sync" + "testing" + "time" +) + +type recordingSharedSessionPool struct { + mu sync.Mutex + acquired []string + released []string + renewed []string + acquireFn func(ctx context.Context, agent string) (string, error) +} + +func (p *recordingSharedSessionPool) Acquire(ctx context.Context, agent string) (string, error) { + p.mu.Lock() + p.acquired = append(p.acquired, agent) + fn := p.acquireFn + p.mu.Unlock() + if fn != nil { + return fn(ctx, agent) + } + return "lease-" + agent, nil +} + +func (p *recordingSharedSessionPool) Renew(_ context.Context, id string) error { + p.mu.Lock() + p.renewed = append(p.renewed, id) + p.mu.Unlock() + return nil +} + +func (p *recordingSharedSessionPool) Release(_ context.Context, id string) error { + p.mu.Lock() + p.released = append(p.released, id) + p.mu.Unlock() + return nil +} + +func TestSharedPoolEvictionStopsCLIAndReleasesLease(t *testing.T) { + pool := &recordingSharedSessionPool{} + m := NewManager(map[string]config.AgentConfig{"scanner": {Backend: "claude"}}, discardLogger(), ProjectContext{}) + m.sharedPool = pool + a := m.agents["scanner"] + a.State = StateRunning + a.poolLeaseID = "lease-1" + + m.evictSharedPoolSession("scanner", "lease-1") + + if a.State != StateStopped { + t.Fatalf("agent state = %q, want stopped", a.State) + } + if a.poolLeaseID != "" { + t.Fatalf("agent retained lease %q", a.poolLeaseID) + } + pool.mu.Lock() + defer pool.mu.Unlock() + if len(pool.released) != 1 || pool.released[0] != "lease-1" { + t.Fatalf("released leases = %v, want [lease-1]", pool.released) + } +} + +func TestSharedPoolStopReleasesLease(t *testing.T) { + pool := &recordingSharedSessionPool{} + m := NewManager(map[string]config.AgentConfig{"scanner": {Backend: "claude"}}, discardLogger(), ProjectContext{}) + m.sharedPool = pool + a := m.agents["scanner"] + a.State = StateRunning + a.poolLeaseID = "lease-stop" + + if err := m.Stop("scanner"); err != nil { + t.Fatalf("Stop failed: %v", err) + } + + if a.State != StateStopped { + t.Fatalf("agent state = %q, want stopped", a.State) + } + if a.poolLeaseID != "" { + t.Fatalf("agent retained lease %q", a.poolLeaseID) + } + pool.mu.Lock() + defer pool.mu.Unlock() + if len(pool.released) != 1 || pool.released[0] != "lease-stop" { + t.Fatalf("released leases = %v, want [lease-stop]", pool.released) + } +} + +func TestSharedPoolKickOutcomeEvictsSession(t *testing.T) { + pool := &recordingSharedSessionPool{} + m := NewManager(map[string]config.AgentConfig{"scanner": {Backend: "claude"}}, discardLogger(), ProjectContext{}) + m.sharedPool = pool + a := m.agents["scanner"] + now := time.Now() + a.State = StateRunning + a.StartedAt = &now + a.LastKick = &now + a.poolLeaseID = "lease-outcome" + + outcome := KickOutcome{ + Kind: KickOutcomeEnded, + At: now, + KickAt: now, + } + a.KickOutcome = outcome + + m.notifyKickOutcome("scanner", outcome) + + testutil.EventuallyEvery(t, 2*time.Second, 10*time.Millisecond, func() bool { + m.mu.RLock() + defer m.mu.RUnlock() + return a.State == StateStopped && a.poolLeaseID == "" + }, "session was not evicted after kick outcome") + pool.mu.Lock() + defer pool.mu.Unlock() + if len(pool.released) != 1 || pool.released[0] != "lease-outcome" { + t.Fatalf("released leases = %v, want [lease-outcome]", pool.released) + } +} + +func TestWithSharedSessionPoolOption(t *testing.T) { + pool := &recordingSharedSessionPool{} + m := NewManagerWithOptions(map[string]config.AgentConfig{"scanner": {Backend: "claude"}}, + discardLogger(), + ProjectContext{}, + WithSharedSessionPool(pool), + ) + if m.sharedPool != pool { + t.Fatal("sharedPool option was not applied") + } +} diff --git a/src/crux/runtime/runtime.go b/src/crux/runtime/runtime.go index cee764ad00..bad1b89709 100644 --- a/src/crux/runtime/runtime.go +++ b/src/crux/runtime/runtime.go @@ -55,6 +55,7 @@ type options struct { logger *slog.Logger project crux.ProjectContext uidMapPath *string + sharedPool agent.SharedSessionPool } // WithDefinitions supplies the agent definitions the runtime manages. The @@ -81,6 +82,13 @@ func WithUIDMapPath(path string) Option { return func(o *options) { o.uidMapPath = &path } } +// WithSharedSessionPool coordinates heavyweight CLI sessions through the Hub. +// It affects only the in-process Manager; sidecar mode must implement the same +// lease contract before it can opt in. +func WithSharedSessionPool(pool agent.SharedSessionPool) Option { + return func(o *options) { o.sharedPool = pool } +} + // New validates cfg and returns the runtime its Mode selects. Options are // applied in order; a nil option is ignored. func New(cfg config.RuntimeConfig, opts ...Option) (crux.Runtime, error) { @@ -93,6 +101,9 @@ func New(cfg config.RuntimeConfig, opts ...Option) (crux.Runtime, error) { opt(&o) } } + if o.sharedPool != nil && cfg.Mode != config.ModeInProcess { + return nil, fmt.Errorf("shared session pool is not supported by runtime mode %q", cfg.Mode) + } switch cfg.Mode { case config.ModeInProcess: return newInProcess(o), nil @@ -122,6 +133,9 @@ func newInProcess(o options) crux.Runtime { if o.uidMapPath != nil { mopts = append(mopts, agent.WithUIDMapPath(*o.uidMapPath)) } + if o.sharedPool != nil { + mopts = append(mopts, agent.WithSharedSessionPool(o.sharedPool)) + } return agent.AsRuntime(agent.NewManagerWithOptions(o.defs, o.logger, o.project, mopts...)) } diff --git a/src/crux/runtime/runtime_test.go b/src/crux/runtime/runtime_test.go index 6703ce48ce..d06985fe10 100644 --- a/src/crux/runtime/runtime_test.go +++ b/src/crux/runtime/runtime_test.go @@ -239,3 +239,30 @@ func TestRunAgyTurnHelp(t *testing.T) { t.Fatalf("RunAgyTurn(-h) = %d, stderr %q", code, stderr.String()) } } + +type stubSharedSessionPool struct{} + +func (stubSharedSessionPool) Acquire(context.Context, string) (string, error) { return "lease", nil } +func (stubSharedSessionPool) Renew(context.Context, string) error { return nil } +func (stubSharedSessionPool) Release(context.Context, string) error { return nil } + +func TestWithSharedSessionPool(t *testing.T) { + t.Setenv("HIVE_WORK_DIR", t.TempDir()) + pool := stubSharedSessionPool{} + rt, err := cruxruntime.New(config.RuntimeConfig{Mode: config.ModeInProcess}, + cruxruntime.WithSharedSessionPool(pool), + ) + if err != nil { + t.Fatalf("in-process with shared pool: %v", err) + } + if rt == nil { + t.Fatal("expected non-nil runtime") + } + + _, err = cruxruntime.New(config.RuntimeConfig{Mode: config.ModeSidecar, Server: "http://127.0.0.1:9", Token: "t"}, + cruxruntime.WithSharedSessionPool(pool), + ) + if err == nil || !strings.Contains(err.Error(), "shared session pool is not supported") { + t.Fatalf("sidecar with shared pool err = %v, want unsupported error", err) + } +} diff --git a/src/docs/api-reference.md b/src/docs/api-reference.md index 6ca064ebea..4855b20b3c 100644 --- a/src/docs/api-reference.md +++ b/src/docs/api-reference.md @@ -723,31 +723,34 @@ always resolved server-side from the validated token. | `GET` | `/api/hub/clusters` | Hub auth | List Clusters | `pkg/hub/saas.go:528` | | `GET` | `/api/hub/image-pulls` | Hub auth | Per-Release Image Pull Series | `pkg/hub/saas.go:377` | | `GET` | `/api/reach` | Hub admin | PR Reach Report (?pr=NNN or ?recent=K) | `pkg/hub/saas.go:519` | -| `GET` | `/fleet` | Hub handler-specific | My-Hives Fleet Page (static; data via `/api/saas/my-hives`) | `pkg/hub/server.go:1750` | -| `GET` | `/my-hives` | Hub handler-specific | 301 redirect to `/fleet` (query preserved) | `pkg/hub/server.go:1751` | -| `POST` | `/api/heartbeat` | Hub handler-specific | Heartbeat | `pkg/hub/server.go:1691` | -| `POST` | `/api/task-status` | Hub handler-specific | Task Status | `pkg/hub/server.go:1692` | -| `GET` | `/api/registry` | Hub handler-specific | Registry | `pkg/hub/server.go:1700` | -| `GET` | `/api/hub/leaderboard` | Hub handler-specific | Leaderboard | `pkg/hub/server.go:1701` | -| `GET` | `/api/hub/stats` | Hub handler-specific | Stats | `pkg/hub/server.go:1702` | -| `GET` | `/api/fleet-stats` | Hub handler-specific | Fleet Stats | `pkg/hub/server.go:1703` | +| `GET` | `/fleet` | Hub handler-specific | My-Hives Fleet Page (static; data via `/api/saas/my-hives`) | `pkg/hub/server.go:1758` | +| `GET` | `/my-hives` | Hub handler-specific | 301 redirect to `/fleet` (query preserved) | `pkg/hub/server.go:1759` | +| `POST` | `/api/shared-agent-pool/leases` | Hub handler-specific | Acquire or renew a shared CLI session lease | `pkg/hub/server.go:1696` | +| `PUT` | `/api/shared-agent-pool/leases` | Hub handler-specific | Renew an active shared CLI session lease | `pkg/hub/server.go:1697` | +| `DELETE` | `/api/shared-agent-pool/leases` | Hub handler-specific | Release a shared CLI session lease | `pkg/hub/server.go:1698` | +| `POST` | `/api/heartbeat` | Hub handler-specific | Heartbeat | `pkg/hub/server.go:1699` | +| `POST` | `/api/task-status` | Hub handler-specific | Task Status | `pkg/hub/server.go:1700` | +| `GET` | `/api/registry` | Hub handler-specific | Registry | `pkg/hub/server.go:1708` | +| `GET` | `/api/hub/leaderboard` | Hub handler-specific | Leaderboard | `pkg/hub/server.go:1709` | +| `GET` | `/api/hub/stats` | Hub handler-specific | Stats | `pkg/hub/server.go:1710` | +| `GET` | `/api/fleet-stats` | Hub handler-specific | Fleet Stats | `pkg/hub/server.go:1711` | | `GET` | `/api/hub/evidence` | Hub admin | Read-only cross-hive evidence drill-down from each spoke's stored heartbeat `evidence` summary ([#12116](https://github.com/hivecommons/hive/issues/12116)); non-admins get `403`. Returns `spokes`, `reporting`, pooled `calibration` (bucket 0–5 → `{n, bad, rate, lo, hi}`; the hub view folds the six buckets into four reviewer-confidence ranges), `findings[]` (the plain-language *What this says* sentences, generated from the same fields; [#12456](https://github.com/hivecommons/hive/issues/12456)), `by_acmm_level` (`level`, `hives`, `merged_90d`, `revert_rate`, and `time_to_merge` — `median_of_hive_medians_s`, `hives`, `n` — the median of the level's per-hive median time-to-merge, omitted when no hive at the level reports one; [#12464](https://github.com/hivecommons/hive/issues/12464)), `family_false_approve_rate` (`same`/`cross`/`mixed`, hives classed by their reviewer vs author family sets), fleet `record_coverage` (merged PRs with an evidence record over `prs_merged_90d`), fleet panels ([#12370](https://github.com/hivecommons/hive/issues/12370)) `outcomes_by_author_kind[]` (`kind`, `hives`, `n`, `merged`, `reverted`, `merge_rate`, `revert_rate`), `write_counters[]` (`tier`, `hives`, `attempted`, `blocked`, `block_rate`) and `shadow_gate` (`review_gate` / `intent` → `held` / `passed` outcomes with `revert_rate`), each omitted while no spoke reports it and repeated per `by_acmm_level[]` row, and per hive `hives[]` with `acmm_level`, `prs_merged_90d`, `record_coverage`, `family_mix`, false-approve / false-block / revert rates, families, a calibration series the raw `counts` behind them (including `by_author_kind`) so the drill-down can re-pool a hive / ACMM-level selection client-side, and `criteria[]`: the per-axis criteria table derived hub-side from the quadrant criteria (`axis`, `label`, `scored`, `criteria[]` with `name`, `label`, `status` `met`/`unmet`/`not_reported`, `display`/`score` when measured, `reason` otherwise; offline, evidence-less or unscored hives are `not_reported`, never `unmet`; [#12463](https://github.com/hivecommons/hive/issues/12463)). Every rate is `{n, rate, lo, hi}` with a 95% Wilson interval; `rate`/`lo`/`hi` are null when `n` < 10. Window selection ([RFC #12062](https://github.com/hivecommons/hive/issues/12062)): `?hours=24|168|720` pools each hive's matching heartbeat `windows` block instead of its 90-day block (any other or absent `hours` → 90 days); `?from=YYYY-MM-DD|RFC3339[&to=…]` (date-only `to` is inclusive, `to` defaults to now, `to` ≥ `from`, span ≤ 366 days, else `400`) is served from the smallest reported window that reaches back to `from` and flagged `window.approximate`. The response carries `window` (`hours`, `key`, `label`, `from`, `to`, `approximate`, `coverage_available` — `record_coverage` is only computed for the 90-day window because `prs_merged_90d` is a 90-day count), `window_unreported` and per-hive `window_unreported` for hives whose spoke predates per-window reporting. `?hive=[,…]` (repeatable) and `?level=` restrict the registry server-side to those hives / that ACMM level before pooling. Hub admins open it from the Fleet quadrant card or its **🔬 Evidence** button ([#12117](https://github.com/hivecommons/hive/issues/12117)), pick a window (24h / 7d / 30d / 90d / custom), filter it by hive and ACMM level and read each section's ⓘ explainer; the per-hive author-kind revert counts behind the *Revert rate by author kind* panel are `hives[].counts.by_author_kind` ([#12326](https://github.com/hivecommons/hive/issues/12326)). Never consulted by merge decisions | `pkg/hub/evidence.go` | | `GET` | `/api/hub/evidence/report` | Hub admin | Plain-language report of the hub evidence view ([#12456](https://github.com/hivecommons/hive/issues/12456)), rendered server-side from the same `EvidenceView` as `/api/hub/evidence` so every figure reconciles with the modal; non-admins get `403`. Same `hours=` / `from=` / `to=` window and `hive=` / `level=` filter parameters. `format=md` (default, `text/markdown`) or `format=html` (standalone page, inline CSS, no scripts or external assets, hive names escaped); any other value → `400`. `download=1` sets `Content-Disposition: attachment; filename="hive-evidence---.md|html"`, otherwise inline. Sections: Summary (the generated findings), What we measured and why, The numbers, How to read these, What is not reported yet and why, Grade per section, footer link to `review-evidence.md`. `Cache-Control: no-store`. Read-only; never consulted by merge decisions | `pkg/hub/evidence_report.go` | -| `GET` | `/api/hub/version` | Hub handler-specific | Hub Version | `pkg/hub/server.go:1708` | -| `DELETE` | `/api/hub/registry/{id}` | Hub handler-specific | Registry Delete | `pkg/hub/server.go:1718` | -| `POST` | `/api/contribute/register` | Hub handler-specific | Contribute Proxy | `pkg/hub/server.go:1719` | -| `GET` | `/api/contribute/status` | Hub handler-specific | Contribute Status | `pkg/hub/server.go:1720` | -| `GET` | `/api/contribute/ws` | Hub handler-specific | Contribute WSProxy | `pkg/hub/server.go:1721` | -| `POST` | `/api/github/webhook` | Hub handler-specific | GitHub Webhook | `pkg/hub/server.go:1722` | -| `GET` | `/gh-setup` | Hub handler-specific | GitHub App Setup Router | `pkg/hub/server.go:1723` | -| `GET` | `/learn` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1724` | -| `GET` | `/get-started` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1725` | -| `GET` | `/api/docs` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1728` | -| `GET` | `/api/reading-list` | Hub handler-specific | Reading List | `pkg/hub/server.go:1731` | -| `GET` | `/reading` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1732` | -| `GET` | `/cncf-reference-architecture` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1739` | -| `GET` | `/cncf-reference-architecture/console` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1740` | -| `GET` | `/cncf-reference-architecture/bluefin` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1741` | -| `GET` | `/{$}` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1758` | -| `GET` | `/og-card.png` | Hub handler-specific | OGCard | `pkg/hub/server.go:1763` | -| `GET` | `/` | Public | Static asset fallback (`http.FileServerFS` over the embedded `static/` tree) for any path no other route claims | `pkg/hub/server.go:1765` | +| `GET` | `/api/hub/version` | Hub handler-specific | Hub Version | `pkg/hub/server.go:1716` | +| `DELETE` | `/api/hub/registry/{id}` | Hub handler-specific | Registry Delete | `pkg/hub/server.go:1726` | +| `POST` | `/api/contribute/register` | Hub handler-specific | Contribute Proxy | `pkg/hub/server.go:1727` | +| `GET` | `/api/contribute/status` | Hub handler-specific | Contribute Status | `pkg/hub/server.go:1728` | +| `GET` | `/api/contribute/ws` | Hub handler-specific | Contribute WSProxy | `pkg/hub/server.go:1729` | +| `POST` | `/api/github/webhook` | Hub handler-specific | GitHub Webhook | `pkg/hub/server.go:1730` | +| `GET` | `/gh-setup` | Hub handler-specific | GitHub App Setup Router | `pkg/hub/server.go:1731` | +| `GET` | `/learn` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1732` | +| `GET` | `/get-started` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1733` | +| `GET` | `/api/docs` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1736` | +| `GET` | `/api/reading-list` | Hub handler-specific | Reading List | `pkg/hub/server.go:1739` | +| `GET` | `/reading` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1740` | +| `GET` | `/cncf-reference-architecture` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1747` | +| `GET` | `/cncf-reference-architecture/console` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1748` | +| `GET` | `/cncf-reference-architecture/bluefin` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1749` | +| `GET` | `/{$}` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1766` | +| `GET` | `/og-card.png` | Hub handler-specific | OGCard | `pkg/hub/server.go:1771` | +| `GET` | `/` | Public | Static asset fallback (`http.FileServerFS` over the embedded `static/` tree) for any path no other route claims | `pkg/hub/server.go:1773` | diff --git a/src/docs/env-vars.md b/src/docs/env-vars.md index 07cc69e78b..a04f99010f 100644 --- a/src/docs/env-vars.md +++ b/src/docs/env-vars.md @@ -28,6 +28,9 @@ This reference is compiled by hand from the Go source under `src/`, the deployme | `HIVE_ID` | No | config or generated id | Stable hive/spoke identifier override; passed through to launched agents. | | `HIVE_CLUSTER_ID` | No | config or hub-provisioned value | Hosted cluster identifier override. | | `HIVE_HUB_URL` | No | `hub.url` from config | Hub URL override for spoke heartbeats/registration. On the hub it is also the last environment variable consulted in the hub public-origin chain (see `HIVE_HUB_PUBLIC_URL`). | +| `HIVE_SHARED_AGENT_POOL_ENABLED` | No | `false` | Spoke opt-in for Hub-coordinated CLI session sharing. Requires an in-process runtime and a Hub link; each Hive keeps its own workspace, UID, credentials, and tmux session. Idle CLI sessions are terminated after turns to return the global slot. | +| `HIVE_SHARED_AGENT_POOL_MAX_SESSIONS` | No | `0` (disabled) | Hub-side maximum number of live CLI session leases. Set to `1` to serialize agent sessions across participating Hives. | +| `HIVE_SHARED_AGENT_POOL_LEASE_SECONDS` | No | `90` | Hub-side lease expiry in seconds. Values below 30 or invalid values use the 90-second default. Spokes renew active leases every 20 seconds. | | `HIVE_NPS_ENABLED` | No | `hub.nps_enabled`; when both are unset, on for hosted spokes (`hub.hive_type: hosted`) and off otherwise | Turns the dashboard NPS feedback prompt on or off. Accepts `1/true/yes/on` and `0/false/no/off`; other values are ignored. Responses are forwarded over the spoke's hub link or, for a standalone hive with a relay configured, to the NPS relay, never to anyone else. See [NPS feedback prompt](nps.md). | | `HIVE_NPS_RELAY_URL` | No | `hub.nps_relay_url`; empty (relay disabled) | Base URL of the NPS relay for standalone hives (the hivecommons relay is `https://docs.hivecommons.dev/api/nps`). A spoke with NPS enabled and no hub link self-registers a generated Ed25519 key at `/register` and POSTs signed responses here, with no token to configure; a hub pulls from `/pending` and acks at `/ack`. Must be `https` (plain `http` only to a loopback host). See [NPS feedback prompt](nps.md#standalone-hives-the-nps-relay). | | `HIVE_NPS_RELAY_PULL_SECRET` | No | `hub.nps_relay_pull_secret`; empty | Secret, hub only. Authenticates the hub's periodic pull from the NPS relay. Without it (or without a relay URL) the hub does not pull. Never logged. | diff --git a/src/pkg/hub/hub_testing_helpers_test.go b/src/pkg/hub/hub_testing_helpers_test.go index d0c4efa9bc..21dc2b21e6 100644 --- a/src/pkg/hub/hub_testing_helpers_test.go +++ b/src/pkg/hub/hub_testing_helpers_test.go @@ -83,6 +83,7 @@ var hubTestPathVars = []hubTestPathVar{ {ptr: ®istryPath, prod: registryPath, rel: "hub-registry.json", always: true}, {ptr: &hubSecretPath, prod: hubSecretPath, rel: filepath.Join("saas", "hub-secret.key"), always: true}, {ptr: &hubBannersPath, prod: hubBannersPath, rel: filepath.Join("saas", "hub-banners.json")}, + {ptr: &sharedAgentPoolPath, prod: sharedAgentPoolPath, rel: filepath.Join("saas", "shared-agent-pool.json")}, {ptr: &stablePromotionPath, prod: stablePromotionPath, rel: filepath.Join("saas", "stable-promotion.json")}, {ptr: &hubGenerationsPath, prod: hubGenerationsPath, rel: filepath.Join("saas", "hub-generations.json")}, {ptr: &clustersConfigPath, prod: clustersConfigPath, rel: filepath.Join("saas", "clusters.json")}, diff --git a/src/pkg/hub/server.go b/src/pkg/hub/server.go index 59a742f979..eff1145b11 100644 --- a/src/pkg/hub/server.go +++ b/src/pkg/hub/server.go @@ -1032,12 +1032,13 @@ type HubServer struct { // goroutine — which outlives test servers, since nothing closes saveCh — // only ever sees its own path and cannot race a test redirecting the // global for the next server (the TestLoadRegistry -race failure). - registryPath string - hubGitHash string - hubGitBranch string - hubSecret string - myHivesCacheMu sync.Mutex - myHivesCache map[string]myHivesSnapshot + registryPath string + hubGitHash string + hubGitBranch string + sharedAgentPool *sharedAgentPool + hubSecret string + myHivesCacheMu sync.Mutex + myHivesCache map[string]myHivesSnapshot // keyGenerations is the ordered set of master generations this hub accepts // (hub_generations.go). Until a rotation happens it holds exactly ONE // generation whose secret IS hubSecret, so every derived key is @@ -1540,15 +1541,16 @@ func NewHubServer(port int, logger *slog.Logger, gitHash, gitBranch string) *Hub logger.Info("generated hub secret", "path", hubSecretPath) } s := &HubServer{ - mux: http.NewServeMux(), - logger: logger, - saveCh: make(chan struct{}, 1), - saveLoopStop: make(chan struct{}), - saveLoopDone: make(chan struct{}), - registryPath: registryPath, - hubGitHash: gitHash, - hubGitBranch: gitBranch, - hubSecret: secret, + mux: http.NewServeMux(), + logger: logger, + saveCh: make(chan struct{}, 1), + saveLoopStop: make(chan struct{}), + saveLoopDone: make(chan struct{}), + registryPath: registryPath, + hubGitHash: gitHash, + hubGitBranch: gitBranch, + sharedAgentPool: newSharedAgentPool(), + hubSecret: secret, // Provisional single generation holding the existing master. Replaced // immediately below by loadGenerations, which reads any persisted // rotation off the PVC. It is set here too so the field is never nil @@ -1688,6 +1690,12 @@ func NewHubServer(port int, logger *slog.Logger, gitHash, gitBranch string) *Hub // a hub that has just rolled never honours a session revoked before it. s.loadRevokedSessions() + handleSharedPoolLeases := func(w http.ResponseWriter, r *http.Request) { + s.sharedAgentPool.handle(w, r, s) + } + s.mux.HandleFunc("POST /api/shared-agent-pool/leases", handleSharedPoolLeases) + s.mux.HandleFunc("PUT /api/shared-agent-pool/leases", handleSharedPoolLeases) + s.mux.HandleFunc("DELETE /api/shared-agent-pool/leases", handleSharedPoolLeases) s.mux.HandleFunc("POST /api/heartbeat", s.handleHeartbeat) s.mux.HandleFunc("POST /api/task-status", s.handleTaskStatus) // NPS feedback forwarded by spokes (#9610): per-hive heartbeat bearer, the diff --git a/src/pkg/hub/shared_agent_pool.go b/src/pkg/hub/shared_agent_pool.go new file mode 100644 index 0000000000..10d649d73e --- /dev/null +++ b/src/pkg/hub/shared_agent_pool.go @@ -0,0 +1,268 @@ +package hub + +import ( + "crypto/rand" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "net/http" + "os" + "path/filepath" + "strconv" + "strings" + "sync" + "time" +) + +var sharedAgentPoolPath = "/data/saas/shared-agent-pool.json" + +const defaultSharedAgentLeaseTTL = 90 * time.Second + +// A waiter that has not retried within this window is treated as gone so a +// crashed spoke cannot starve the fleet. +const sharedPoolWaiterStaleAfter = 15 * time.Second + +type sharedAgentLease struct { + ID string `json:"id"` + HiveID string `json:"hive_id"` + AgentName string `json:"agent_name"` + ExpiresAt time.Time `json:"expires_at"` +} + +type sharedAgentPoolState struct { + Version int `json:"version"` + Leases []sharedAgentLease `json:"leases"` +} + +type sharedAgentPool struct { + mu sync.Mutex + path string + max int + ttl time.Duration + leases map[string]sharedAgentLease + waitingHives map[string]time.Time // hive_id -> first 409 (FIFO position) + lastSeen map[string]time.Time // hive_id -> last retry + loadError error +} + +func newSharedAgentPool() *sharedAgentPool { + max, _ := strconv.Atoi(strings.TrimSpace(os.Getenv("HIVE_SHARED_AGENT_POOL_MAX_SESSIONS"))) + ttl := defaultSharedAgentLeaseTTL + if seconds, err := strconv.Atoi(strings.TrimSpace(os.Getenv("HIVE_SHARED_AGENT_POOL_LEASE_SECONDS"))); err == nil && seconds >= 30 { + ttl = time.Duration(seconds) * time.Second + } + p := &sharedAgentPool{ + path: sharedAgentPoolPath, + max: max, + ttl: ttl, + leases: make(map[string]sharedAgentLease), + waitingHives: make(map[string]time.Time), + lastSeen: make(map[string]time.Time), + } + p.load() + return p +} + +func sharedPoolOwnerKey(hiveID, agentName string) string { return hiveID + "\x00" + agentName } + +func (p *sharedAgentPool) load() { + data, err := os.ReadFile(p.path) + if errors.Is(err, os.ErrNotExist) { + return + } + if err != nil { + p.loadError = err + return + } + var state sharedAgentPoolState + if err := json.Unmarshal(data, &state); err != nil || state.Version != 1 { + p.loadError = fmt.Errorf("invalid shared agent pool state") + return + } + now := time.Now() + for _, lease := range state.Leases { + if lease.ID == "" || lease.HiveID == "" || lease.AgentName == "" || !lease.ExpiresAt.After(now) { + continue + } + p.leases[sharedPoolOwnerKey(lease.HiveID, lease.AgentName)] = lease + } +} + +func (p *sharedAgentPool) saveLocked() error { + state := sharedAgentPoolState{Version: 1, Leases: make([]sharedAgentLease, 0, len(p.leases))} + for _, lease := range p.leases { + state.Leases = append(state.Leases, lease) + } + data, err := json.Marshal(state) + if err != nil { + return err + } + if err := os.MkdirAll(filepath.Dir(p.path), 0o755); err != nil { + return err + } + tmpPath := p.path + ".tmp" + if err := os.WriteFile(tmpPath, data, 0o600); err != nil { + return err + } + if err := os.Rename(tmpPath, p.path); err != nil { + _ = os.Remove(tmpPath) + return err + } + return nil +} + +// hasEarlierWaitingHive reports whether another hive has been waiting longer +// than hiveID (which may not be waiting yet). Stale waiters are pruned first. +func (p *sharedAgentPool) hasEarlierWaitingHive(hiveID string, now time.Time) bool { + if p.waitingHives == nil { + p.waitingHives = make(map[string]time.Time) + } + if p.lastSeen == nil { + p.lastSeen = make(map[string]time.Time) + } + for h := range p.waitingHives { + if now.Sub(p.lastSeen[h]) > sharedPoolWaiterStaleAfter { + delete(p.waitingHives, h) + delete(p.lastSeen, h) + } + } + mySince, waiting := p.waitingHives[hiveID] + if !waiting { + mySince = now + } + for h, since := range p.waitingHives { + if h != hiveID && since.Before(mySince) { + return true + } + } + return false +} + +func (p *sharedAgentPool) handle(w http.ResponseWriter, r *http.Request, s *HubServer) { + if p == nil || p.max < 1 { + http.Error(w, "shared agent pool is disabled", http.StatusServiceUnavailable) + return + } + if p.loadError != nil { + s.logger.Error("shared agent pool state is untrusted; refusing leases", "error", p.loadError) + http.Error(w, "shared agent pool state unavailable", http.StatusServiceUnavailable) + return + } + auth := r.Header.Get("Authorization") + if !strings.HasPrefix(auth, "Bearer ") { + http.Error(w, "unauthorized", http.StatusUnauthorized) + return + } + var req struct { + HiveID string `json:"hive_id"` + AgentName string `json:"agent_name"` + LeaseID string `json:"lease_id"` + CancelWait bool `json:"cancel_wait"` + } + if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 4096)).Decode(&req); err != nil || req.HiveID == "" { + http.Error(w, "invalid request", http.StatusBadRequest) + return + } + if !s.verifyHeartbeatBearer(strings.TrimPrefix(auth, "Bearer "), req.HiveID) { + http.Error(w, "unauthorized", http.StatusUnauthorized) + return + } + p.mu.Lock() + defer p.mu.Unlock() + now := time.Now() + for key, lease := range p.leases { + if !lease.ExpiresAt.After(now) { + delete(p.leases, key) + } + } + switch r.Method { + case http.MethodPost: + if req.AgentName == "" || len(req.AgentName) > 128 { + http.Error(w, "invalid agent name", http.StatusBadRequest) + return + } + key := sharedPoolOwnerKey(req.HiveID, req.AgentName) + if lease, ok := p.leases[key]; ok { + lease.ExpiresAt = now.Add(p.ttl) + p.leases[key] = lease + delete(p.waitingHives, req.HiveID) + delete(p.lastSeen, req.HiveID) + if err := p.saveLocked(); err != nil { + http.Error(w, "shared agent pool persistence failed", http.StatusServiceUnavailable) + return + } + writeSharedPoolLease(w, lease.ID) + return + } + if len(p.leases) >= p.max || p.hasEarlierWaitingHive(req.HiveID, now) { + if p.waitingHives == nil { + p.waitingHives = make(map[string]time.Time) + } + if p.lastSeen == nil { + p.lastSeen = make(map[string]time.Time) + } + if _, ok := p.waitingHives[req.HiveID]; !ok { + p.waitingHives[req.HiveID] = now + } + p.lastSeen[req.HiveID] = now + w.Header().Set("Retry-After", "3") + http.Error(w, "shared agent pool is busy", http.StatusConflict) + return + } + var raw [16]byte + if _, err := rand.Read(raw[:]); err != nil { + http.Error(w, "lease generation failed", http.StatusInternalServerError) + return + } + lease := sharedAgentLease{ID: hex.EncodeToString(raw[:]), HiveID: req.HiveID, AgentName: req.AgentName, ExpiresAt: now.Add(p.ttl)} + p.leases[key] = lease + delete(p.waitingHives, req.HiveID) + delete(p.lastSeen, req.HiveID) + if err := p.saveLocked(); err != nil { + delete(p.leases, key) + http.Error(w, "shared agent pool persistence failed", http.StatusServiceUnavailable) + return + } + writeSharedPoolLease(w, lease.ID) + case http.MethodPut, http.MethodDelete: + if r.Method == http.MethodDelete && req.CancelWait { + delete(p.waitingHives, req.HiveID) + delete(p.lastSeen, req.HiveID) + w.WriteHeader(http.StatusNoContent) + return + } + if req.LeaseID == "" { + http.Error(w, "lease ID required", http.StatusBadRequest) + return + } + var key string + for candidate, lease := range p.leases { + if lease.ID == req.LeaseID && lease.HiveID == req.HiveID { + key = candidate + break + } + } + if key == "" { + http.Error(w, "lease not found", http.StatusNotFound) + return + } + if r.Method == http.MethodPut { + lease := p.leases[key] + lease.ExpiresAt = now.Add(p.ttl) + p.leases[key] = lease + } else { + delete(p.leases, key) + } + if err := p.saveLocked(); err != nil { + http.Error(w, "shared agent pool persistence failed", http.StatusServiceUnavailable) + return + } + w.WriteHeader(http.StatusNoContent) + } +} + +func writeSharedPoolLease(w http.ResponseWriter, leaseID string) { + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]string{"lease_id": leaseID}) +} diff --git a/src/pkg/hub/shared_agent_pool_test.go b/src/pkg/hub/shared_agent_pool_test.go new file mode 100644 index 0000000000..745f9db6bd --- /dev/null +++ b/src/pkg/hub/shared_agent_pool_test.go @@ -0,0 +1,196 @@ +package hub + +import ( + "bytes" + "context" + "encoding/json" + "log/slog" + "net/http" + "net/http/httptest" + "path/filepath" + "testing" + "time" + + cruxagent "github.com/hivecommons/hive/crux/agent" +) + +func serveSharedPoolRequest(t *testing.T, pool *sharedAgentPool, s *HubServer, method, bearerHive string, body map[string]string) *httptest.ResponseRecorder { + t.Helper() + payload, err := json.Marshal(body) + if err != nil { + t.Fatal(err) + } + req := httptest.NewRequest(method, "/api/shared-agent-pool/leases", bytes.NewReader(payload)) + req.Header.Set("Authorization", "Bearer "+s.heartbeatKeyFor(bearerHive)) + rec := httptest.NewRecorder() + pool.handle(rec, req, s) + return rec +} + +func TestSharedAgentPoolTenantAuthCapacityAndPersistence(t *testing.T) { + path := filepath.Join(t.TempDir(), "shared-agent-pool.json") + pool := &sharedAgentPool{path: path, max: 1, ttl: time.Minute, leases: make(map[string]sharedAgentLease)} + s := &HubServer{logger: slog.Default()} + s.setHubSecret("test-shared-agent-pool-master") + + school := map[string]string{"hive_id": "school", "agent_name": "scanner"} + firstRec := serveSharedPoolRequest(t, pool, s, http.MethodPost, "school", school) + if firstRec.Code != http.StatusOK { + t.Fatalf("first acquire status = %d, body %q", firstRec.Code, firstRec.Body.String()) + } + var first struct { + LeaseID string `json:"lease_id"` + } + if err := json.Unmarshal(firstRec.Body.Bytes(), &first); err != nil || first.LeaseID == "" { + t.Fatalf("first lease response = %q, err %v", firstRec.Body.String(), err) + } + + spoof := map[string]string{"hive_id": "reef", "agent_name": "scanner"} + spoofRec := serveSharedPoolRequest(t, pool, s, http.MethodPost, "school", spoof) + if spoofRec.Code != http.StatusUnauthorized { + t.Fatalf("cross-tenant acquire status = %d, want 401", spoofRec.Code) + } + + reef := map[string]string{"hive_id": "reef", "agent_name": "reviewer"} + busyRec := serveSharedPoolRequest(t, pool, s, http.MethodPost, "reef", reef) + if busyRec.Code != http.StatusConflict { + t.Fatalf("second tenant acquire status = %d, want 409", busyRec.Code) + } + + restored := &sharedAgentPool{path: path, max: 1, ttl: time.Minute, leases: make(map[string]sharedAgentLease)} + restored.load() + if restored.loadError != nil || len(restored.leases) != 1 { + t.Fatalf("restored pool: leases=%d loadError=%v", len(restored.leases), restored.loadError) + } + lease := restored.leases[sharedPoolOwnerKey("school", "scanner")] + if lease.ID != first.LeaseID { + t.Fatalf("restored lease ID = %q, want %q", lease.ID, first.LeaseID) + } + + release := map[string]string{"hive_id": "school", "lease_id": first.LeaseID} + releaseRec := serveSharedPoolRequest(t, restored, s, http.MethodDelete, "school", release) + if releaseRec.Code != http.StatusNoContent { + t.Fatalf("release status = %d, want 204 (%s)", releaseRec.Code, releaseRec.Body.String()) + } + + acquireRec := serveSharedPoolRequest(t, restored, s, http.MethodPost, "reef", reef) + if acquireRec.Code != http.StatusOK { + t.Fatalf("acquire after release status = %d, want 200 (%s)", acquireRec.Code, acquireRec.Body.String()) + } +} + +func TestSharedAgentPoolHTTPSerializesTwoHives(t *testing.T) { + t.Setenv("HIVE_SHARED_AGENT_POOL_MAX_SESSIONS", "1") + srv := newHubServerForTest(t) + ts := httptest.NewServer(srv.mux) + t.Cleanup(ts.Close) + + school, err := cruxagent.NewSharedSessionPoolClient(ts.URL, "school", srv.heartbeatKeyFor("school")) + if err != nil { + t.Fatal(err) + } + reef, err := cruxagent.NewSharedSessionPoolClient(ts.URL, "reef", srv.heartbeatKeyFor("reef")) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + lease, err := school.Acquire(ctx, "scanner") + if err != nil { + t.Fatalf("school acquire: %v", err) + } + + busyCtx, busyCancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer busyCancel() + if _, err := reef.Acquire(busyCtx, "reviewer"); err == nil { + t.Fatal("reef acquired a second live CLI slot while school held the only slot") + } + if err := school.Release(ctx, lease); err != nil { + t.Fatalf("school release: %v", err) + } + reefCtx, reefCancel := context.WithTimeout(context.Background(), time.Second) + defer reefCancel() + if _, err := reef.Acquire(reefCtx, "reviewer"); err != nil { + t.Fatalf("reef acquire after school release: %v", err) + } +} + +func TestSharedAgentPoolFIFOWaitersAndStaleExpiry(t *testing.T) { + path := filepath.Join(t.TempDir(), "shared-agent-pool.json") + pool := &sharedAgentPool{ + path: path, + max: 1, + ttl: time.Minute, + leases: make(map[string]sharedAgentLease), + waitingHives: make(map[string]time.Time), + lastSeen: make(map[string]time.Time), + } + s := &HubServer{logger: slog.Default()} + s.setHubSecret("test-fifo-waiters-master") + + h0 := map[string]string{"hive_id": "h0", "agent_name": "a"} + h1 := map[string]string{"hive_id": "h1", "agent_name": "b"} + h2 := map[string]string{"hive_id": "h2", "agent_name": "c"} + + // h0 acquires the slot. + r0 := serveSharedPoolRequest(t, pool, s, http.MethodPost, "h0", h0) + if r0.Code != http.StatusOK { + t.Fatalf("h0 acquire status = %d", r0.Code) + } + var lease0 struct { + LeaseID string `json:"lease_id"` + } + _ = json.Unmarshal(r0.Body.Bytes(), &lease0) + + // h1 tries and gets 409, becomes first waiter. + r1 := serveSharedPoolRequest(t, pool, s, http.MethodPost, "h1", h1) + if r1.Code != http.StatusConflict { + t.Fatalf("h1 status = %d, want 409", r1.Code) + } + time.Sleep(10 * time.Millisecond) + + // h2 tries and gets 409, becomes second waiter. + r2 := serveSharedPoolRequest(t, pool, s, http.MethodPost, "h2", h2) + if r2.Code != http.StatusConflict { + t.Fatalf("h2 status = %d, want 409", r2.Code) + } + + // h0 releases. + rel := map[string]string{"hive_id": "h0", "lease_id": lease0.LeaseID} + rRel := serveSharedPoolRequest(t, pool, s, http.MethodDelete, "h0", rel) + if rRel.Code != http.StatusNoContent { + t.Fatalf("h0 release status = %d", rRel.Code) + } + + // h2 tries to acquire before h1, but h1 is earlier in queue -> 409. + r2Blocked := serveSharedPoolRequest(t, pool, s, http.MethodPost, "h2", h2) + if r2Blocked.Code != http.StatusConflict { + t.Fatalf("h2 status = %d, want 409 (h1 was earlier waiter)", r2Blocked.Code) + } + + // h1 acquires -> 200 OK. + r1Acq := serveSharedPoolRequest(t, pool, s, http.MethodPost, "h1", h1) + if r1Acq.Code != http.StatusOK { + t.Fatalf("h1 acquire status = %d, want 200", r1Acq.Code) + } + var lease1 struct { + LeaseID string `json:"lease_id"` + } + _ = json.Unmarshal(r1Acq.Body.Bytes(), &lease1) + + // h1 releases. + rel1 := map[string]string{"hive_id": "h1", "lease_id": lease1.LeaseID} + serveSharedPoolRequest(t, pool, s, http.MethodDelete, "h1", rel1) + + // Now make h2 stale (simulate h2 crashing and not retrying for > 15s). + pool.mu.Lock() + pool.lastSeen["h2"] = time.Now().Add(-20 * time.Second) + pool.mu.Unlock() + + // Fresh hive h3 can acquire because h2 is stale and pruned! + h3 := map[string]string{"hive_id": "h3", "agent_name": "d"} + r3 := serveSharedPoolRequest(t, pool, s, http.MethodPost, "h3", h3) + if r3.Code != http.StatusOK { + t.Fatalf("h3 acquire status = %d, want 200 after stale waiter pruned", r3.Code) + } +}