From 873220f6523287a4b9b9b69fd8d240d6a93afd36 Mon Sep 17 00:00:00 2001 From: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Thu, 29 Oct 2026 04:53:41 -0400 Subject: [PATCH 1/4] =?UTF-8?q?=F0=9F=8C=B1=20chore(crux):=20split=20manag?= =?UTF-8?q?er=5Frestart.go=20under=20the=20crux=20file-size=20ratchet?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- src/crux/agent/manager_restart.go | 138 ----------------- .../agent/manager_restart_bob_relaunch.go | 144 ++++++++++++++++++ 2 files changed, 144 insertions(+), 138 deletions(-) create mode 100644 src/crux/agent/manager_restart_bob_relaunch.go diff --git a/src/crux/agent/manager_restart.go b/src/crux/agent/manager_restart.go index 4d09c1645d..fd4e074e4e 100644 --- a/src/crux/agent/manager_restart.go +++ b/src/crux/agent/manager_restart.go @@ -251,144 +251,6 @@ func (m *Manager) CheckAndRestartCrashedAgents(ctx context.Context) []string { return restarted } -// RelaunchBobAgentsAwaitingKey restarts the bob-backend agents that -// launchInTmux parked in StateFailed because no API key was configured, and -// returns their names. It is the "absent → present" half of the key lifecycle: -// the resolver makes a newly-saved key visible to the NEXT launch, but nothing -// carries it into a session whose shell already exists, so an agent parked for -// a missing key would otherwise sit at bob's key prompt until someone -// intervened. -// -// Each relaunch RECREATES the tmux session rather than just re-typing the -// launch command — see killSessionForRelaunch for why that is required and not -// merely tidy. -// -// Call it after a key is stored. It is a no-op — returning nil — when the key -// still does not resolve, so a clear (or a failed save) never launches -// anything. -// -// Selection is deliberately narrow: awaitingBobKey is set on exactly one -// branch of launchInTmux and cleared on every launch attempt, so a running -// agent, a non-bob agent, and an agent failed for any other reason are all -// skipped. An operator-paused agent is skipped too — a key save must not -// override a deliberate pause. That also makes a double save harmless: the -// first relaunch clears the flag, so the second finds nothing to do. -// -// Locking: candidates are collected under RLock, the lock is RELEASED, and -// only then is Start called — Start takes m.mu.Lock() itself, and m.mu is a -// non-reentrant RWMutex, so calling it under the read lock would deadlock -// (incident #1980→#1988). This mirrors CheckAndRestartCrashedAgents. -func (m *Manager) RelaunchBobAgentsAwaitingKey(ctx context.Context) []string { - // The key must actually resolve now; otherwise every relaunch would just - // re-park on the same branch. Read lock-free via the atomic resolver, and - // never bind or log the value — only its presence matters here. - if m.bobAPIKey() == "" { - return nil - } - - candidates := m.bobAgentsAwaitingKey() - - var relaunched []string - for _, name := range candidates { - m.logger.Info("relaunching bob agent parked for missing API key", "name", name) - // Kill the stale tmux session FIRST. This is load-bearing, not - // hygiene: BOBSHELL_API_KEY is Secret, so buildEnvPrefix omits it from - // the typed command line and it is delivered ONLY by tmux - // set-environment — which updates the SESSION environment and is - // inherited just by shells created afterwards. The pane's bash was - // started before the key existed, so it does not have the variable and - // never will; reapAgentCLI kills the bob CLI but leaves that bash - // alive, so a plain relaunch retypes the command into the same - // key-less shell and bob prompts for a key again (observed on - // hosted-available-vllmd-01: BOBSHELL_API_KEY present in the tmux - // session env, absent from every bob /proc//environ). - // Killing the session makes ensureTmuxSession build a new one and - // re-run set-environment before any shell exists, so the fresh bash — - // and the bob it spawns — inherit the key. - if err := m.killSessionForRelaunch(name); err != nil { - m.logger.Warn("could not kill stale session before bob relaunch; launching anyway", - "name", name, "error", err) - } - // The saved key is new information about the exact condition that - // parked this agent, so the backoff must not hold the retry (#5958). - m.mu.Lock() - m.clearStartFailureLocked(m.agents[name]) - m.mu.Unlock() - if err := m.Start(ctx, name); err != nil { - // One agent's failure must never abort the rest of the fleet, - // exactly as in the crash-restart loop above. - m.logger.Error("failed to relaunch bob agent after key save", "name", name, "error", err) - continue - } - relaunched = append(relaunched, name) - } - return relaunched -} - -// killSessionForRelaunch tears down an agent's tmux session so the next -// ensureTmuxSession creates a fresh one whose shell inherits the current -// set-environment values (including a newly-saved BOBSHELL_API_KEY). -// -// This is KillSession's behaviour, but it deliberately does not call -// KillSession: that method takes m.mu.Lock() and is a public entry point. -// Keeping a private helper with the same lock discipline (acquire, act, -// release — never held across Start) keeps the relaunch loop's locking -// obvious and avoids any temptation to hold a lock across the launch, which -// is the deadlock class from incident #1980→#1988. -func (m *Manager) killSessionForRelaunch(name string) error { - m.mu.Lock() - defer m.mu.Unlock() - - agent, ok := m.agents[name] - if !ok { - return fmt.Errorf("agent %s not found", name) - } - // Not an error if the session is already gone — the goal is "no stale - // shell", which a missing session already satisfies. - _ = m.tmuxCmd(agent, "kill-session", "-t", agent.tmuxSession).Run() - // State must not stay Running: Start refuses to launch a running agent, - // and the session backing that state no longer exists. - if agent.State == StateRunning { - agent.State = StateStopped - } - m.logger.Info("killed stale tmux session so the fresh shell inherits the bob key", - "name", name, "session", agent.tmuxSession) - return nil -} - -// bobAgentsAwaitingKey returns the names of agents parked by launchInTmux for a -// missing bob API key and eligible to be started now. Split out from -// RelaunchBobAgentsAwaitingKey so the selection rules can be tested without -// tmux: this is the part that must never pick up a running, paused, or non-bob -// agent. -// -// Takes m.mu.RLock and releases it before returning; the caller must NOT hold -// m.mu (non-reentrant RWMutex). -func (m *Manager) bobAgentsAwaitingKey() []string { - m.mu.RLock() - defer m.mu.RUnlock() - - var candidates []string - for name, agent := range m.agents { - if agent == nil || !agent.awaitingBobKey { - continue - } - // Defensive: awaitingBobKey is only ever set on the bob branch, but - // re-check the effective backend so a backend override applied after - // parking cannot smuggle a non-bob agent into this path. - if effectiveBackend(agent) != bobBackend { - continue - } - // Never disturb a healthy agent, and never override a pause — a key - // save is not a resume. - if agent.State == StateRunning || agent.Paused { - continue - } - candidates = append(candidates, name) - } - return candidates -} - func (m *Manager) KillSession(name string) error { m.mu.Lock() defer m.mu.Unlock() diff --git a/src/crux/agent/manager_restart_bob_relaunch.go b/src/crux/agent/manager_restart_bob_relaunch.go new file mode 100644 index 0000000000..64e5adbc87 --- /dev/null +++ b/src/crux/agent/manager_restart_bob_relaunch.go @@ -0,0 +1,144 @@ +package agent + +import ( + "context" + "fmt" +) + +// RelaunchBobAgentsAwaitingKey restarts the bob-backend agents that +// launchInTmux parked in StateFailed because no API key was configured, and +// returns their names. It is the "absent → present" half of the key lifecycle: +// the resolver makes a newly-saved key visible to the NEXT launch, but nothing +// carries it into a session whose shell already exists, so an agent parked for +// a missing key would otherwise sit at bob's key prompt until someone +// intervened. +// +// Each relaunch RECREATES the tmux session rather than just re-typing the +// launch command — see killSessionForRelaunch for why that is required and not +// merely tidy. +// +// Call it after a key is stored. It is a no-op — returning nil — when the key +// still does not resolve, so a clear (or a failed save) never launches +// anything. +// +// Selection is deliberately narrow: awaitingBobKey is set on exactly one +// branch of launchInTmux and cleared on every launch attempt, so a running +// agent, a non-bob agent, and an agent failed for any other reason are all +// skipped. An operator-paused agent is skipped too — a key save must not +// override a deliberate pause. That also makes a double save harmless: the +// first relaunch clears the flag, so the second finds nothing to do. +// +// Locking: candidates are collected under RLock, the lock is RELEASED, and +// only then is Start called — Start takes m.mu.Lock() itself, and m.mu is a +// non-reentrant RWMutex, so calling it under the read lock would deadlock +// (incident #1980→#1988). This mirrors CheckAndRestartCrashedAgents. +func (m *Manager) RelaunchBobAgentsAwaitingKey(ctx context.Context) []string { + // The key must actually resolve now; otherwise every relaunch would just + // re-park on the same branch. Read lock-free via the atomic resolver, and + // never bind or log the value — only its presence matters here. + if m.bobAPIKey() == "" { + return nil + } + + candidates := m.bobAgentsAwaitingKey() + + var relaunched []string + for _, name := range candidates { + m.logger.Info("relaunching bob agent parked for missing API key", "name", name) + // Kill the stale tmux session FIRST. This is load-bearing, not + // hygiene: BOBSHELL_API_KEY is Secret, so buildEnvPrefix omits it from + // the typed command line and it is delivered ONLY by tmux + // set-environment — which updates the SESSION environment and is + // inherited just by shells created afterwards. The pane's bash was + // started before the key existed, so it does not have the variable and + // never will; reapAgentCLI kills the bob CLI but leaves that bash + // alive, so a plain relaunch retypes the command into the same + // key-less shell and bob prompts for a key again (observed on + // hosted-available-vllmd-01: BOBSHELL_API_KEY present in the tmux + // session env, absent from every bob /proc//environ). + // Killing the session makes ensureTmuxSession build a new one and + // re-run set-environment before any shell exists, so the fresh bash — + // and the bob it spawns — inherit the key. + if err := m.killSessionForRelaunch(name); err != nil { + m.logger.Warn("could not kill stale session before bob relaunch; launching anyway", + "name", name, "error", err) + } + // The saved key is new information about the exact condition that + // parked this agent, so the backoff must not hold the retry (#5958). + m.mu.Lock() + m.clearStartFailureLocked(m.agents[name]) + m.mu.Unlock() + if err := m.Start(ctx, name); err != nil { + // One agent's failure must never abort the rest of the fleet, + // exactly as in the crash-restart loop above. + m.logger.Error("failed to relaunch bob agent after key save", "name", name, "error", err) + continue + } + relaunched = append(relaunched, name) + } + return relaunched +} + +// killSessionForRelaunch tears down an agent's tmux session so the next +// ensureTmuxSession creates a fresh one whose shell inherits the current +// set-environment values (including a newly-saved BOBSHELL_API_KEY). +// +// This is KillSession's behaviour, but it deliberately does not call +// KillSession: that method takes m.mu.Lock() and is a public entry point. +// Keeping a private helper with the same lock discipline (acquire, act, +// release — never held across Start) keeps the relaunch loop's locking +// obvious and avoids any temptation to hold a lock across the launch, which +// is the deadlock class from incident #1980→#1988. +func (m *Manager) killSessionForRelaunch(name string) error { + m.mu.Lock() + defer m.mu.Unlock() + + agent, ok := m.agents[name] + if !ok { + return fmt.Errorf("agent %s not found", name) + } + // Not an error if the session is already gone — the goal is "no stale + // shell", which a missing session already satisfies. + _ = m.tmuxCmd(agent, "kill-session", "-t", agent.tmuxSession).Run() + // State must not stay Running: Start refuses to launch a running agent, + // and the session backing that state no longer exists. + if agent.State == StateRunning { + agent.State = StateStopped + } + m.logger.Info("killed stale tmux session so the fresh shell inherits the bob key", + "name", name, "session", agent.tmuxSession) + return nil +} + +// bobAgentsAwaitingKey returns the names of agents parked by launchInTmux for a +// missing bob API key and eligible to be started now. Split out from +// RelaunchBobAgentsAwaitingKey so the selection rules can be tested without +// tmux: this is the part that must never pick up a running, paused, or non-bob +// agent. +// +// Takes m.mu.RLock and releases it before returning; the caller must NOT hold +// m.mu (non-reentrant RWMutex). +func (m *Manager) bobAgentsAwaitingKey() []string { + m.mu.RLock() + defer m.mu.RUnlock() + + var candidates []string + for name, agent := range m.agents { + if agent == nil || !agent.awaitingBobKey { + continue + } + // Defensive: awaitingBobKey is only ever set on the bob branch, but + // re-check the effective backend so a backend override applied after + // parking cannot smuggle a non-bob agent into this path. + if effectiveBackend(agent) != bobBackend { + continue + } + // Never disturb a healthy agent, and never override a pause — a key + // save is not a resume. + if agent.State == StateRunning || agent.Paused { + continue + } + candidates = append(candidates, name) + } + return candidates +} From 326c1de2ae77c12e9e4766fa28c6747efceb63cb Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Thu, 29 Oct 2026 08:54:32 +0000 Subject: [PATCH 2/4] changelog: fragment for #15362 Generated by changelog-fragment-guard from the PR title (#5675). Edit the wording freely, or replace this file with the no-changelog label if the change is not user-visible. Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Signed-off-by: hivecommons-hive[bot] <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- changelog.d/changed-15362.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/changed-15362.md diff --git a/changelog.d/changed-15362.md b/changelog.d/changed-15362.md new file mode 100644 index 0000000000..71f8605d2a --- /dev/null +++ b/changelog.d/changed-15362.md @@ -0,0 +1 @@ +- chore(crux): split manager_restart.go under the crux file-size ratchet (#15362) From dbb7f4530a4ce1e1f158b29993f36d01c6db4fda Mon Sep 17 00:00:00 2001 From: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Thu, 29 Oct 2026 04:57:40 -0400 Subject: [PATCH 3/4] =?UTF-8?q?=F0=9F=93=96=20docs:=20update=20agent-turn-?= =?UTF-8?q?model=20citations=20after=20manager=5Frestart.go=20split?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- src/docs/design/agent-turn-model.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/docs/design/agent-turn-model.md b/src/docs/design/agent-turn-model.md index 46ff043508..a8baf7ef0f 100644 --- a/src/docs/design/agent-turn-model.md +++ b/src/docs/design/agent-turn-model.md @@ -283,10 +283,10 @@ apply. `tearDownTurnLocked(agent, "restart")` (`src/crux/agent/manager_restart.go:707`), with the comment that `kill-session` destroys the only record of the previous run - (`src/crux/agent/manager_restart.go:703-706`). + (`src/crux/agent/manager_restart.go:565-568`). 3. Reap the CLI process and any UID-owned helpers - (`src/crux/agent/manager_restart.go:724-734`). -4. `tmux kill-session` (`src/crux/agent/manager_restart.go:736`). + (`src/crux/agent/manager_restart.go:586-596`). +4. `tmux kill-session` (`src/crux/agent/manager_restart.go:598`). The replacement CLI starts with **no prompt at all** in the default case. `buildBootstrapPrompt` (`src/crux/agent/manager_env.go:19`) unconditionally returns `""` From 194e17c750187a267f13c7023b759c87208eb448 Mon Sep 17 00:00:00 2001 From: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Thu, 29 Oct 2026 05:08:27 -0400 Subject: [PATCH 4/4] =?UTF-8?q?=F0=9F=93=96=20docs:=20point=20agent-turn-m?= =?UTF-8?q?odel=20restart=20citations=20at=20the=20split=20manager=5Fresta?= =?UTF-8?q?rt.go?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Re-anchor every manager_restart.go line citation in §3 to the head of the split (Restart :434, C-c :574-579, tearDownTurnLocked :585 with its comment :581-584, reap :602-611, kill-session :614, RestartWithBootstrap :286, RestartThenSendKick :383). Verified with check-docs-citations.py. Hive-Fix-Attempt: 1/6 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- src/docs/design/agent-turn-model.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/docs/design/agent-turn-model.md b/src/docs/design/agent-turn-model.md index a8baf7ef0f..65367227b0 100644 --- a/src/docs/design/agent-turn-model.md +++ b/src/docs/design/agent-turn-model.md @@ -275,18 +275,18 @@ apply. ## 3. What a restart actually does to a turn -`Restart` (`src/crux/agent/manager_restart.go:572`) does not resume anything. In order: +`Restart` (`src/crux/agent/manager_restart.go:434`) does not resume anything. In order: 1. `C-c` into the pane and cancel the launch context - (`src/crux/agent/manager_restart.go:696-701`). + (`src/crux/agent/manager_restart.go:574-579`). 2. Archive the scrollback *before* it is destroyed — `tearDownTurnLocked(agent, "restart")` - (`src/crux/agent/manager_restart.go:707`), with the comment that `kill-session` + (`src/crux/agent/manager_restart.go:585`), with the comment that `kill-session` destroys the only record of the previous run - (`src/crux/agent/manager_restart.go:565-568`). + (`src/crux/agent/manager_restart.go:581-584`). 3. Reap the CLI process and any UID-owned helpers - (`src/crux/agent/manager_restart.go:586-596`). -4. `tmux kill-session` (`src/crux/agent/manager_restart.go:598`). + (`src/crux/agent/manager_restart.go:602-611`). +4. `tmux kill-session` (`src/crux/agent/manager_restart.go:614`). The replacement CLI starts with **no prompt at all** in the default case. `buildBootstrapPrompt` (`src/crux/agent/manager_env.go:19`) unconditionally returns `""` @@ -297,9 +297,9 @@ placeholders to the agent (`src/crux/agent/manager_env.go:20-25`). So a restarted agent sits idle at its input prompt until the shared governor ticker (§1.4) next finds it due. Only the explicit override paths supply text -directly: `RestartWithBootstrap` (`src/crux/agent/manager_restart.go:419`) sets +directly: `RestartWithBootstrap` (`src/crux/agent/manager_restart.go:286`) sets `BootstrapOverride`, and `RestartThenSendKick` -(`src/crux/agent/manager_restart.go:505`) is documented as restarting "with a clean +(`src/crux/agent/manager_restart.go:383`) is documented as restarting "with a clean slate". Whichever path runs, delivery is gated on the new pane reaching its input marker (`deliverStartupKick`, `src/crux/agent/manager_kick.go:514`).