From 256ef585f7e5f94f36ce6ebda38285ef511be7e2 Mon Sep 17 00:00:00 2001 From: thatguywob Date: Wed, 28 Oct 2026 04:01:09 +0000 Subject: [PATCH] =?UTF-8?q?=F0=9F=90=9B=20fix:=20serve=20continuous=20re-k?= =?UTF-8?q?icks=20between=20busy=20cycles?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Check due continuous re-kicks between loop iterations and run a cached eval without maintenance sweeps. Defer remaining same-agent kicks after delivery or a live-turn bounce, without recording busy deferrals as governor kick failures. Co-authored-by: clubanderson <407614+clubanderson@users.noreply.github.com> Hive-Run: hivecommons/hive#14820 Hive-Plan: continuous-rekick-wake-and-busy-deferral Hive-Spec: issue-14820#expected Signed-off-by: thatguywob --- changelog.d/fixed-14820-continuous-rekick.md | 1 + src/cmd/hive/continuous_wake_test.go | 47 +++++++++++++++++ .../eval_cycle_kick_dispatch_7232_test.go | 49 +++++++++++++++--- src/cmd/hive/eval_cycle_seams.go | 19 ++++++- src/cmd/hive/main.go | 50 ++++++++++++------- src/crux/agent/busy_visibility.go | 9 ++++ src/crux/agent/busy_visibility_test.go | 29 +++++++++++ src/crux/agent/kick_delivery_failure_test.go | 28 +++++++++++ src/crux/agent/manager_kick.go | 6 +-- src/crux/kick.go | 9 +++- 10 files changed, 216 insertions(+), 31 deletions(-) create mode 100644 changelog.d/fixed-14820-continuous-rekick.md diff --git a/changelog.d/fixed-14820-continuous-rekick.md b/changelog.d/fixed-14820-continuous-rekick.md new file mode 100644 index 0000000000..9cd55b5081 --- /dev/null +++ b/changelog.d/fixed-14820-continuous-rekick.md @@ -0,0 +1 @@ +- Continuous agents receive due re-kicks between busy run-loop cycles without waiting for sweeps; busy kick deferrals no longer escalate continuous backoff or send more messages into the same agent lane ([#14820](https://github.com/hivecommons/hive/issues/14820)). diff --git a/src/cmd/hive/continuous_wake_test.go b/src/cmd/hive/continuous_wake_test.go index ab2ff5531a..b856717be8 100644 --- a/src/cmd/hive/continuous_wake_test.go +++ b/src/cmd/hive/continuous_wake_test.go @@ -211,3 +211,50 @@ func TestRunLoopContinuousWakeBacksOffWhenDueAgentIsSkipped(t *testing.T) { t.Fatalf("evals = %d after the re-kick moved, want 4", len(h.evals)) } } + +// A long cadence cycle leaves another cadence tick ready and a continuous +// re-kick due. No wake timer is fired: the loop must serve the re-kick first. +func TestRunLoopContinuousWakeServedBetweenBusyTicks(t *testing.T) { + b, log, cancel := newLoopBoot(t, 900, 0) + defer cancel() + b.cfg.Governor.Modes = map[string]config.ModeConfig{"idle": {Cadences: map[string]config.Cadence{"alpha": "continuous"}}} + b.gov = governor.New(b.cfg.Governor, b.cfg.EnabledAgents(), b.logger) + b.gov.Evaluate(0, 0, 0, 0) + clock := time.Now() + b.gov.SeedContinuousNextKickForTest("alpha", clock.Add(time.Minute)) + gov := &fakeTicker{ch: make(chan time.Time, 1)} + wake := newFakeTicker() + h := &loopHarness{startupOK: true, pending: []*fakeTicker{gov}, persisted: make(chan struct{}, 8)} + deps := h.deps() + deps.now = func() time.Time { return clock } + deps.newWakeTimer = func(time.Duration) loopTicker { return wake } + eval := deps.runEval + deps.runEval = func(b *boot, restarted []string, source evalCycleSource) { + eval(b, restarted, source) + switch len(h.evals) { + case 2: + clock = clock.Add(10 * time.Minute) + gov.ch <- clock // cadence elapsed again while the cycle was busy + case 3: + // Stop after the competing next cycle, whichever source won. + select { + case <-gov.ch: + default: + } + cancel() + } + } + done := runLoopAsync(b, deps) + <-h.persisted + gov.fire() + waitDone(t, done) + if len(h.sources) != 3 || h.sources[2] != evalSourceCached { + t.Fatalf("sources = %v, want initial/cadence followed by cached re-kick", h.sources) + } + if h.sweeps != 2 || h.rotations != 2 { + t.Fatalf("sweeps=%d rotations=%d: continuous wake must omit maintenance", h.sweeps, h.rotations) + } + if !strings.Contains(log.String(), "continuous re-kick wake: running an early eval cycle") { + t.Fatalf("missing continuous wake: %s", log.String()) + } +} diff --git a/src/cmd/hive/eval_cycle_kick_dispatch_7232_test.go b/src/cmd/hive/eval_cycle_kick_dispatch_7232_test.go index 8db62ea5f1..77d5e15432 100644 --- a/src/cmd/hive/eval_cycle_kick_dispatch_7232_test.go +++ b/src/cmd/hive/eval_cycle_kick_dispatch_7232_test.go @@ -2,11 +2,13 @@ package main import ( "errors" + "fmt" "sort" "sync" "testing" "time" + "github.com/hivecommons/hive/crux" "github.com/hivecommons/hive/pkg/scheduler" ) @@ -362,11 +364,8 @@ func TestDispatchAgentKicks_AgentsDeliverConcurrently(t *testing.T) { } } -// TestDispatchAgentKicks_SameAgentStaysOrdered pins the other half of the -// contract: two kicks to ONE agent (a fix kick ordered ahead of the governor -// kick by orderReviewFixKicksFirst) must still go out in order, because they -// type into the same pane. -func TestDispatchAgentKicks_SameAgentStaysOrdered(t *testing.T) { +// A successful fix kick starts a turn, so the governor kick must be deferred. +func TestDispatchAgentKicks_SameAgentDefersAfterDelivery(t *testing.T) { r := newKickDispatchRecorder() var msgsSeen []string deps := r.deps() @@ -381,10 +380,10 @@ func TestDispatchAgentKicks_SameAgentStaysOrdered(t *testing.T) { {Agent: "alpha", Message: "governor second"}, } delivered := dispatchAgentKicks(msgs, false, deps, discardLogger()) - if want := []string{"alpha", "alpha"}; !equalStrings(delivered, want) { + if want := []string{"alpha"}; !equalStrings(delivered, want) { t.Fatalf("delivered = %v, want %v", delivered, want) } - if want := []string{"fix first", "governor second"}; !equalStrings(msgsSeen, want) { + if want := []string{"fix first"}; !equalStrings(msgsSeen, want) { t.Fatalf("same-agent kicks reordered: %v, want %v", msgsSeen, want) } } @@ -461,3 +460,39 @@ func indexOf(hay []string, needle string) int { } return -1 } + +func TestDispatchAgentKicks_BusyDefersLaneWithoutFailure(t *testing.T) { + for _, busy := range []bool{true, false} { + t.Run(fmt.Sprint(busy), func(t *testing.T) { + r := newKickDispatchRecorder() + if busy { + r.sendErr["alpha"] = fmt.Errorf("prompt wait: %w", crux.ErrAgentBusy) + } else { + r.sendErr["alpha"] = errors.New("CLI stalled") + } + deps := r.deps() + var failures []string + deps.onFailed = func(msg scheduler.KickMessage, err error) { failures = append(failures, msg.Agent) } + delivered := dispatchAgentKicks(kickMsgs("alpha", "alpha", "beta"), true, deps, discardLogger()) + if !equalStrings(delivered, []string{"beta"}) || len(r.probeStamp) != 1 { + t.Fatalf("delivered=%v probe stamps=%v", delivered, r.probeStamp) + } + sends := 0 + for _, call := range r.calls { + if call == "send:alpha" { + sends++ + } + } + if busy { + if len(failures) != 0 || sends != 1 { + t.Fatalf("busy: failures=%v sends=%d", failures, sends) + } + } else if len(failures) != 2 || sends != 2 { + t.Fatalf("genuine failure: failures=%v sends=%d", failures, sends) + } + if len(r.spansEnded) != 3 { + t.Fatalf("unclosed spans: %v", r.spansEnded) + } + }) + } +} diff --git a/src/cmd/hive/eval_cycle_seams.go b/src/cmd/hive/eval_cycle_seams.go index d5406916d9..03ffc9936f 100644 --- a/src/cmd/hive/eval_cycle_seams.go +++ b/src/cmd/hive/eval_cycle_seams.go @@ -2,6 +2,7 @@ package main import ( "context" + "errors" "fmt" "log/slog" "os" @@ -486,6 +487,7 @@ type kickDispatchDeps struct { // kickDeliveryOutcome is what one delivery lane reports back for one message: // which step failed (start or send), if any. type kickDeliveryOutcome struct { + deferred bool startErr error sendErr error } @@ -508,8 +510,9 @@ type kickDeliveryOutcome struct { // (separate tmux panes; SendKick is already called concurrently by the // dashboard and the request watchers), so each agent gets its own delivery // lane and the tick waits only for the SLOWEST agent, not for all of them. -// Messages to the same agent keep their order: a fix kick ordered ahead of the -// governor kick by orderReviewFixKicksFirst still goes first. +// Messages to the same agent keep their priority: a fix kick ordered ahead of +// the governor kick goes first. Once a kick lands or finds a live turn, the +// remaining messages in that lane are deferred until a later cycle. // // Only the slow legs — startStoppedAgent and sendKick — run on the lanes. The // backoff check, the span, and every delivered/failed effect run on the @@ -564,12 +567,19 @@ func dispatchAgentKicks(msgs []scheduler.KickMessage, releaseProbe bool, deps ki } for _, agentName := range laneOrder { go func(indexes []int) { + deferLane := false for _, i := range indexes { msg := msgs[i] var out kickDeliveryOutcome + if deferLane { + results[i] <- kickDeliveryOutcome{deferred: true} + continue + } if out.startErr = deps.startStoppedAgent(msg.Agent); out.startErr == nil { out.sendErr = deps.sendKick(msg.Agent, msg.Message) } + // Success or a busy bounce leaves this lane occupied. + deferLane = out.startErr == nil && (out.sendErr == nil || errors.Is(out.sendErr, crux.ErrAgentBusy)) results[i] <- out } }(lanes[agentName]) @@ -583,6 +593,11 @@ func dispatchAgentKicks(msgs []scheduler.KickMessage, releaseProbe bool, deps ki } out := <-results[i] endSpan := endSpans[i] + if out.deferred || errors.Is(out.sendErr, crux.ErrAgentBusy) { + endSpan(nil) + logger.Info("agent busy: deferring kick", "agent", msg.Agent) + continue + } if out.startErr != nil { endSpan(out.startErr) logger.Warn("failed to start stopped agent before kick", "agent", msg.Agent, "error", out.startErr) diff --git a/src/cmd/hive/main.go b/src/cmd/hive/main.go index d858649305..59425536ed 100644 --- a/src/cmd/hive/main.go +++ b/src/cmd/hive/main.go @@ -6376,7 +6376,38 @@ func (b *boot) runLoopWith(deps runLoopDeps) { } }() + serveContinuousWake := func() { + if deps.newWakeTimer == nil || b.gov == nil || b.ctx.Err() != nil { + return + } + now := wakeNow() + next, who := b.gov.NextContinuousKickAt() + if _, due := continuousWakeDecision(next, now, lastWake, wakeHold.untilFor(next, who)); due { + idleFor := time.Duration(0) + if st, ok := b.gov.GetState().Continuous[who]; ok && !st.LastKickEnd.IsZero() { + idleFor = now.Sub(st.LastKickEnd).Round(time.Second) + } + b.logger.Info("continuous re-kick wake: running an early eval cycle", + "agent", who, "next_kick", next.UTC().Format(time.RFC3339), "idle_for", idleFor.String(), + "was_armed_for", wakeArmedFor.UTC().Format(time.RFC3339)) + lastWake = now + // Keep this cycle lean: planning and delivery without sweeps. + b.cfgReloadMu.Lock() + b.evalCycles++ + deps.runEval(b, nil, evalSourceCached) + deps.persist(b) + b.cfgReloadMu.Unlock() + after, afterWho := b.gov.NextContinuousKickAt() + if backoff, stalled := wakeHold.observe(next, after, who, afterWho, wakeNow()); stalled { + b.logger.Info("continuous re-kick wake: eval cycle left the due re-kick unchanged (agent gated outside the governor); backing off", + "agent", who, "next_kick", next.UTC().Format(time.RFC3339), "backoff", backoff.String()) + } + } + } + for { + // Serve due kicks between cycles even when other channels stay ready. + serveContinuousWake() select { case <-b.ctx.Done(): b.logger.Info("shutting down, persisting state") @@ -6434,24 +6465,7 @@ func (b *boot) runLoopWith(deps runLoopDeps) { b.cfgReloadMu.Unlock() armWebhookHealthRecheck() case <-wakeCh: - now := wakeNow() - next, who := b.gov.NextContinuousKickAt() - if _, due := continuousWakeDecision(next, now, lastWake, wakeHold.untilFor(next, who)); due { - idleFor := time.Duration(0) - if st, ok := b.gov.GetState().Continuous[who]; ok && !st.LastKickEnd.IsZero() { - idleFor = now.Sub(st.LastKickEnd).Round(time.Second) - } - b.logger.Info("continuous re-kick wake: running an early eval cycle", - "agent", who, "next_kick", next.UTC().Format(time.RFC3339), "idle_for", idleFor.String(), - "was_armed_for", wakeArmedFor.UTC().Format(time.RFC3339)) - lastWake = now - tick(evalSourceCached) - after, afterWho := b.gov.NextContinuousKickAt() - if backoff, stalled := wakeHold.observe(next, after, who, afterWho, wakeNow()); stalled { - b.logger.Info("continuous re-kick wake: eval cycle left the due re-kick unchanged (agent gated outside the governor); backing off", - "agent", who, "next_kick", next.UTC().Format(time.RFC3339), "backoff", backoff.String()) - } - } + // The loop-top due check services this timer. armContinuousWake() case <-agentTickCh: b.cfgReloadMu.Lock() diff --git a/src/crux/agent/busy_visibility.go b/src/crux/agent/busy_visibility.go index 299319e1e8..3be89cf659 100644 --- a/src/crux/agent/busy_visibility.go +++ b/src/crux/agent/busy_visibility.go @@ -565,3 +565,12 @@ func formatTimeForLog(t time.Time) string { } return t.UTC().Format(time.RFC3339) } + +// kickNotReadyErrorLocked distinguishes a live turn from a stalled CLI. +// Caller holds m.mu. Keep the original diagnostic for genuine failures. +func (m *Manager) kickNotReadyErrorLocked(agent *AgentProcess, now time.Time, err error) error { + if _, live := kickUndeliverableTranscriptLiveness(agent, now); live { + return fmt.Errorf("%w: %v", crux.ErrAgentBusy, err) + } + return err +} diff --git a/src/crux/agent/busy_visibility_test.go b/src/crux/agent/busy_visibility_test.go index c104f86b41..cc4b673e2f 100644 --- a/src/crux/agent/busy_visibility_test.go +++ b/src/crux/agent/busy_visibility_test.go @@ -587,3 +587,32 @@ func TestScanTranscriptActivityWalksAgentsConcurrently(t *testing.T) { } } } + +func TestKickNotReadyLiveTurnIsBenignDeferral(t *testing.T) { + m := busyVisibilityTestManager() + a := m.agents["scanner"] + now := time.Now() + original := kickPromptNotReadyErr(a.Name, "") + for _, tc := range []struct { + name string + last time.Time + busy bool + }{ + {"live", now.Add(-time.Second), true}, + {"stalled", now.Add(-config.DefaultBusyNoActivityThreshold - time.Minute), false}, + {"unknown", time.Time{}, false}, + } { + t.Run(tc.name, func(t *testing.T) { + a.LastTranscriptActivity = tc.last + m.mu.Lock() + err := m.kickNotReadyErrorLocked(a, now, original) + m.mu.Unlock() + if errors.Is(err, crux.ErrAgentBusy) != tc.busy { + t.Fatalf("error=%v, want busy=%v", err, tc.busy) + } + if !tc.busy && err != original { + t.Fatalf("lost genuine failure: %v", err) + } + }) + } +} diff --git a/src/crux/agent/kick_delivery_failure_test.go b/src/crux/agent/kick_delivery_failure_test.go index f6cd3aa112..0faba591cf 100644 --- a/src/crux/agent/kick_delivery_failure_test.go +++ b/src/crux/agent/kick_delivery_failure_test.go @@ -205,3 +205,31 @@ func TestKickCancelledMidSendIsNotDelivered(t *testing.T) { t.Fatal("cancelled delivery recorded success, blamed the new session, or retained the pane claim") } } + +// A turn can become busy after the prompt wait but before the final locked +// delivery check. The caller must see a deferral and no successful kick. +func TestSendKickBusyBounceIsDeferral(t *testing.T) { + origExists := tmuxSessionExists + tmuxSessionExists = func(*Manager, *AgentProcess) bool { return true } + t.Cleanup(func() { tmuxSessionExists = origExists }) + m, a, _ := kickLogTestManager(t, "") + a.LastTranscriptActivity = time.Now() + captures := 0 + termSeams(m).captureVisiblePane = func(*AgentProcess) string { + captures++ + if captures >= 3 { + return "shell$" + } + return "❯" + } + termSeams(m).sendLiteralErr = func(*AgentProcess, string) error { + t.Fatal("typed into an unready pane") + return nil + } + if err := m.SendKick(a.Name, "task"); !errors.Is(err, crux.ErrAgentBusy) { + t.Fatalf("busy bounce = %v, want ErrAgentBusy", err) + } + if a.LastKick != nil || len(a.KickHistory) != 0 { + t.Fatal("busy bounce recorded a delivered kick") + } +} diff --git a/src/crux/agent/manager_kick.go b/src/crux/agent/manager_kick.go index 909a9339dd..122f0a7c2d 100644 --- a/src/crux/agent/manager_kick.go +++ b/src/crux/agent/manager_kick.go @@ -193,7 +193,7 @@ func (m *Manager) SendKick(name string, message string) (err error) { return fmt.Errorf("%w: agent %s restarted while the kick was waiting for its input prompt", errKickCancelledByRestart, name) } m.markKickUndeliverableLocked(agent, time.Now(), reason) - return kickPromptNotReadyErr(name, reason) + return m.kickNotReadyErrorLocked(agent, time.Now(), kickPromptNotReadyErr(name, reason)) } m.mu.Lock() agent, ok = m.agents[name] @@ -268,7 +268,7 @@ func (m *Manager) deliverKickLocked(agent *AgentProcess, message, trigger string if !agent.kickDelivering.CompareAndSwap(false, true) { m.logger.Warn("kick delivery skipped: another kick is already being typed into this pane", "agent", agent.Name, "trigger", trigger) - return fmt.Errorf("agent %s kick delivery already in progress", agent.Name) + return fmt.Errorf("%w: agent %s kick delivery already in progress", crux.ErrAgentBusy, agent.Name) } defer agent.kickDelivering.Store(false) @@ -287,7 +287,7 @@ func (m *Manager) deliverKickLocked(agent *AgentProcess, message, trigger string if menuFooter != "" { return fmt.Errorf("agent %s pane not at a ready input prompt: %s", agent.Name, reason) } - return fmt.Errorf("agent %s pane not at a ready input prompt", agent.Name) + return m.kickNotReadyErrorLocked(agent, time.Now(), fmt.Errorf("agent %s pane not at a ready input prompt", agent.Name)) } // Archive the PREVIOUS kick's scrollback and clear the history before any diff --git a/src/crux/kick.go b/src/crux/kick.go index 4c96de634a..7863db5999 100644 --- a/src/crux/kick.go +++ b/src/crux/kick.go @@ -1,6 +1,13 @@ package crux -import "time" +import ( + "errors" + "time" +) + +// ErrAgentBusy means no kick was delivered because an agent is mid-turn. +// Callers should defer the kick without recording a delivery failure. +var ErrAgentBusy = errors.New("agent turn in progress") // Kick dispatch phases. A dispatch is INDETERMINATE while it is pending: the // prompt may still be delivered. Only KickPhaseFailed is a definitive failure.