Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/fixed-14820-continuous-rekick.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- Continuous agents receive due re-kicks between busy run-loop cycles without waiting for sweeps; busy kick deferrals no longer escalate continuous backoff or send more messages into the same agent lane ([#14820](https://github.com/hivecommons/hive/issues/14820)).
47 changes: 47 additions & 0 deletions src/cmd/hive/continuous_wake_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -211,3 +211,50 @@ func TestRunLoopContinuousWakeBacksOffWhenDueAgentIsSkipped(t *testing.T) {
t.Fatalf("evals = %d after the re-kick moved, want 4", len(h.evals))
}
}

// A long cadence cycle leaves another cadence tick ready and a continuous
// re-kick due. No wake timer is fired: the loop must serve the re-kick first.
func TestRunLoopContinuousWakeServedBetweenBusyTicks(t *testing.T) {
b, log, cancel := newLoopBoot(t, 900, 0)
defer cancel()
b.cfg.Governor.Modes = map[string]config.ModeConfig{"idle": {Cadences: map[string]config.Cadence{"alpha": "continuous"}}}
b.gov = governor.New(b.cfg.Governor, b.cfg.EnabledAgents(), b.logger)
b.gov.Evaluate(0, 0, 0, 0)
clock := time.Now()
b.gov.SeedContinuousNextKickForTest("alpha", clock.Add(time.Minute))
gov := &fakeTicker{ch: make(chan time.Time, 1)}
wake := newFakeTicker()
h := &loopHarness{startupOK: true, pending: []*fakeTicker{gov}, persisted: make(chan struct{}, 8)}
deps := h.deps()
deps.now = func() time.Time { return clock }
deps.newWakeTimer = func(time.Duration) loopTicker { return wake }
eval := deps.runEval
deps.runEval = func(b *boot, restarted []string, source evalCycleSource) {
eval(b, restarted, source)
switch len(h.evals) {
case 2:
clock = clock.Add(10 * time.Minute)
gov.ch <- clock // cadence elapsed again while the cycle was busy
case 3:
// Stop after the competing next cycle, whichever source won.
select {
case <-gov.ch:
default:
}
cancel()
}
}
done := runLoopAsync(b, deps)
<-h.persisted
gov.fire()
waitDone(t, done)
if len(h.sources) != 3 || h.sources[2] != evalSourceCached {
t.Fatalf("sources = %v, want initial/cadence followed by cached re-kick", h.sources)
}
if h.sweeps != 2 || h.rotations != 2 {
t.Fatalf("sweeps=%d rotations=%d: continuous wake must omit maintenance", h.sweeps, h.rotations)
}
if !strings.Contains(log.String(), "continuous re-kick wake: running an early eval cycle") {
t.Fatalf("missing continuous wake: %s", log.String())
}
}
49 changes: 42 additions & 7 deletions src/cmd/hive/eval_cycle_kick_dispatch_7232_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,13 @@ package main

import (
"errors"
"fmt"
"sort"
"sync"
"testing"
"time"

"github.com/hivecommons/hive/crux"
"github.com/hivecommons/hive/pkg/scheduler"
)

Expand Down Expand Up @@ -362,11 +364,8 @@ func TestDispatchAgentKicks_AgentsDeliverConcurrently(t *testing.T) {
}
}

// TestDispatchAgentKicks_SameAgentStaysOrdered pins the other half of the
// contract: two kicks to ONE agent (a fix kick ordered ahead of the governor
// kick by orderReviewFixKicksFirst) must still go out in order, because they
// type into the same pane.
func TestDispatchAgentKicks_SameAgentStaysOrdered(t *testing.T) {
// A successful fix kick starts a turn, so the governor kick must be deferred.
func TestDispatchAgentKicks_SameAgentDefersAfterDelivery(t *testing.T) {
r := newKickDispatchRecorder()
var msgsSeen []string
deps := r.deps()
Expand All @@ -381,10 +380,10 @@ func TestDispatchAgentKicks_SameAgentStaysOrdered(t *testing.T) {
{Agent: "alpha", Message: "governor second"},
}
delivered := dispatchAgentKicks(msgs, false, deps, discardLogger())
if want := []string{"alpha", "alpha"}; !equalStrings(delivered, want) {
if want := []string{"alpha"}; !equalStrings(delivered, want) {
t.Fatalf("delivered = %v, want %v", delivered, want)
}
if want := []string{"fix first", "governor second"}; !equalStrings(msgsSeen, want) {
if want := []string{"fix first"}; !equalStrings(msgsSeen, want) {
t.Fatalf("same-agent kicks reordered: %v, want %v", msgsSeen, want)
}
}
Expand Down Expand Up @@ -461,3 +460,39 @@ func indexOf(hay []string, needle string) int {
}
return -1
}

func TestDispatchAgentKicks_BusyDefersLaneWithoutFailure(t *testing.T) {
for _, busy := range []bool{true, false} {
t.Run(fmt.Sprint(busy), func(t *testing.T) {
r := newKickDispatchRecorder()
if busy {
r.sendErr["alpha"] = fmt.Errorf("prompt wait: %w", crux.ErrAgentBusy)
} else {
r.sendErr["alpha"] = errors.New("CLI stalled")
}
deps := r.deps()
var failures []string
deps.onFailed = func(msg scheduler.KickMessage, err error) { failures = append(failures, msg.Agent) }
delivered := dispatchAgentKicks(kickMsgs("alpha", "alpha", "beta"), true, deps, discardLogger())
if !equalStrings(delivered, []string{"beta"}) || len(r.probeStamp) != 1 {
t.Fatalf("delivered=%v probe stamps=%v", delivered, r.probeStamp)
}
sends := 0
for _, call := range r.calls {
if call == "send:alpha" {
sends++
}
}
if busy {
if len(failures) != 0 || sends != 1 {
t.Fatalf("busy: failures=%v sends=%d", failures, sends)
}
} else if len(failures) != 2 || sends != 2 {
t.Fatalf("genuine failure: failures=%v sends=%d", failures, sends)
}
if len(r.spansEnded) != 3 {
t.Fatalf("unclosed spans: %v", r.spansEnded)
}
})
}
}
19 changes: 17 additions & 2 deletions src/cmd/hive/eval_cycle_seams.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package main

import (
"context"
"errors"
"fmt"
"log/slog"
"os"
Expand Down Expand Up @@ -486,6 +487,7 @@ type kickDispatchDeps struct {
// kickDeliveryOutcome is what one delivery lane reports back for one message:
// which step failed (start or send), if any.
type kickDeliveryOutcome struct {
deferred bool
startErr error
sendErr error
}
Expand All @@ -508,8 +510,9 @@ type kickDeliveryOutcome struct {
// (separate tmux panes; SendKick is already called concurrently by the
// dashboard and the request watchers), so each agent gets its own delivery
// lane and the tick waits only for the SLOWEST agent, not for all of them.
// Messages to the same agent keep their order: a fix kick ordered ahead of the
// governor kick by orderReviewFixKicksFirst still goes first.
// Messages to the same agent keep their priority: a fix kick ordered ahead of
// the governor kick goes first. Once a kick lands or finds a live turn, the
// remaining messages in that lane are deferred until a later cycle.
//
// Only the slow legs — startStoppedAgent and sendKick — run on the lanes. The
// backoff check, the span, and every delivered/failed effect run on the
Expand Down Expand Up @@ -564,12 +567,19 @@ func dispatchAgentKicks(msgs []scheduler.KickMessage, releaseProbe bool, deps ki
}
for _, agentName := range laneOrder {
go func(indexes []int) {
deferLane := false
for _, i := range indexes {
msg := msgs[i]
var out kickDeliveryOutcome
if deferLane {
results[i] <- kickDeliveryOutcome{deferred: true}
continue
}
if out.startErr = deps.startStoppedAgent(msg.Agent); out.startErr == nil {
out.sendErr = deps.sendKick(msg.Agent, msg.Message)
}
// Success or a busy bounce leaves this lane occupied.
deferLane = out.startErr == nil && (out.sendErr == nil || errors.Is(out.sendErr, crux.ErrAgentBusy))
results[i] <- out
}
}(lanes[agentName])
Expand All @@ -583,6 +593,11 @@ func dispatchAgentKicks(msgs []scheduler.KickMessage, releaseProbe bool, deps ki
}
out := <-results[i]
endSpan := endSpans[i]
if out.deferred || errors.Is(out.sendErr, crux.ErrAgentBusy) {
endSpan(nil)
logger.Info("agent busy: deferring kick", "agent", msg.Agent)
continue
}
if out.startErr != nil {
endSpan(out.startErr)
logger.Warn("failed to start stopped agent before kick", "agent", msg.Agent, "error", out.startErr)
Expand Down
50 changes: 32 additions & 18 deletions src/cmd/hive/main.go
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
package main

import (
Expand Down Expand Up @@ -6376,7 +6376,38 @@
}
}()

serveContinuousWake := func() {
if deps.newWakeTimer == nil || b.gov == nil || b.ctx.Err() != nil {
return
}
now := wakeNow()
next, who := b.gov.NextContinuousKickAt()
if _, due := continuousWakeDecision(next, now, lastWake, wakeHold.untilFor(next, who)); due {
idleFor := time.Duration(0)
if st, ok := b.gov.GetState().Continuous[who]; ok && !st.LastKickEnd.IsZero() {
idleFor = now.Sub(st.LastKickEnd).Round(time.Second)
}
b.logger.Info("continuous re-kick wake: running an early eval cycle",
"agent", who, "next_kick", next.UTC().Format(time.RFC3339), "idle_for", idleFor.String(),
"was_armed_for", wakeArmedFor.UTC().Format(time.RFC3339))
lastWake = now
// Keep this cycle lean: planning and delivery without sweeps.
b.cfgReloadMu.Lock()
b.evalCycles++
deps.runEval(b, nil, evalSourceCached)
deps.persist(b)
b.cfgReloadMu.Unlock()
after, afterWho := b.gov.NextContinuousKickAt()
if backoff, stalled := wakeHold.observe(next, after, who, afterWho, wakeNow()); stalled {
b.logger.Info("continuous re-kick wake: eval cycle left the due re-kick unchanged (agent gated outside the governor); backing off",
"agent", who, "next_kick", next.UTC().Format(time.RFC3339), "backoff", backoff.String())
}
}
}

for {
// Serve due kicks between cycles even when other channels stay ready.
serveContinuousWake()
select {
case <-b.ctx.Done():
b.logger.Info("shutting down, persisting state")
Expand Down Expand Up @@ -6434,24 +6465,7 @@
b.cfgReloadMu.Unlock()
armWebhookHealthRecheck()
case <-wakeCh:
now := wakeNow()
next, who := b.gov.NextContinuousKickAt()
if _, due := continuousWakeDecision(next, now, lastWake, wakeHold.untilFor(next, who)); due {
idleFor := time.Duration(0)
if st, ok := b.gov.GetState().Continuous[who]; ok && !st.LastKickEnd.IsZero() {
idleFor = now.Sub(st.LastKickEnd).Round(time.Second)
}
b.logger.Info("continuous re-kick wake: running an early eval cycle",
"agent", who, "next_kick", next.UTC().Format(time.RFC3339), "idle_for", idleFor.String(),
"was_armed_for", wakeArmedFor.UTC().Format(time.RFC3339))
lastWake = now
tick(evalSourceCached)
after, afterWho := b.gov.NextContinuousKickAt()
if backoff, stalled := wakeHold.observe(next, after, who, afterWho, wakeNow()); stalled {
b.logger.Info("continuous re-kick wake: eval cycle left the due re-kick unchanged (agent gated outside the governor); backing off",
"agent", who, "next_kick", next.UTC().Format(time.RFC3339), "backoff", backoff.String())
}
}
// The loop-top due check services this timer.
armContinuousWake()
case <-agentTickCh:
b.cfgReloadMu.Lock()
Expand Down
9 changes: 9 additions & 0 deletions src/crux/agent/busy_visibility.go
Original file line number Diff line number Diff line change
Expand Up @@ -565,3 +565,12 @@ func formatTimeForLog(t time.Time) string {
}
return t.UTC().Format(time.RFC3339)
}

// kickNotReadyErrorLocked distinguishes a live turn from a stalled CLI.
// Caller holds m.mu. Keep the original diagnostic for genuine failures.
func (m *Manager) kickNotReadyErrorLocked(agent *AgentProcess, now time.Time, err error) error {
if _, live := kickUndeliverableTranscriptLiveness(agent, now); live {
return fmt.Errorf("%w: %v", crux.ErrAgentBusy, err)
}
return err
}
29 changes: 29 additions & 0 deletions src/crux/agent/busy_visibility_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -587,3 +587,32 @@ func TestScanTranscriptActivityWalksAgentsConcurrently(t *testing.T) {
}
}
}

func TestKickNotReadyLiveTurnIsBenignDeferral(t *testing.T) {
m := busyVisibilityTestManager()
a := m.agents["scanner"]
now := time.Now()
original := kickPromptNotReadyErr(a.Name, "")
for _, tc := range []struct {
name string
last time.Time
busy bool
}{
{"live", now.Add(-time.Second), true},
{"stalled", now.Add(-config.DefaultBusyNoActivityThreshold - time.Minute), false},
{"unknown", time.Time{}, false},
} {
t.Run(tc.name, func(t *testing.T) {
a.LastTranscriptActivity = tc.last
m.mu.Lock()
err := m.kickNotReadyErrorLocked(a, now, original)
m.mu.Unlock()
if errors.Is(err, crux.ErrAgentBusy) != tc.busy {
t.Fatalf("error=%v, want busy=%v", err, tc.busy)
}
if !tc.busy && err != original {
t.Fatalf("lost genuine failure: %v", err)
}
})
}
}
28 changes: 28 additions & 0 deletions src/crux/agent/kick_delivery_failure_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -205,3 +205,31 @@ func TestKickCancelledMidSendIsNotDelivered(t *testing.T) {
t.Fatal("cancelled delivery recorded success, blamed the new session, or retained the pane claim")
}
}

// A turn can become busy after the prompt wait but before the final locked
// delivery check. The caller must see a deferral and no successful kick.
func TestSendKickBusyBounceIsDeferral(t *testing.T) {
origExists := tmuxSessionExists
tmuxSessionExists = func(*Manager, *AgentProcess) bool { return true }
t.Cleanup(func() { tmuxSessionExists = origExists })
m, a, _ := kickLogTestManager(t, "")
a.LastTranscriptActivity = time.Now()
captures := 0
termSeams(m).captureVisiblePane = func(*AgentProcess) string {
captures++
if captures >= 3 {
return "shell$"
}
return "❯"
}
termSeams(m).sendLiteralErr = func(*AgentProcess, string) error {
t.Fatal("typed into an unready pane")
return nil
}
if err := m.SendKick(a.Name, "task"); !errors.Is(err, crux.ErrAgentBusy) {
t.Fatalf("busy bounce = %v, want ErrAgentBusy", err)
}
if a.LastKick != nil || len(a.KickHistory) != 0 {
t.Fatal("busy bounce recorded a delivered kick")
}
}
6 changes: 3 additions & 3 deletions src/crux/agent/manager_kick.go
Original file line number Diff line number Diff line change
Expand Up @@ -193,7 +193,7 @@ func (m *Manager) SendKick(name string, message string) (err error) {
return fmt.Errorf("%w: agent %s restarted while the kick was waiting for its input prompt", errKickCancelledByRestart, name)
}
m.markKickUndeliverableLocked(agent, time.Now(), reason)
return kickPromptNotReadyErr(name, reason)
return m.kickNotReadyErrorLocked(agent, time.Now(), kickPromptNotReadyErr(name, reason))
}
m.mu.Lock()
agent, ok = m.agents[name]
Expand Down Expand Up @@ -268,7 +268,7 @@ func (m *Manager) deliverKickLocked(agent *AgentProcess, message, trigger string
if !agent.kickDelivering.CompareAndSwap(false, true) {
m.logger.Warn("kick delivery skipped: another kick is already being typed into this pane",
"agent", agent.Name, "trigger", trigger)
return fmt.Errorf("agent %s kick delivery already in progress", agent.Name)
return fmt.Errorf("%w: agent %s kick delivery already in progress", crux.ErrAgentBusy, agent.Name)
}
defer agent.kickDelivering.Store(false)

Expand All @@ -287,7 +287,7 @@ func (m *Manager) deliverKickLocked(agent *AgentProcess, message, trigger string
if menuFooter != "" {
return fmt.Errorf("agent %s pane not at a ready input prompt: %s", agent.Name, reason)
}
return fmt.Errorf("agent %s pane not at a ready input prompt", agent.Name)
return m.kickNotReadyErrorLocked(agent, time.Now(), fmt.Errorf("agent %s pane not at a ready input prompt", agent.Name))
}

// Archive the PREVIOUS kick's scrollback and clear the history before any
Expand Down
9 changes: 8 additions & 1 deletion src/crux/kick.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,13 @@
package crux

import "time"
import (
"errors"
"time"
)

// ErrAgentBusy means no kick was delivered because an agent is mid-turn.
// Callers should defer the kick without recording a delivery failure.
var ErrAgentBusy = errors.New("agent turn in progress")

// Kick dispatch phases. A dispatch is INDETERMINATE while it is pending: the
// prompt may still be delivered. Only KickPhaseFailed is a definitive failure.
Expand Down
Loading