Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/fixed-16536-fixer-fallback.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- Add opt-in ordered fixer fallback agents, configurable in Settings, so review fixes and stale red-PR repairs can bypass an undeliverable primary without interrupting its turn ([#16536](https://github.com/hivecommons/hive/issues/16536)).
196 changes: 196 additions & 0 deletions src/cmd/hive/fixer_fallback_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,196 @@
package main

import (
"bytes"
"fmt"
"log/slog"
"strings"
"testing"
"time"

"github.com/hivecommons/hive/crux"
"github.com/hivecommons/hive/pkg/config"
"github.com/hivecommons/hive/pkg/escalation"
"github.com/hivecommons/hive/pkg/review"
)

// Only status reads are implemented: fallback selection must never interrupt
// or restart the busy primary, even when its transcript has gone silent.
type fixerFallbackRuntime struct {
crux.Runtime
statuses map[string]*crux.Process
}

func (f *fixerFallbackRuntime) IsPaused(name string) bool {
return f.statuses[name] != nil && f.statuses[name].Paused
}

func (f *fixerFallbackRuntime) GetStatusFast(name string) (*crux.Process, error) {
if st := f.statuses[name]; st != nil {
return st, nil
}
return nil, fmt.Errorf("unknown agent %s", name)
}

func TestFixerUndeliverable(t *testing.T) {
now := time.Now()
for _, tc := range []struct {
name string
fallbacks []string
threshold int
kicks int
activity time.Time
want bool
}{
{"unset", nil, 0, 10, now.Add(-time.Hour), false},
{"threshold alone", nil, 1, 10, now.Add(-time.Hour), false},
{"blank list", []string{" "}, 0, 10, now.Add(-time.Hour), false},
{"below default", []string{"fixer"}, 0, 2, now, false},
{"at default", []string{"fixer"}, 0, 3, now, true},
{"below override", []string{"fixer"}, 5, 4, now, false},
{"at override", []string{"fixer"}, 5, 5, now, true},
{"silent busy", []string{"fixer"}, 0, 1, now.Add(-time.Hour), true},
{"idle silent", []string{"fixer"}, 0, 0, now.Add(-time.Hour), false},
{"unknown transcript", []string{"fixer"}, 0, 1, time.Time{}, false},
{"future transcript", []string{"fixer"}, 0, 1, now.Add(time.Minute), false},
} {
t.Run(tc.name, func(t *testing.T) {
cfg := config.ReviewConfig{FixerFallbackAgents: tc.fallbacks, FixerFallbackAfterUndeliverable: tc.threshold}
st := &crux.Process{KicksUndeliverable: tc.kicks, LastTranscriptActivity: tc.activity}
st.Config.BusyNoActivityThreshold = time.Hour
if got := fixerUndeliverable(cfg, st, now); got != tc.want {
t.Fatalf("fixerUndeliverable = %v, want %v", got, tc.want)
}
})
}
}

func fallbackTestFleet() (*config.Config, *fixerFallbackRuntime) {
cfg := reviewSwarmConfig()
cfg.Review.FixerFallbackAgents = []string{" paused ", "disabled", "on-demand", "non-kick", "off-cadence", "unknown", "busy", "backoff", "fixer", "spare"}
mgr := &fixerFallbackRuntime{statuses: map[string]*crux.Process{
"scanner": {KicksUndeliverable: 3},
"paused": {Paused: true},
"busy": {KicksUndeliverable: 1},
"backoff": {ProviderErrorBackoffUntil: time.Now().Add(time.Hour)},
"fixer": {},
"spare": {},
}}
for _, name := range []string{"paused", "on-demand", "non-kick", "off-cadence", "unknown", "busy", "backoff", "fixer", "spare"} {
cfg.Agents[name] = config.AgentConfig{Enabled: true}
}
ac := cfg.Agents["on-demand"]
ac.OnDemand = true
cfg.Agents["on-demand"] = ac
kickEnabled := false
cfg.Agents["non-kick"] = config.AgentConfig{Enabled: true, Channels: []config.ChannelConfig{{Type: "kick", Enabled: &kickEnabled}}}
cfg.Governor.Modes = map[string]config.ModeConfig{
"idle": {Cadences: map[string]config.Cadence{"off-cadence": "paused"}},
}
return cfg, mgr
}

func TestReEngagementFallsBackAndReturnsToRecoveredPrimary(t *testing.T) {
store, clock := newTestEscalationStore(t)
cfg, mgr := fallbackTestFleet()
o := escalation.Observation{Repo: "acme/widgets", Number: 7, Agent: "scanner"}
kicker := &fakeKicker{}
var logs bytes.Buffer
logger := slog.New(slog.NewTextHandler(&logs, nil))

if !deliverReEngagement(cfg, store, o, "abc", kicker, agentAvailability(cfg, mgr), logger) {
t.Fatal("repair was not delivered around the busy primary")
}
if len(kicker.kicks) != 1 || kicker.kicks[0].agent != "fixer" || !strings.Contains(kicker.kicks[0].message, "FIX-BEFORE-NEW") {
t.Fatalf("expected first available fallback to receive repair, got %+v", kicker.kicks)
}
if !strings.Contains(logs.String(), "routed to fallback fixer") || !strings.Contains(logs.String(), "agent=fixer") {
t.Fatalf("missing routing diagnostic: %s", logs.String())
}

*clock = clock.Add(escalation.MaxReEngageCooldown + time.Minute)
mgr.statuses["scanner"].KicksUndeliverable = 0
if !deliverReEngagement(cfg, store, o, "abc", kicker, agentAvailability(cfg, mgr), logger) || kicker.kicks[1].agent != "scanner" {
t.Fatalf("recovered primary did not regain repair routing: %+v", kicker.kicks)
}

*clock = clock.Add(escalation.MaxReEngageCooldown + time.Minute)
mgr.statuses["scanner"].KicksUndeliverable = 3
mgr.statuses["fixer"].KicksUndeliverable = 1
mgr.statuses["spare"].Paused = true
if deliverReEngagement(cfg, store, o, "abc", kicker, agentAvailability(cfg, mgr), logger) {
t.Fatal("repair reported delivered while all fixers were unavailable")
}
if len(kicker.kicks) != 2 || store.ReEngagements(o.Repo, o.Number) != 2 {
t.Fatal("undeliverable repair spent budget or sent a kick")
}
}

func TestReviewFixerFallbackWiring(t *testing.T) {
for _, tc := range []struct {
name string
kicks int
fallbacks bool
want string
}{
{"unset preserves busy primary", 10, false, "scanner"},
{"below threshold", 2, true, "scanner"},
{"fallback", 3, true, "fixer"},
{"recovered primary", 0, true, "scanner"},
} {
t.Run(tc.name, func(t *testing.T) {
redirectReviewPaths(t)
if err := review.WriteArtifact("", review.Artifact{Items: []review.Aggregate{{
Repo: "hivecommons/hive", Number: 4321, HeadSHA: reviewTestSHA, Verdict: review.VerdictChangesRequested,
}}}); err != nil {
t.Fatal(err)
}
cfg, mgr := fallbackTestFleet()
if !tc.fallbacks {
cfg.Review.FixerFallbackAgents = nil
}
mgr.statuses["scanner"].KicksUndeliverable = tc.kicks
plan := planReviewDispatch(cfg, actionableWithPR("hive-bot"), mgr, nil, discardLogger())
if len(plan.FixKicks) != 1 || plan.FixKicks[0].Agent != tc.want {
t.Fatalf("fix kicks = %+v, want one for %s", plan.FixKicks, tc.want)
}
if len(plan.State.Fixes) != 1 || plan.State.Fixes[0].Agent != tc.want || plan.State.Fixes[0].Attempts != 1 {
t.Fatalf("fallback not reflected in dispatch accounting: %+v", plan.State.Fixes)
}
if tc.want == "fixer" {
// Recovery must not duplicate work already delivered to the
// fallback. It only changes routing for the next eligible fix.
state := review.ConfirmDelivered(plan.State, plan.FixKicks, plan.FixKicks)
if err := review.WriteDispatchState("", state); err != nil {
t.Fatal(err)
}
mgr.statuses["scanner"].KicksUndeliverable = 0
again := planReviewDispatch(cfg, actionableWithPR("hive-bot"), mgr, nil, discardLogger())
if len(again.FixKicks) != 0 || len(again.State.Fixes) != 1 || again.State.Fixes[0].Agent != "fixer" {
t.Fatalf("recovery duplicated the pending fallback fix: %+v", again)
}
}
})
}
}

func TestReviewFixerFallbackAllBusyRetriesWithoutHumanHold(t *testing.T) {
redirectReviewPaths(t)
if err := review.WriteArtifact("", review.Artifact{Items: []review.Aggregate{{
Repo: "hivecommons/hive", Number: 4321, HeadSHA: reviewTestSHA, Verdict: review.VerdictChangesRequested,
}}}); err != nil {
t.Fatal(err)
}
cfg, mgr := fallbackTestFleet()
mgr.statuses["fixer"].KicksUndeliverable = 1
mgr.statuses["spare"].Paused = true
plan := planReviewDispatch(cfg, actionableWithPR("hive-bot"), mgr, nil, discardLogger())
if len(plan.FixKicks) != 0 || len(plan.State.Fixes) != 0 || len(plan.State.Human) != 0 {
t.Fatalf("busy fleet sent a repair or created a human hold: %+v", plan)
}
mgr.statuses["scanner"].KicksUndeliverable = 0
plan = planReviewDispatch(cfg, actionableWithPR("hive-bot"), mgr, nil, discardLogger())
if len(plan.FixKicks) != 1 || plan.FixKicks[0].Agent != "scanner" {
t.Fatalf("repair did not retry after recovery: %+v", plan.FixKicks)
}
}
62 changes: 60 additions & 2 deletions src/cmd/hive/main_sweeps.go
Original file line number Diff line number Diff line change
Expand Up @@ -231,10 +231,54 @@ func agentAvailability(cfg *config.Config, mgr crux.Runtime) func(string) bool {
if mgr != nil && mgr.IsPaused(name) {
return false
}
return cfg == nil || cadenceReachable(cfg, name)
if cfg != nil && !cadenceReachable(cfg, name) {
return false
}
// Keep the old reachability contract unless fallback routing is opted
// into. In particular, a busy primary still receives kick attempts when
// the list is unset, even if a threshold was configured on its own.
if cfg == nil || cfg.Review.EffectiveFixerFallbackAfterUndeliverable() == 0 {
return true
}
ac, ok := cfg.Agents[name]
if !ok || !ac.Enabled || ac.Paused || ac.OnDemand || !ac.UsesGovernorKick() {
return false
}
if mgr == nil {
return true
}
st, err := mgr.GetStatusFast(name)
if err != nil || st == nil || st.ProviderErrorBackoffUntil.After(time.Now()) {
return false
}
// A fallback already refusing input cannot pick up another repair.
if name != redPRFixOwner(cfg.Review.FixerAgent) && slices.Contains(cfg.Review.EffectiveFixerFallbackAgents(), name) && st.KicksUndeliverable > 0 {
return false
}
return !fixerUndeliverable(cfg.Review, st, time.Now())
}
}

// fixerUndeliverable only reads the busy episode; it never interrupts the
// agent's turn. Old transcript activity alone must not disqualify an idle
// agent, and a missing transcript is unknown rather than proof of silence.
func fixerUndeliverable(cfg config.ReviewConfig, st *crux.Process, now time.Time) bool {
threshold := cfg.EffectiveFixerFallbackAfterUndeliverable()
if threshold == 0 || st == nil || st.KicksUndeliverable == 0 {
return false
}
return st.KicksUndeliverable >= threshold ||
(!st.LastTranscriptActivity.IsZero() && now.Sub(st.LastTranscriptActivity) >= st.Config.EffectiveBusyNoActivityThreshold())
}

// reviewFixerAvailability adds runtime liveness only for opted-in hives.
func reviewFixerAvailability(cfg *config.Config, mgr crux.Runtime) func(string) bool {
if cfg.Review.EffectiveFixerFallbackAfterUndeliverable() == 0 {
return nil
}
return agentAvailability(cfg, mgr)
}

// cadenceReachable reports whether ANY governor mode gives the agent a cadence
// that is not paused/off. An agent with no cadence entry at all is treated as
// reachable: "nothing schedules it" is a fleet-configuration signal of its own
Expand Down Expand Up @@ -280,11 +324,13 @@ func ownerReEngageCooldown(cfg *config.Config, name string) time.Duration {

// resolveRedPRFixer picks the agent a stuck red PR's fix kick should go to: its
// owner, else the configured review fixer agent, else the fleet's default
// fixer — the same chain the review-fix kick uses (src/docs/review-swarm.md).
// fixer, then the optional fallback list. Busy candidates are excluded by
// agentAvailability only when that list is configured.
// It returns "" when no candidate is reachable, which tells the caller to leave
// the PR alone rather than spend budget nobody can answer.
func resolveRedPRFixer(cfg *config.Config, owner string, available func(string) bool) string {
candidates := []string{redPRFixOwner(owner), strings.TrimSpace(cfg.Review.FixerAgent), review.DefaultFixerAgent}
candidates = append(candidates, cfg.Review.EffectiveFixerFallbackAgents()...)
for _, c := range candidates {
if c == "" {
continue
Expand Down Expand Up @@ -420,6 +466,10 @@ func deliverReEngagement(
}
return false
}
if logger != nil && fixer != redPRFixOwner(o.Agent) {
logger.Info("re-engagement routed to fallback fixer", "repo", o.Repo, "pr", o.Number,
"owner", redPRFixOwner(o.Agent), "agent", fixer)
}
store.RecordReEngagement(o.Repo, o.Number)
return true
}
Expand Down Expand Up @@ -2523,6 +2573,8 @@ func planReviewDispatchWith(ctx context.Context, cfg *config.Config, actionable
CombinedPerspectives: cfg.Review.CombinedPerspectives,
ReviewerAgents: cfg.Review.ReviewerAgents,
FixerAgent: cfg.Review.FixerAgent,
FixerFallbackAgents: cfg.Review.EffectiveFixerFallbackAgents(),
FixerAvailable: reviewFixerAvailability(cfg, agentMgr),
PostComments: cfg.Review.PostComments,
WritingGuideSection: cfg.Project.WritingGuideSection(),
FixHumanPRs: cfg.Review.FixHumanPRsEnabled(),
Expand All @@ -2547,6 +2599,12 @@ func planReviewDispatchWith(ctx context.Context, cfg *config.Config, actionable
DeltaTrivialLines: cfg.Review.EffectiveDeltaTrivialLines(),
DeltaMaxAge: cfg.Review.EffectiveDeltaMaxAge(),
})
for _, k := range plan.FixKicks {
if slices.Contains(cfg.Review.EffectiveFixerFallbackAgents(), k.Agent) {
logger.Info("review fix routed to fallback fixer", "repo", k.Repo, "pr", k.Number,
"primary", cfg.Review.FixerAgent, "agent", k.Agent)
}
}
recordReviewDeltaRestamps(plan, logger)
pendingRecords, pendingAgents, confirmedAgents := review.PendingReviewerOccupancy(plan.State)
if pendingRecords > 0 || plan.SettledReviews > 0 {
Expand Down
10 changes: 9 additions & 1 deletion src/docs/review-swarm.md
Original file line number Diff line number Diff line change
Expand Up @@ -182,6 +182,8 @@ review:
max_parallel_reviews: 5
reviewer_agents: [reviewer-a, reviewer-b] # optional; otherwise agents with review role/keywords are selected
fixer_agent: scanner # optional; defaults to the PR lane, then scanner
fixer_fallback_agents: [] # opt-in ordered fallback lanes, e.g. [fixer, scanner-2]
fixer_fallback_after_undeliverable: 3 # 0 defaults to 3 only when fallbacks are configured
confidence_score: true # optional; append the 0–5 mergeability line to review comments
self_authored_decision_policy: resolve # resolve (default) non-risk-class Hive-authored decision questions, or escalate all of them
# all_authors: true # deprecated no-op; every open PR is reviewed regardless of author
Expand Down Expand Up @@ -555,7 +557,13 @@ Expiry also covers a head whose aggregate is `incomplete`: the delivered perspec

The PR review queue (`GET /api/review/queue`) shows this state on unreviewed items: `pending_since` (first dispatch for the current head still pending) and `redispatch_count`, plus a reason such as `review dispatched 32m ago, no verdict yet (re-dispatched 2x after expiry)` or `review dispatch exhausted: …; waiting on a human` after the generic `not reviewed at this head` line.

When an aggregate verdict is `changes_requested`, Hive builds a review-fix kick containing the aggregate findings and sends it to `review.fixer_agent`, the classified PR lane, or `scanner`. Fix dispatches are capped by `escalation.MaxReEngagements`; once exhausted, dispatch state records a `requires_human` hold for the PR head so the automated loop stops.
When an aggregate verdict is `changes_requested`, Hive builds a review-fix kick containing the aggregate findings and sends it to `review.fixer_agent`, the classified PR lane, or `scanner`.

Set `review.fixer_fallback_agents` to an ordered list of enabled, kick-capable repair agents to route around an undeliverable fixer. An empty or unset list preserves existing behavior; setting `review.fixer_fallback_after_undeliverable` alone has no effect. With the list set, a candidate yields after that many consecutive undeliverable kicks (default 3), or after at least one refusal when its last known transcript activity is older than its agent-level `busy_no_activity_threshold` (default 30m). Missing transcript activity is unknown, and an idle agent with no outstanding refusals does not yield merely because its transcript is old.

Both review-fix kicks and stale CI-red PR re-engagements retain their existing owner/fixer selection order, then try the configured fallback list in order. Paused, disabled, on-demand, non-kick, cadence-paused, provider-backed-off or already refusing fallback agents are skipped. Routing is logged, and the next repair returns to the primary once its refusal count clears. No turn is interrupted or restarted, and a fix already delivered to a fallback remains pending under the usual `pending_fix_ttl` rather than being dispatched twice.

Fix dispatches are capped by `escalation.MaxReEngagements`; once exhausted, dispatch state records a `requires_human` hold for the PR head so the automated loop stops.

Once that cap is hit the hive is no longer fixing the PR, and the stage must say so ([#15744](https://github.com/hivecommons/hive/issues/15744)):

Expand Down
5 changes: 5 additions & 0 deletions src/hive.yaml.example
Original file line number Diff line number Diff line change
Expand Up @@ -857,6 +857,11 @@ governor:
# # perspective; shared review boilerplate is sent once instead of once
# # per perspective. Verdicts stay per-perspective. Default false.
# fixer_agent: "" # optional override; defaults to the PR lane, then scanner
# fixer_fallback_agents: [] # opt-in ordered repair lanes, e.g. [fixer, scanner-2]
# fixer_fallback_after_undeliverable: 3 # consecutive refusals before rerouting;
# # 0 defaults to 3 only when the fallback list is non-empty.
# # A busy fixer silent past its busy_no_activity_threshold
# # (default 30m) also yields. No turn is interrupted.
# pending_ttl: 45m # re-dispatch a review whose kick posted no verdict after this long
# # (Go duration; default 45m; negative disables expiry)
# pending_fix_ttl: 2h # dispatch the next bounded fix attempt when a fixer turn has
Expand Down
6 changes: 6 additions & 0 deletions src/pkg/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -7831,6 +7831,12 @@ type ReviewConfig struct {
MaxParallelReviews int `yaml:"max_parallel_reviews,omitempty" json:"max_parallel_reviews,omitempty"`
ReviewerAgents []string `yaml:"reviewer_agents,omitempty" json:"reviewer_agents,omitempty"`
FixerAgent string `yaml:"fixer_agent,omitempty" json:"fixer_agent,omitempty"`
// FixerFallbackAgents opts into routing repairs around an undeliverable
// fixer, in list order. Empty preserves the historical delivery behavior.
FixerFallbackAgents []string `yaml:"fixer_fallback_agents,omitempty" json:"fixer_fallback_agents,omitempty"`
// FixerFallbackAfterUndeliverable is the consecutive refusal threshold;
// zero uses 3, only when FixerFallbackAgents is non-empty.
FixerFallbackAfterUndeliverable int `yaml:"fixer_fallback_after_undeliverable,omitempty" json:"fixer_fallback_after_undeliverable,omitempty"`
// PostComments tells review-swarm reviewers to publish their verdict as a
// PR comment via the `hive-review` relay, in addition to returning the
// JSON aggregate. Opt-in: the zero value keeps the verdict internal, which
Expand Down
24 changes: 24 additions & 0 deletions src/pkg/config/review_dispatch.go
Original file line number Diff line number Diff line change
Expand Up @@ -237,3 +237,27 @@ func (r ReviewConfig) ValidateDeltaMaxAge() error {
}
return nil
}

// EffectiveFixerFallbackAfterUndeliverable is zero while fallback routing is
// disabled, so setting a threshold alone cannot change existing behavior.
func (r ReviewConfig) EffectiveFixerFallbackAfterUndeliverable() int {
if len(r.EffectiveFixerFallbackAgents()) == 0 {
return 0
}
if r.FixerFallbackAfterUndeliverable > 0 {
return r.FixerFallbackAfterUndeliverable
}
return 3
}

// EffectiveFixerFallbackAgents returns the ordered, non-empty agent names.
func (r ReviewConfig) EffectiveFixerFallbackAgents() []string {
return trimmedNonEmpty(r.FixerFallbackAgents)
}

func (r ReviewConfig) ValidateFixerFallback() error {
if r.FixerFallbackAfterUndeliverable < 0 {
return fmt.Errorf("review: fixer_fallback_after_undeliverable must be >= 0")
}
return nil
}
Loading
Loading