diff --git a/changelog.d/fixed-19033-backoff-kick-refusal.md b/changelog.d/fixed-19033-backoff-kick-refusal.md new file mode 100644 index 0000000000..9ab08a16d2 --- /dev/null +++ b/changelog.d/fixed-19033-backoff-kick-refusal.md @@ -0,0 +1 @@ +- Record a governor kick withheld during provider backoff as a `refused` (`provider_backoff`) outcome in `GET /api/kick-outcomes`, without quoting the provider error line (#19033). diff --git a/src/cmd/hive/kick_outcome_wire.go b/src/cmd/hive/kick_outcome_wire.go index 3f336f84db..27e4cfef7f 100644 --- a/src/cmd/hive/kick_outcome_wire.go +++ b/src/cmd/hive/kick_outcome_wire.go @@ -105,6 +105,18 @@ func kickOutcomeSender(s *kickoutcome.Store, send func(agent, message string) er } } +// kickOutcomeBackoffWithheld files a governor kick withheld during provider +// backoff as a refusal (provider_backoff, kick-refusals.md). The reason is +// built from the hive's own error class and retry time only: the provider +// error line is agent output, so it is never quoted here. +func kickOutcomeBackoffWithheld(s *kickoutcome.Store, agent, class string, retryIn time.Duration, at time.Time) { + if s == nil { + return + } + reason := "agent " + agent + " blocked: inference (" + class + "); next provider probe in " + retryIn.Round(time.Second).String() + s.Refused(agent, kickoutcome.Trigger{Kind: kickoutcome.TriggerGovernor, Actor: "governor"}, "provider_backoff", reason, at) +} + // isKickBusy reports a busy bounce (agent mid-turn). func isKickBusy(err error) bool { return errors.Is(err, crux.ErrAgentBusy) } diff --git a/src/cmd/hive/kick_outcome_wire_test.go b/src/cmd/hive/kick_outcome_wire_test.go index 9804fb168e..67c48ba2fa 100644 --- a/src/cmd/hive/kick_outcome_wire_test.go +++ b/src/cmd/hive/kick_outcome_wire_test.go @@ -1,7 +1,9 @@ package main import ( + "encoding/json" "errors" + "strings" "testing" "time" @@ -67,3 +69,23 @@ func TestKickOutcomeDeliveredIgnoresOtherEvents(t *testing.T) { t.Fatal("a delivered event did not") } } + +func TestKickOutcomeBackoffWithheldIsRefusedWithoutAgentText(t *testing.T) { + s, _ := kickoutcome.Open("", nil) + kickOutcomeBackoffWithheld(s, "scanner", "rate_limit", 90*time.Second, time.Now()) + latest, detail := kickOutcomeSource{s}.Recent("scanner", 10) + b, err := json.Marshal(map[string]any{"latest": latest, "kicks": detail}) + if err != nil { + t.Fatal(err) + } + out := string(b) + if !strings.Contains(out, `"refused"`) || !strings.Contains(out, "provider_backoff") { + t.Fatalf("withheld kick not recorded as a provider_backoff refusal: %s", out) + } + // The reason is the hive's own text; the provider error line (agent + // output) is never passed in, so nothing of it can be quoted. + if strings.Contains(out, "quota exceeded") { + t.Fatalf("agent text quoted: %s", out) + } + kickOutcomeBackoffWithheld(nil, "scanner", "x", 0, time.Now()) +} diff --git a/src/cmd/hive/main_eval_cycle.go b/src/cmd/hive/main_eval_cycle.go index 0032ce4f91..4ecff8998d 100644 --- a/src/cmd/hive/main_eval_cycle.go +++ b/src/cmd/hive/main_eval_cycle.go @@ -1094,6 +1094,7 @@ func runEvalCycle( "class", class, "retry_in", remaining.Round(time.Second), "error", line) + kickOutcomeBackoffWithheld(kickOutcomesFor(cfg, logger), msg.Agent, class, remaining, time.Now()) gov.RecordKickFailure(msg.Agent, fmt.Errorf("provider inference backoff (%s): %s", class, line), time.Now()) continue } diff --git a/src/docs/kick-refusals.md b/src/docs/kick-refusals.md index 4bdff020e6..4c5245ba27 100644 --- a/src/docs/kick-refusals.md +++ b/src/docs/kick-refusals.md @@ -220,7 +220,7 @@ workflow hand-off), when it started and ended, and what it came to: | `pushed_commit` | the relay pushed a branch for the agent; the record keeps the branch and sha | | `commented` | the relay posted a comment for the agent | | `stood_down` | the agent ended its turn on a stand-down line; the record quotes that line | -| `refused` | the hive refused to deliver the kick, or the agent's model declined it; the record keeps the reason code from this page and the quoted text | +| `refused` | the hive refused to deliver the kick (including a governor kick withheld during `provider_backoff`, recorded without the provider's error line), or the agent's model declined it; the record keeps the reason code from this page and the quoted text | | `still_running` | the kick's turn has not ended and no signal has arrived yet | | `ended_no_visible_action` | the turn ended, or can no longer be told apart from one that ended, and no signal above speaks for it |