diff --git a/changelog.d/fixed-reserve-floor-refusal-wording.md b/changelog.d/fixed-reserve-floor-refusal-wording.md new file mode 100644 index 0000000000..4dace2d16b --- /dev/null +++ b/changelog.d/fixed-reserve-floor-refusal-wording.md @@ -0,0 +1 @@ +- **An agent refused because the hive's shared GitHub API budget is low is now told so, with the reset time, and keeps working.** When the App installation's remaining core budget fell under `github.agent_reserve_floor`, the proxy's 429 and pane nudge said "hourly cap reached" and "stop all GitHub reads for this session", the wording for an agent's own `agents.github_api_hourly_cap`. On knuckle (2026-11-08) sec-check was refused at 379/400 and abandoned its whole audit. The reserve refusal now says the shared budget is low (not the agent's cap), gives the reset time and `Retry-After`, and tells the agent to pause GitHub reads until then and carry on with local work. The cap refusal's wording is unchanged. diff --git a/src/crux/proxy/agent_github_budget.go b/src/crux/proxy/agent_github_budget.go index d76bd28647..279bbf7015 100644 --- a/src/crux/proxy/agent_github_budget.go +++ b/src/crux/proxy/agent_github_budget.go @@ -274,9 +274,10 @@ func (p *GitHubProxy) writeAgentGitHubBudget429(client netWriterConn, st agentGi if st.RetryAfter <= 0 { st.RetryAfter = 1 } + nudge, message := agentGitHubBudgetMessages(st, p.agentGitHubBudgetNow()) first := p.githubBudget != nil && p.githubBudget.markNudged(st.Agent) if first && p.githubBudgetNudge != nil { - p.githubBudgetNudge(st.Agent, "Stop polling GitHub. gh returned 429 hourly cap reached; stop all GitHub reads for this session and finish with local work.") + p.githubBudgetNudge(st.Agent, nudge) } if p.logger != nil { // Warn once per agent per cap window; repeats are expected and self-limiting. @@ -288,7 +289,7 @@ func (p *GitHubProxy) writeAgentGitHubBudget429(client netWriterConn, st agentGi "agent", st.Agent, "count", st.Count, "cap", st.Cap, "retry_after", st.RetryAfter, "reason", st.Reason) } payload, _ := json.Marshal(map[string]string{ - "message": "hive: agent GitHub API hourly cap reached (" + strconv.Itoa(st.Count) + "/" + strconv.Itoa(st.Cap) + "); stop polling and continue with local work", + "message": message, "documentation_url": githubAPICapDocsURL, }) resp := &http.Response{ @@ -312,6 +313,51 @@ func (p *GitHubProxy) writeAgentGitHubBudget429(client netWriterConn, st agentGi return true } +func (p *GitHubProxy) agentGitHubBudgetNow() time.Time { + if p != nil && p.githubBudget != nil && p.githubBudget.now != nil { + return p.githubBudget.now() + } + return time.Now() +} + +// agentGitHubBudgetMessages returns the nudge typed into the agent's pane and +// the 429 body gh prints for a budget refusal. +// +// The two refusals need different advice. "cap" is the agent's own +// agents.github_api_hourly_cap: it has polled too much, so it stops GitHub +// reads for the session. "reserve" is github.agent_reserve_floor: the App +// installation's shared core bucket, which the daemon also spends, is nearly +// empty. That is not the agent's doing and it ends at the bucket's reset, so +// the agent pauses until then rather than giving up. Saying "hourly cap +// reached ... for this session" there made sec-check abandon its whole audit +// on knuckle (2026-11-08, refused at 379/400). +// +// The reserve wording warns that git fetch and clone may be refused too: by +// default github.com is tunneled and never reaches this gate, but when it is +// intercepted (HIVE_PROXY_INJECT_GH_AUTH, or a lane under +// write_surface.enforce) agentGitHubRequestIsRead counts info/refs and +// git-upload-pack as reads, so the floor refuses them as well. +func agentGitHubBudgetMessages(st agentGitHubBudgetStatus, now time.Time) (nudge, message string) { + if st.Reason != "reserve" { + return "Stop polling GitHub. gh returned 429 hourly cap reached; stop all GitHub reads for this session and finish with local work.", + "hive: agent GitHub API hourly cap reached (" + strconv.Itoa(st.Count) + "/" + strconv.Itoa(st.Cap) + "); stop polling and continue with local work" + } + retry := st.RetryAfter + if retry <= 0 { + retry = 1 + } + until := now.Add(time.Duration(retry) * time.Second).UTC().Format(time.RFC3339) + minutes := strconv.Itoa((retry + 59) / 60) + left := strconv.Itoa(st.Count) + " left, reserve floor " + strconv.Itoa(st.Cap) + nudge = "Pause GitHub reads until " + until + " (about " + minutes + " min), then try again. " + + "gh returned 429 because the hive's shared GitHub API budget is low (" + left + "), not because you reached your own cap. " + + "Until then gh and other GitHub API reads are refused, and git fetch or clone may be too; carry on with local work and do not end the task over this." + message = "hive: the hive's shared GitHub API budget is low (" + left + "); this is not this agent's own cap. " + + "GitHub API reads are paused until " + until + " (Retry-After " + strconv.Itoa(retry) + "s), and git fetch or clone may be too; " + + "try again then and continue with local work meanwhile" + return nudge, message +} + type netWriterConn interface { SetWriteDeadline(time.Time) error Write([]byte) (int, error) diff --git a/src/crux/proxy/agent_github_budget_test.go b/src/crux/proxy/agent_github_budget_test.go index 2583a40896..cd7bbb46ea 100644 --- a/src/crux/proxy/agent_github_budget_test.go +++ b/src/crux/proxy/agent_github_budget_test.go @@ -118,6 +118,91 @@ func TestAgentGitHubBudget429ShapeAndNudge(t *testing.T) { } } +// readAgentGitHubBudget429 writes st through writeAgentGitHubBudget429 and +// returns the nudge typed into the pane and the 429 body's message. +func readAgentGitHubBudget429(t *testing.T, now time.Time, st agentGitHubBudgetStatus) (nudge, message string, resp *http.Response) { + t.Helper() + server, client := net.Pipe() + defer server.Close() + defer client.Close() + p := &GitHubProxy{ + githubBudget: newAgentGitHubBudget(func() time.Time { return now }), + githubBudgetNudge: func(_, msg string) { nudge = msg }, + } + done := make(chan bool, 1) + go func() { done <- p.writeAgentGitHubBudget429(server, st) }() + resp, err := http.ReadResponse(bufio.NewReader(client), nil) + if err != nil { + t.Fatalf("ReadResponse: %v", err) + } + var body map[string]string + if err := json.NewDecoder(resp.Body).Decode(&body); err != nil { + t.Fatalf("decode body: %v", err) + } + _ = resp.Body.Close() + if ok := <-done; !ok { + t.Fatal("writeAgentGitHubBudget429 returned false") + } + return nudge, body["message"], resp +} + +// The agent's own hourly cap keeps its wording: agents (and the lane policies +// that quote "hourly cap reached") stop GitHub reads for the session. +func TestAgentGitHubBudgetCapRefusalWordingUnchanged(t *testing.T) { + now := time.Date(2026, 11, 8, 11, 35, 20, 0, time.UTC) + nudge, message, _ := readAgentGitHubBudget429(t, now, agentGitHubBudgetStatus{ + Agent: "scanner", Count: 300, Cap: 300, RetryAfter: 900, Reason: "cap", + }) + if want := "Stop polling GitHub. gh returned 429 hourly cap reached; stop all GitHub reads for this session and finish with local work."; nudge != want { + t.Fatalf("cap nudge = %q, want %q", nudge, want) + } + if want := "hive: agent GitHub API hourly cap reached (300/300); stop polling and continue with local work"; message != want { + t.Fatalf("cap message = %q, want %q", message, want) + } +} + +// A refusal under github.agent_reserve_floor is the hive's shared budget, not +// the agent's cap. On knuckle (2026-11-08 11:35:22Z) sec-check was refused at +// 379/400, read "hourly cap reached ... stop all GitHub reads for this +// session", and gave up its whole audit. The wording must say whose budget it +// is, when it resets, and that the agent pauses rather than stops. +func TestAgentGitHubBudgetReserveRefusalNamesSharedBudgetAndReset(t *testing.T) { + now := time.Date(2026, 11, 8, 11, 35, 22, 0, time.UTC) + nudge, message, resp := readAgentGitHubBudget429(t, now, agentGitHubBudgetStatus{ + Agent: "sec-check", Count: 379, Cap: 400, RetryAfter: 1260, Reason: "reserve", + Method: http.MethodGet, Path: "/repos/o/r/contents/x", + }) + if got := resp.Header.Get("Retry-After"); got != "1260" { + t.Fatalf("Retry-After = %q, want 1260", got) + } + for name, text := range map[string]string{"nudge": nudge, "message": message} { + for _, want := range []string{ + "shared GitHub API budget is low (379 left, reserve floor 400)", + "2026-11-08T11:56:22Z", + "git fetch or clone may be too", + "local work", + } { + if !strings.Contains(text, want) { + t.Errorf("reserve %s = %q, want it to contain %q", name, text, want) + } + } + for _, bad := range []string{"hourly cap reached", "for this session", "Stop polling"} { + if strings.Contains(text, bad) { + t.Errorf("reserve %s = %q, must not contain %q", name, text, bad) + } + } + } + if !strings.Contains(nudge, "about 21 min") { + t.Errorf("reserve nudge = %q, want the wait in minutes", nudge) + } + if !strings.Contains(nudge, "do not end the task") { + t.Errorf("reserve nudge = %q, want it to say the task goes on", nudge) + } + if !strings.Contains(message, "Retry-After 1260s") { + t.Errorf("reserve message = %q, want the Retry-After seconds", message) + } +} + func TestAgentGitHubBudgetCapWarnOncePerWindow(t *testing.T) { now := time.Date(2026, 10, 8, 12, 0, 0, 0, time.UTC) var buf bytes.Buffer diff --git a/src/docs/operator-reference/runtime-and-quota.md b/src/docs/operator-reference/runtime-and-quota.md index 14d32a4eb1..8f0532ac61 100644 --- a/src/docs/operator-reference/runtime-and-quota.md +++ b/src/docs/operator-reference/runtime-and-quota.md @@ -170,8 +170,8 @@ loops, so the proxy enforces two safeguards: A config reload reaches the proxy without a restart. - `github.agent_reserve_floor` (default `400`) protects the shared core bucket. When the latest rate-limit reading shows `core.remaining` below the floor, - agent read requests receive the same proxy `429` even if the agent has not - reached its personal cap. This keeps daemon merge/scan work from being + agent read requests receive a proxy `429` even if the agent has not + reached its personal cap, with its own wording (below). This keeps daemon merge/scan work from being starved by agent polling. Only readings from the hive's own bucket update that snapshot: responses whose `X-RateLimit-Limit` is 60 (GitHub's per-IP unauthenticated bucket, e.g. an agent's `gh` call without a token) or whose @@ -196,15 +196,33 @@ loops, so the proxy enforces two safeguards: once. Bodies are stored gzip-compressed, so the cap is several times its size in raw JSON. -The proxy returns JSON shaped for the GitHub CLI: +The proxy returns JSON shaped for the GitHub CLI. For the agent's own hourly +cap: ```json {"message":"hive: agent GitHub API hourly cap reached (N/cap); stop polling and continue with local work","documentation_url":"…docs/operator-reference.md#github-api-quota"} ``` +For the reserve floor, the message says the shared budget is low rather than +that the agent hit its cap, and gives the time the shared bucket resets: + +```json +{"message":"hive: the hive's shared GitHub API budget is low (N left, reserve floor F); this is not this agent's own cap. GitHub API reads are paused until (Retry-After Ss), and git fetch or clone may be too; try again then and continue with local work meanwhile","documentation_url":"…docs/operator-reference.md#github-api-quota"} +``` + +Git fetch and clone travel over `github.com`, which the proxy tunnels without +looking inside by default, so the floor does not reach them. When it +intercepts `github.com` (`HIVE_PROXY_INJECT_GH_AUTH=true`, or a lane under +`write_surface.enforce`), it counts `info/refs` and `git-upload-pack` as reads +and refuses them below the floor too; hence "may be". + It also sets `Retry-After` to when the one-hour window (or observed reset) can -make progress, logs once at 80% of each agent window, and sends one -stop-polling nudge per agent window. Hive-authored write relays are not +make progress, logs once at 80% of each agent window, and sends one nudge per +agent window. The cap nudge tells the agent to stop GitHub reads for the +session. The reserve nudge tells it to pause GitHub reads until the reset time, +carry on with local work, and not end the task over it. Both refusals log +`agent GitHub API hourly cap reached`, told apart by the `reason` field +(`cap` or `reserve`). Hive-authored write relays are not agent-attributed and are not capped by this path. Open-PR scans use `github.graphql_pr_batch` by default to move mergeability, diff --git a/src/pkg/review/unreadable_test.go b/src/pkg/review/unreadable_test.go index e45e7b0e37..98fcdba393 100644 --- a/src/pkg/review/unreadable_test.go +++ b/src/pkg/review/unreadable_test.go @@ -33,6 +33,7 @@ func TestUnreadableReviewReasonRecognisesRateLimitedReviews(t *testing.T) { "Could not fetch the diff: gh returned 429 Too Many Requests.", "Unable to read the PR files: API rate limit exceeded for installation.", "I cannot read the diff; hive: agent GitHub API hourly cap reached (400/400).", + "Could not read the diff; hive: the hive's shared GitHub API budget is low (379 left, reserve floor 400); this is not this agent's own cap.", "Could not read the diff: secondary rate-limit hit on every request.", } for _, summary := range others {