From be481f3ed58411806a966f9a9b469bf17e1245a6 Mon Sep 17 00:00:00 2001 From: Danathar Date: Mon, 9 Nov 2026 06:56:30 -0500 Subject: [PATCH] fix(proxy): a reserve-floor refusal says the shared budget is low and when it resets When the App installation's shared core budget fell under github.agent_reserve_floor, the proxy's 429 body and pane nudge used the per-agent cap wording ("hourly cap reached ... stop all GitHub reads for this session"). On knuckle (2026-11-08 11:35:22Z) sec-check was refused at 379/400 with reason=reserve and abandoned its whole audit. The reserve refusal now says the hive's shared GitHub API budget is low (not the agent's own cap), gives the reset time and Retry-After, warns that git fetch or clone may be refused too while github.com is intercepted, and tells the agent to pause and carry on with local work. The cap refusal's wording is unchanged. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BQYnPGFJx81c8JXR3o1sg1 Signed-off-by: Danathar --- .../fixed-reserve-floor-refusal-wording.md | 1 + src/crux/proxy/agent_github_budget.go | 50 ++++++++++- src/crux/proxy/agent_github_budget_test.go | 85 +++++++++++++++++++ .../operator-reference/runtime-and-quota.md | 28 ++++-- src/pkg/review/unreadable_test.go | 1 + 5 files changed, 158 insertions(+), 7 deletions(-) create mode 100644 changelog.d/fixed-reserve-floor-refusal-wording.md diff --git a/changelog.d/fixed-reserve-floor-refusal-wording.md b/changelog.d/fixed-reserve-floor-refusal-wording.md new file mode 100644 index 0000000000..4dace2d16b --- /dev/null +++ b/changelog.d/fixed-reserve-floor-refusal-wording.md @@ -0,0 +1 @@ +- **An agent refused because the hive's shared GitHub API budget is low is now told so, with the reset time, and keeps working.** When the App installation's remaining core budget fell under `github.agent_reserve_floor`, the proxy's 429 and pane nudge said "hourly cap reached" and "stop all GitHub reads for this session", the wording for an agent's own `agents.github_api_hourly_cap`. On knuckle (2026-11-08) sec-check was refused at 379/400 and abandoned its whole audit. The reserve refusal now says the shared budget is low (not the agent's cap), gives the reset time and `Retry-After`, and tells the agent to pause GitHub reads until then and carry on with local work. The cap refusal's wording is unchanged. diff --git a/src/crux/proxy/agent_github_budget.go b/src/crux/proxy/agent_github_budget.go index 3f00143cc0..4001cd9667 100644 --- a/src/crux/proxy/agent_github_budget.go +++ b/src/crux/proxy/agent_github_budget.go @@ -194,9 +194,10 @@ func (p *GitHubProxy) writeAgentGitHubBudget429(client netWriterConn, st agentGi if st.RetryAfter <= 0 { st.RetryAfter = 1 } + nudge, message := agentGitHubBudgetMessages(st, p.agentGitHubBudgetNow()) first := p.githubBudget != nil && p.githubBudget.markNudged(st.Agent) if first && p.githubBudgetNudge != nil { - p.githubBudgetNudge(st.Agent, "Stop polling GitHub. gh returned 429 hourly cap reached; stop all GitHub reads for this session and finish with local work.") + p.githubBudgetNudge(st.Agent, nudge) } if p.logger != nil { // Warn once per agent per cap window; repeats are expected and self-limiting. @@ -208,7 +209,7 @@ func (p *GitHubProxy) writeAgentGitHubBudget429(client netWriterConn, st agentGi "agent", st.Agent, "count", st.Count, "cap", st.Cap, "retry_after", st.RetryAfter, "reason", st.Reason) } payload, _ := json.Marshal(map[string]string{ - "message": "hive: agent GitHub API hourly cap reached (" + strconv.Itoa(st.Count) + "/" + strconv.Itoa(st.Cap) + "); stop polling and continue with local work", + "message": message, "documentation_url": githubAPICapDocsURL, }) resp := &http.Response{ @@ -232,6 +233,51 @@ func (p *GitHubProxy) writeAgentGitHubBudget429(client netWriterConn, st agentGi return true } +func (p *GitHubProxy) agentGitHubBudgetNow() time.Time { + if p != nil && p.githubBudget != nil && p.githubBudget.now != nil { + return p.githubBudget.now() + } + return time.Now() +} + +// agentGitHubBudgetMessages returns the nudge typed into the agent's pane and +// the 429 body gh prints for a budget refusal. +// +// The two refusals need different advice. "cap" is the agent's own +// agents.github_api_hourly_cap: it has polled too much, so it stops GitHub +// reads for the session. "reserve" is github.agent_reserve_floor: the App +// installation's shared core bucket, which the daemon also spends, is nearly +// empty. That is not the agent's doing and it ends at the bucket's reset, so +// the agent pauses until then rather than giving up. Saying "hourly cap +// reached ... for this session" there made sec-check abandon its whole audit +// on knuckle (2026-11-08, refused at 379/400). +// +// The reserve wording warns that git fetch and clone may be refused too: by +// default github.com is tunneled and never reaches this gate, but when it is +// intercepted (HIVE_PROXY_INJECT_GH_AUTH, or a lane under +// write_surface.enforce) agentGitHubRequestIsRead counts info/refs and +// git-upload-pack as reads, so the floor refuses them as well. +func agentGitHubBudgetMessages(st agentGitHubBudgetStatus, now time.Time) (nudge, message string) { + if st.Reason != "reserve" { + return "Stop polling GitHub. gh returned 429 hourly cap reached; stop all GitHub reads for this session and finish with local work.", + "hive: agent GitHub API hourly cap reached (" + strconv.Itoa(st.Count) + "/" + strconv.Itoa(st.Cap) + "); stop polling and continue with local work" + } + retry := st.RetryAfter + if retry <= 0 { + retry = 1 + } + until := now.Add(time.Duration(retry) * time.Second).UTC().Format(time.RFC3339) + minutes := strconv.Itoa((retry + 59) / 60) + left := strconv.Itoa(st.Count) + " left, reserve floor " + strconv.Itoa(st.Cap) + nudge = "Pause GitHub reads until " + until + " (about " + minutes + " min), then try again. " + + "gh returned 429 because the hive's shared GitHub API budget is low (" + left + "), not because you reached your own cap. " + + "Until then gh and other GitHub API reads are refused, and git fetch or clone may be too; carry on with local work and do not end the task over this." + message = "hive: the hive's shared GitHub API budget is low (" + left + "); this is not this agent's own cap. " + + "GitHub API reads are paused until " + until + " (Retry-After " + strconv.Itoa(retry) + "s), and git fetch or clone may be too; " + + "try again then and continue with local work meanwhile" + return nudge, message +} + type netWriterConn interface { SetWriteDeadline(time.Time) error Write([]byte) (int, error) diff --git a/src/crux/proxy/agent_github_budget_test.go b/src/crux/proxy/agent_github_budget_test.go index 102409c4d8..f5bc617d44 100644 --- a/src/crux/proxy/agent_github_budget_test.go +++ b/src/crux/proxy/agent_github_budget_test.go @@ -118,6 +118,91 @@ func TestAgentGitHubBudget429ShapeAndNudge(t *testing.T) { } } +// readAgentGitHubBudget429 writes st through writeAgentGitHubBudget429 and +// returns the nudge typed into the pane and the 429 body's message. +func readAgentGitHubBudget429(t *testing.T, now time.Time, st agentGitHubBudgetStatus) (nudge, message string, resp *http.Response) { + t.Helper() + server, client := net.Pipe() + defer server.Close() + defer client.Close() + p := &GitHubProxy{ + githubBudget: newAgentGitHubBudget(func() time.Time { return now }), + githubBudgetNudge: func(_, msg string) { nudge = msg }, + } + done := make(chan bool, 1) + go func() { done <- p.writeAgentGitHubBudget429(server, st) }() + resp, err := http.ReadResponse(bufio.NewReader(client), nil) + if err != nil { + t.Fatalf("ReadResponse: %v", err) + } + var body map[string]string + if err := json.NewDecoder(resp.Body).Decode(&body); err != nil { + t.Fatalf("decode body: %v", err) + } + _ = resp.Body.Close() + if ok := <-done; !ok { + t.Fatal("writeAgentGitHubBudget429 returned false") + } + return nudge, body["message"], resp +} + +// The agent's own hourly cap keeps its wording: agents (and the lane policies +// that quote "hourly cap reached") stop GitHub reads for the session. +func TestAgentGitHubBudgetCapRefusalWordingUnchanged(t *testing.T) { + now := time.Date(2026, 11, 8, 11, 35, 20, 0, time.UTC) + nudge, message, _ := readAgentGitHubBudget429(t, now, agentGitHubBudgetStatus{ + Agent: "scanner", Count: 300, Cap: 300, RetryAfter: 900, Reason: "cap", + }) + if want := "Stop polling GitHub. gh returned 429 hourly cap reached; stop all GitHub reads for this session and finish with local work."; nudge != want { + t.Fatalf("cap nudge = %q, want %q", nudge, want) + } + if want := "hive: agent GitHub API hourly cap reached (300/300); stop polling and continue with local work"; message != want { + t.Fatalf("cap message = %q, want %q", message, want) + } +} + +// A refusal under github.agent_reserve_floor is the hive's shared budget, not +// the agent's cap. On knuckle (2026-11-08 11:35:22Z) sec-check was refused at +// 379/400, read "hourly cap reached ... stop all GitHub reads for this +// session", and gave up its whole audit. The wording must say whose budget it +// is, when it resets, and that the agent pauses rather than stops. +func TestAgentGitHubBudgetReserveRefusalNamesSharedBudgetAndReset(t *testing.T) { + now := time.Date(2026, 11, 8, 11, 35, 22, 0, time.UTC) + nudge, message, resp := readAgentGitHubBudget429(t, now, agentGitHubBudgetStatus{ + Agent: "sec-check", Count: 379, Cap: 400, RetryAfter: 1260, Reason: "reserve", + Method: http.MethodGet, Path: "/repos/o/r/contents/x", + }) + if got := resp.Header.Get("Retry-After"); got != "1260" { + t.Fatalf("Retry-After = %q, want 1260", got) + } + for name, text := range map[string]string{"nudge": nudge, "message": message} { + for _, want := range []string{ + "shared GitHub API budget is low (379 left, reserve floor 400)", + "2026-11-08T11:56:22Z", + "git fetch or clone may be too", + "local work", + } { + if !strings.Contains(text, want) { + t.Errorf("reserve %s = %q, want it to contain %q", name, text, want) + } + } + for _, bad := range []string{"hourly cap reached", "for this session", "Stop polling"} { + if strings.Contains(text, bad) { + t.Errorf("reserve %s = %q, must not contain %q", name, text, bad) + } + } + } + if !strings.Contains(nudge, "about 21 min") { + t.Errorf("reserve nudge = %q, want the wait in minutes", nudge) + } + if !strings.Contains(nudge, "do not end the task") { + t.Errorf("reserve nudge = %q, want it to say the task goes on", nudge) + } + if !strings.Contains(message, "Retry-After 1260s") { + t.Errorf("reserve message = %q, want the Retry-After seconds", message) + } +} + func TestAgentGitHubBudgetCapWarnOncePerWindow(t *testing.T) { now := time.Date(2026, 10, 8, 12, 0, 0, 0, time.UTC) var buf bytes.Buffer diff --git a/src/docs/operator-reference/runtime-and-quota.md b/src/docs/operator-reference/runtime-and-quota.md index 0587cf4e56..d987f34030 100644 --- a/src/docs/operator-reference/runtime-and-quota.md +++ b/src/docs/operator-reference/runtime-and-quota.md @@ -165,8 +165,8 @@ loops, so the proxy enforces two safeguards: override one agent (`0` disables that agent's cap). - `github.agent_reserve_floor` (default `400`) protects the shared core bucket. When the latest rate-limit reading shows `core.remaining` below the floor, - agent read requests receive the same proxy `429` even if the agent has not - reached its personal cap. This keeps daemon merge/scan work from being + agent read requests receive a proxy `429` even if the agent has not + reached its personal cap, with its own wording (below). This keeps daemon merge/scan work from being starved by agent polling. Only readings from the hive's own bucket update that snapshot: responses whose `X-RateLimit-Limit` is 60 (GitHub's per-IP unauthenticated bucket, e.g. an agent's `gh` call without a token) or whose @@ -191,15 +191,33 @@ loops, so the proxy enforces two safeguards: once. Bodies are stored gzip-compressed, so the cap is several times its size in raw JSON. -The proxy returns JSON shaped for the GitHub CLI: +The proxy returns JSON shaped for the GitHub CLI. For the agent's own hourly +cap: ```json {"message":"hive: agent GitHub API hourly cap reached (N/cap); stop polling and continue with local work","documentation_url":"…docs/operator-reference.md#github-api-quota"} ``` +For the reserve floor, the message says the shared budget is low rather than +that the agent hit its cap, and gives the time the shared bucket resets: + +```json +{"message":"hive: the hive's shared GitHub API budget is low (N left, reserve floor F); this is not this agent's own cap. GitHub API reads are paused until (Retry-After Ss), and git fetch or clone may be too; try again then and continue with local work meanwhile","documentation_url":"…docs/operator-reference.md#github-api-quota"} +``` + +Git fetch and clone travel over `github.com`, which the proxy tunnels without +looking inside by default, so the floor does not reach them. When it +intercepts `github.com` (`HIVE_PROXY_INJECT_GH_AUTH=true`, or a lane under +`write_surface.enforce`), it counts `info/refs` and `git-upload-pack` as reads +and refuses them below the floor too; hence "may be". + It also sets `Retry-After` to when the one-hour window (or observed reset) can -make progress, logs once at 80% of each agent window, and sends one -stop-polling nudge per agent window. Hive-authored write relays are not +make progress, logs once at 80% of each agent window, and sends one nudge per +agent window. The cap nudge tells the agent to stop GitHub reads for the +session. The reserve nudge tells it to pause GitHub reads until the reset time, +carry on with local work, and not end the task over it. Both refusals log +`agent GitHub API hourly cap reached`, told apart by the `reason` field +(`cap` or `reserve`). Hive-authored write relays are not agent-attributed and are not capped by this path. Open-PR scans use `github.graphql_pr_batch` by default to move mergeability, diff --git a/src/pkg/review/unreadable_test.go b/src/pkg/review/unreadable_test.go index 9636823193..9a36bcd96f 100644 --- a/src/pkg/review/unreadable_test.go +++ b/src/pkg/review/unreadable_test.go @@ -33,6 +33,7 @@ func TestUnreadableReviewReasonRecognisesRateLimitedReviews(t *testing.T) { "Could not fetch the diff: gh returned 429 Too Many Requests.", "Unable to read the PR files: API rate limit exceeded for installation.", "I cannot read the diff; hive: agent GitHub API hourly cap reached (400/400).", + "Could not read the diff; hive: the hive's shared GitHub API budget is low (379 left, reserve floor 400); this is not this agent's own cap.", "Could not read the diff: secondary rate-limit hit on every request.", } for _, summary := range others {