diff --git a/changelog.d/fixed-review-rate-limit-no-verdict.md b/changelog.d/fixed-review-rate-limit-no-verdict.md new file mode 100644 index 0000000000..8d2f41db7b --- /dev/null +++ b/changelog.d/fixed-review-rate-limit-no-verdict.md @@ -0,0 +1 @@ +- A reviewer that could not read a PR because of a rate limit (HTTP 429, GitHub's rate limit, or the hive's own per-agent `agents.github_api_hourly_cap`) no longer produces a `requires_human` verdict. The relay drops that report, the perspective's dispatch expires after `review.pending_ttl`, and it is asked again. Before, five "could not read … HTTP 429 … not reviewed" reports parked projectbluefin/testsuite#1116 at `stage/085-needs-decision` and the owner's delegate asked a person about "the security review withheld approval". On a contributor PR without the security report, the same verdict would have resolved hands-free to approve on a head nobody read. diff --git a/src/docs/review-swarm.md b/src/docs/review-swarm.md index a223881663..4dd1411121 100644 --- a/src/docs/review-swarm.md +++ b/src/docs/review-swarm.md @@ -583,6 +583,8 @@ The retry is bounded. Once a head's expiry count reaches `max_redispatch` (defau Expiry also covers a head whose aggregate is `incomplete`: the delivered perspectives are taken off the pending list (their reports exist) and only the undelivered ones expire and re-dispatch, against the same per-head budget. +**A review the reviewer could not read because of a rate limit is no verdict.** The read instruction tells a reviewer that could not read the diff to return `requires_human` and say so. When the only reason is a rate limit — an HTTP 429, GitHub's rate limit, or the hive proxy's per-agent cap (`agents.github_api_hourly_cap`) — the relay does not record that report (`review.UnreadableReviewReason`, `src/pkg/review/unreadable.go`). It must be `requires_human`, carry no in-scope finding, no `hive-decision-risk: high` marker and no explicit decision question (``, since a reviewer that asks the owner something read enough to ask it), and its summary must say both that the change was not read and that a rate limit was the cause. The perspective's pending entry stays open, expires after `pending_ttl` like any kick that posted nothing, and is asked again, so no person is asked and no security hold is claimed. Other perspectives in the same combined review that did read the PR are recorded as usual, and the head reads as `incomplete` until the re-asked ones report. The review-request watcher logs `unreadable review not recorded as a verdict`. Any review comment the reviewer already posted saying it could not read the PR stays on the PR with no verdict behind it, and the re-asked reviewer may post another. A `requires_human` for any other reason, such as a diff too large to read, still parks the PR. Before this, the five `could not read … HTTP 429` reports on projectbluefin/testsuite#1116 parked it at 085 as "the security review withheld approval". + The PR review queue (`GET /api/review/queue`) shows this state on unreviewed items: `pending_since` (first dispatch for the current head still pending) and `redispatch_count`, plus a reason such as `review dispatched 32m ago, no verdict yet (re-dispatched 2x after expiry)` or `review dispatch exhausted: …; waiting on a human` after the generic `not reviewed at this head` line. When an aggregate verdict is `changes_requested`, Hive builds a review-fix kick containing the aggregate findings and sends it to `review.fixer_agent`, the classified PR lane, or `scanner`. diff --git a/src/pkg/github/review_verdict_relay.go b/src/pkg/github/review_verdict_relay.go index 75996c6a04..a6c5e9e94d 100644 --- a/src/pkg/github/review_verdict_relay.go +++ b/src/pkg/github/review_verdict_relay.go @@ -111,6 +111,18 @@ func (c *Client) recordReviewVerdict(ctx context.Context, req ReviewRequest, dir report.RedundantOf = redundantOf } report.PermissionsTighteningOnly = tightening + if reason, unread := review.UnreadableReviewReason(report); unread { + // A rate limit kept the reviewer from reading the PR: that is + // no verdict, not a requires_human. Nothing is recorded, so no + // person is asked and no security hold is claimed; the dispatch + // row stays open, expires after review.pending_ttl, and the + // planner asks again (projectbluefin/testsuite#1116). + c.logger.Info("review-request watcher: unreadable review not recorded as a verdict; the perspective is re-asked after the pending TTL", + slog.String("repo", req.Repo), slog.Int("number", req.Number), + slog.String("perspective", string(report.Perspective)), + slog.String("agent", req.Agent), slog.String("reason", reason)) + continue + } if written, ok, dispatched := c.writeOneVerdict(ctx, req, report, dir, prHead, prSnapshot); ok { recorded = append(recorded, written) wake = wake || dispatched diff --git a/src/pkg/github/review_verdict_unreadable_test.go b/src/pkg/github/review_verdict_unreadable_test.go new file mode 100644 index 0000000000..9e432cd400 --- /dev/null +++ b/src/pkg/github/review_verdict_unreadable_test.go @@ -0,0 +1,162 @@ +package github + +import ( + "context" + "encoding/json" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/review" +) + +// The five summaries the reviewer wrote on projectbluefin/testsuite#1116 at +// head ae22434 (2026-11-07), when every GitHub read hit the agent API cap. +var rateLimitedSummaries1116 = map[review.Perspective]string{ + review.PerspectiveCorrectness: "Could not read the delta d397ece..ae22434: every GitHub read returned HTTP 429 (hive agent API hourly cap, 181-211/400) and no local checkout of projectbluefin/testsuite exists on this host. Not reviewed; previous correctness verdict (approve) was for head d397ece only.", + review.PerspectiveDocsCurrency: "Delta touches docs/skills/test-authoring/suite-map/SKILL.md but could not be read (GitHub API 429 cap, no local checkout); could not confirm whether prior finding SKILL.md:170 (immutability checks still marked @pending) is fixed. Status unknown.", + review.PerspectiveIntentAlignment: "Delta not readable (GitHub API 429 cap, no local checkout); could not re-check prior finding tests/common/features/common_immutable.feature:13 ('Closes #838' may over-claim scope) nor read issue #838. Status of prior finding unknown.", + review.PerspectiveSecurity: "Delta not readable (GitHub API 429 cap, no local checkout). Not reviewed at this head.", + review.PerspectiveStyle: "Delta not readable (GitHub API 429 cap, no local checkout). Not reviewed at this head.", +} + +func verdictJSONWithSummary(t *testing.T, repo string, number int, perspective review.Perspective, verdict, head, summary string) string { + t.Helper() + var report map[string]any + if err := json.Unmarshal([]byte(validVerdictJSONWithHead(t, repo, number, string(perspective), verdict, head)), &report); err != nil { + t.Fatalf("unmarshal verdict fixture: %v", err) + } + report["summary"] = summary + raw, err := json.Marshal(report) + if err != nil { + t.Fatalf("marshal: %v", err) + } + return string(raw) +} + +// projectbluefin/testsuite#1116: every GitHub read the reviewer made returned +// 429, so all five perspectives came back requires_human with "could not +// read … not reviewed". Recorded, that parked the PR at 085 and the +// delegate asked a person about "the security review withheld approval". +// It must be no verdict at all: nothing recorded, nobody asked, and the same +// perspectives asked again once the pending TTL has passed. +func TestRecordReviewVerdictRateLimitedReviewIsNoVerdictAndRetries(t *testing.T) { + const ( + repo = "projectbluefin/testsuite" + num = 1116 + head = "ae22434cfe05ce9c119f2e508634d599201ab327" + ) + dir := t.TempDir() + c := &Client{logger: testLogger()} + dispatched := time.Date(2026, 11, 7, 10, 30, 0, 0, time.UTC) + var pending []review.PendingReview + var elems []string + for _, p := range review.DefaultPerspectives { + pending = append(pending, review.PendingReview{Repo: repo, Number: num, HeadSHA: head, Perspective: p, Agent: "reviewer", Dispatched: dispatched}) + elems = append(elems, verdictJSONWithSummary(t, repo, num, p, "requires_human", head, rateLimitedSummaries1116[p])) + } + withVerdictDispatchState(t, review.DispatchState{Pending: pending}) + + recorded := c.recordReviewVerdict(context.Background(), ReviewRequest{ + Repo: repo, Number: num, Agent: "reviewer", Event: ReviewEventRecordVerdict, + Report: "[" + strings.Join(elems, ",") + "]", + }, dir) + + if len(recorded) != 0 { + t.Fatalf("recorded %d verdicts from a review that read nothing: %+v", len(recorded), recorded) + } + if matches, _ := filepath.Glob(filepath.Join(dir, review.ReviewReportFilePrefix+"*")); len(matches) != 0 { + t.Fatalf("unreadable review was written where the collector reads verdicts: %v", matches) + } + artifact, err := review.Collect(dir, review.AggregateOptions{}) + if err != nil { + t.Fatalf("Collect: %v", err) + } + for _, agg := range artifact.Items { + if agg.RequiresHuman || agg.Verdict == review.VerdictRequiresHuman { + t.Fatalf("unreadable review became a requires_human aggregate: %+v", agg) + } + if strings.Contains(agg.DecisionFinding, "security") { + t.Fatalf("unreadable review claimed a security finding: %q", agg.DecisionFinding) + } + } + + state, err := review.LoadDispatchState("") + if err != nil { + t.Fatalf("load dispatch state: %v", err) + } + pr := review.PullRequest{Repo: repo, Number: num, Title: "enable immutability checks", Author: "Karibhou", HeadSHA: head} + opts := review.DispatchOptions{ + RequireApproval: true, + FanOut: true, + CombinedPerspectives: true, + MaxParallelReviews: 5, + Agents: []review.AgentCapability{{Name: "reviewer", Enabled: true, UsesKick: true, Role: "reviewer"}}, + } + + // Inside the TTL the dispatch is still open: no re-kick, no person. + opts.Now = dispatched.Add(10 * time.Minute) + early := review.PlanDispatch([]review.PullRequest{pr}, artifact, state, opts) + if len(early.ReviewKicks) != 0 { + t.Fatalf("re-kicked inside the pending TTL: %+v", early.ReviewKicks) + } + if len(early.NewHuman) != 0 || len(early.State.Human) != 0 { + t.Fatalf("an unreadable review asked a person: new=%+v state=%+v", early.NewHuman, early.State.Human) + } + + // Past the TTL the planner asks again for every perspective. + opts.Now = dispatched.Add(review.DefaultPendingReviewTTL + time.Minute) + retry := review.PlanDispatch([]review.PullRequest{pr}, artifact, state, opts) + if len(retry.ReviewKicks) != 1 { + t.Fatalf("review kicks after the TTL = %d, want one retry", len(retry.ReviewKicks)) + } + if got := len(retry.ReviewKicks[0].Perspectives); got != len(review.DefaultPerspectives) { + t.Fatalf("retry asked %d perspectives, want all %d", got, len(review.DefaultPerspectives)) + } + if len(retry.NewHuman) != 0 || len(retry.State.Human) != 0 { + t.Fatalf("the retry asked a person: new=%+v state=%+v", retry.NewHuman, retry.State.Human) + } +} + +// Only the rate-limited perspectives are dropped. A perspective that did read +// the PR is recorded as usual, and the head reads as under-asked +// (incomplete), not approved: an unread correctness review must not resolve +// hands-free to approve and make the head merge-eligible. +func TestRecordReviewVerdictKeepsReadPerspectivesAndDropsRateLimitedOnes(t *testing.T) { + const ( + repo = "o/r" + num = 7 + head = "abc123" + ) + dir := t.TempDir() + c := &Client{logger: testLogger()} + withVerdictDispatchState(t, review.DispatchState{Pending: []review.PendingReview{ + {Repo: repo, Number: num, HeadSHA: head, Perspective: review.PerspectiveCorrectness, Agent: "reviewer", Dispatched: time.Now().UTC()}, + {Repo: repo, Number: num, HeadSHA: head, Perspective: review.PerspectiveStyle, Agent: "reviewer", Dispatched: time.Now().UTC()}, + }}) + raw := "[" + strings.Join([]string{ + verdictJSONWithSummary(t, repo, num, review.PerspectiveCorrectness, "requires_human", head, rateLimitedSummaries1116[review.PerspectiveCorrectness]), + verdictJSONWithSummary(t, repo, num, review.PerspectiveStyle, "approve", head, "Read the diff; naming is consistent."), + }, ",") + "]" + + recorded := c.recordReviewVerdict(context.Background(), ReviewRequest{Repo: repo, Number: num, Agent: "reviewer", Report: raw}, dir) + + if len(recorded) != 1 || recorded[0].Perspective != review.PerspectiveStyle { + t.Fatalf("recorded = %+v, want only the style verdict", recorded) + } + artifact, err := review.Collect(dir, review.AggregateOptions{}) + if err != nil { + t.Fatalf("Collect: %v", err) + } + if len(artifact.Items) != 1 { + t.Fatalf("want one aggregate, got %+v", artifact.Items) + } + got := artifact.Items[0] + if _, ok := got.Perspectives[review.PerspectiveCorrectness]; ok { + t.Fatalf("unread correctness perspective was counted: %v", got.Perspectives) + } + if got.Verdict != review.VerdictIncomplete || got.RequiresHuman || got.MergeEligible { + t.Fatalf("aggregate = %s (requires_human=%t merge_eligible=%t), want incomplete", got.Verdict, got.RequiresHuman, got.MergeEligible) + } +} diff --git a/src/pkg/review/unreadable.go b/src/pkg/review/unreadable.go new file mode 100644 index 0000000000..9d206ac779 --- /dev/null +++ b/src/pkg/review/unreadable.go @@ -0,0 +1,80 @@ +package review + +import ( + "regexp" + "strings" +) + +// A reviewer that could not read the PR is told to say so with +// requires_human (buildReadInstruction) rather than guess. That is the right +// answer when the diff is genuinely unreviewable — too large, masked — and a +// person has to look. It is the wrong answer when the cause is the hive's own +// plumbing: the agent's GitHub API cap (agents.github_api_hourly_cap, the +// proxy's 429) or GitHub's rate limit. Then nothing about the PR needs a +// person; the read has to be tried again once the limit resets. +// +// Recorded as a verdict, that report did real damage +// (projectbluefin/testsuite#1116, 2026-11-07): five perspectives each wrote +// "requires human: could not read the delta: every GitHub read returned HTTP +// 429 … Not reviewed". The security one counted as "the security review +// withheld approval" (IsSecuritySensitiveFinding), the PR parked at +// stage/085-needs-decision, and the owner's delegate asked a person to rule +// on a security finding that did not exist. On a contributor PR without the +// security one, the same requires_human would have resolved hands-free to +// approve — an unread head marked merge-eligible. +// +// UnreadableReviewReason recognises such a report, so the relay can decline +// to record it as a verdict: the dispatch row it answered stays open, +// expires after review.pending_ttl, and the planner asks again. + +// unreadableReviewTerms say the reviewer did not read the change. +var unreadableReviewTerms = []string{ + "could not read", "couldn't read", "cannot read", "can't read", "unable to read", + "not readable", "unreadable", "could not fetch", "couldn't fetch", "unable to fetch", + "could not load", "could not access", "unable to access", "not reviewed", + "could not review", "unable to review", "could not be read", +} + +// rateLimitCauseRe names a rate limit as the reason: an HTTP 429, GitHub's +// rate limit (primary or secondary), or the hive proxy's per-agent hourly +// API cap. "429" is matched as a whole number so a line number or count that +// happens to contain it does not. +var rateLimitCauseRe = regexp.MustCompile(`(?i)(?:\b429\b|too many requests|rate[- ]?limit|hourly cap|api cap|api (?:hourly )?budget|quota exhausted|quota exceeded)`) + +// UnreadableReviewReason reports whether r is a requires_human that only +// says the reviewer could not read the PR because of a rate limit, and +// returns the reviewer's own summary as the reason. +// +// It is deliberately narrow. The report must: +// - be requires_human (an approve or changes_requested is a judgement); +// - not carry the reviewer's explicit high-risk marker; +// - not ask a decision question (the needs-decision marker): a reviewer +// that asks the owner something read enough to ask it, even when the +// PR it read is about rate limits; +// - raise no in-scope finding (a finding means it read something); +// - say, in its summary or decision question, both that it could not read +// the change and that a rate limit was the cause. +// +// A requires_human for a diff that is too large, or for any other reason +// than a rate limit, still parks the PR as before. +func UnreadableReviewReason(r PerspectiveReport) (string, bool) { + if r.Verdict != VerdictRequiresHuman || r.DecisionRiskHigh || r.DecisionRequested { + return "", false + } + for _, f := range r.Findings { + if InScopeFinding(f) { + return "", false + } + } + text := strings.TrimSpace(r.Summary + "\n" + r.DecisionQuestion) + if text == "" || !rateLimitCauseRe.MatchString(text) { + return "", false + } + lower := strings.ToLower(text) + for _, term := range unreadableReviewTerms { + if strings.Contains(lower, term) { + return strings.TrimSpace(r.Summary), true + } + } + return "", false +} diff --git a/src/pkg/review/unreadable_test.go b/src/pkg/review/unreadable_test.go new file mode 100644 index 0000000000..9636823193 --- /dev/null +++ b/src/pkg/review/unreadable_test.go @@ -0,0 +1,96 @@ +package review + +import ( + "testing" + + "github.com/hivecommons/hive/pkg/outputschema" +) + +// The five summaries the reviewer wrote on projectbluefin/testsuite#1116 at +// head ae22434 (2026-11-07), when every GitHub read hit the agent API cap. +var testsuite1116Summaries = map[Perspective]string{ + PerspectiveCorrectness: "Could not read the delta d397ece..ae22434: every GitHub read returned HTTP 429 (hive agent API hourly cap, 181-211/400) and no local checkout of projectbluefin/testsuite exists on this host. Not reviewed; previous correctness verdict (approve) was for head d397ece only.", + PerspectiveDocsCurrency: "Delta touches docs/skills/test-authoring/suite-map/SKILL.md but could not be read (GitHub API 429 cap, no local checkout); could not confirm whether prior finding SKILL.md:170 (immutability checks still marked @pending) is fixed. Status unknown.", + PerspectiveIntentAlignment: "Delta not readable (GitHub API 429 cap, no local checkout); could not re-check prior finding tests/common/features/common_immutable.feature:13 ('Closes #838' may over-claim scope) nor read issue #838. Status of prior finding unknown.", + PerspectiveSecurity: "Delta not readable (GitHub API 429 cap, no local checkout). Not reviewed at this head.", + PerspectiveStyle: "Delta not readable (GitHub API 429 cap, no local checkout). Not reviewed at this head.", +} + +func TestUnreadableReviewReasonRecognisesRateLimitedReviews(t *testing.T) { + for p, summary := range testsuite1116Summaries { + r := PerspectiveReport{Perspective: p, Verdict: VerdictRequiresHuman} + r.Summary = summary + reason, ok := UnreadableReviewReason(r) + if !ok { + t.Errorf("%s: rate-limited unreadable review not recognised: %q", p, summary) + continue + } + if reason != summary { + t.Errorf("%s: reason = %q, want the reviewer's summary", p, reason) + } + } + others := []string{ + "Could not fetch the diff: gh returned 429 Too Many Requests.", + "Unable to read the PR files: API rate limit exceeded for installation.", + "I cannot read the diff; hive: agent GitHub API hourly cap reached (400/400).", + "Could not read the diff: secondary rate-limit hit on every request.", + } + for _, summary := range others { + r := PerspectiveReport{Perspective: PerspectiveCorrectness, Verdict: VerdictRequiresHuman} + r.Summary = summary + if _, ok := UnreadableReviewReason(r); !ok { + t.Errorf("not recognised: %q", summary) + } + } +} + +func TestUnreadableReviewReasonLeavesRealHoldsAlone(t *testing.T) { + unread := testsuite1116Summaries[PerspectiveSecurity] + cases := []struct { + name string + report PerspectiveReport + }{ + {"approve is a judgement", unreadableTestReport(PerspectiveSecurity, VerdictApprove, unread, nil)}, + {"changes_requested is a judgement", unreadableTestReport(PerspectiveSecurity, VerdictChangesRequested, unread, nil)}, + {"too large is a real hold", unreadableTestReport(PerspectiveCorrectness, VerdictRequiresHuman, "Could not read the diff: it is too large (12k lines). A person needs to review it.", nil)}, + {"rate limit without an unread claim", unreadableTestReport(PerspectiveSecurity, VerdictRequiresHuman, "The new client retries a 429 forever without backoff; a person should decide the rate-limit policy.", nil)}, + {"security finding stays a security finding", unreadableTestReport(PerspectiveSecurity, VerdictRequiresHuman, "Token is written to the log.", nil)}, + {"an in-scope finding means it read something", unreadableTestReport(PerspectiveSecurity, VerdictRequiresHuman, unread, + []outputschema.Finding{{Title: "secret leaks into logs", Severity: outputschema.SeverityHigh, File: "a.go", Line: 3}})}, + {"empty summary", unreadableTestReport(PerspectiveSecurity, VerdictRequiresHuman, "", nil)}, + } + for _, tc := range cases { + if reason, ok := UnreadableReviewReason(tc.report); ok { + t.Errorf("%s: recognised as unreadable (%q), want a judgement kept", tc.name, reason) + } + } + marked := unreadableTestReport(PerspectiveSecurity, VerdictRequiresHuman, unread, nil) + marked.DecisionRiskHigh = true + if _, ok := UnreadableReviewReason(marked); ok { + t.Error("the reviewer's explicit high-risk marker was overridden") + } + // hivecommons/hive#16994 review: a reviewer that read a PR about rate + // limiting and asks the owner a question uses both kinds of words. + asked := unreadableTestReport(PerspectiveCorrectness, VerdictRequiresHuman, + "I could not review the retry budget against production numbers; @owner should decide whether the hourly cap may be bypassed here.", nil) + if _, ok := UnreadableReviewReason(asked); !ok { + t.Fatal("test premise: without the decision marker this wording reads as unreadable") + } + asked.DecisionRequested = true + asked.DecisionQuestion = "May the hourly cap be bypassed for this retry path?" + if _, ok := UnreadableReviewReason(asked); ok { + t.Error("an explicit decision question was dropped as an unreadable review") + } + outOfScope := unreadableTestReport(PerspectiveSecurity, VerdictRequiresHuman, unread, + []outputschema.Finding{{Title: "pre-existing", Severity: outputschema.SeverityLow, ReviewScope: "out-of-scope"}}) + if _, ok := UnreadableReviewReason(outOfScope); !ok { + t.Error("an out-of-scope note kept a rate-limited unread review as a verdict") + } +} + +func unreadableTestReport(p Perspective, v Verdict, summary string, findings []outputschema.Finding) PerspectiveReport { + r := PerspectiveReport{Perspective: p, Verdict: v} + r.Summary = summary + r.Findings = findings + return r +}