diff --git a/changelog.d/fixed-review-unreadable-stored-verdict.md b/changelog.d/fixed-review-unreadable-stored-verdict.md new file mode 100644 index 0000000000..e89d97907a --- /dev/null +++ b/changelog.d/fixed-review-unreadable-stored-verdict.md @@ -0,0 +1 @@ +- A `requires_human` verdict already stored from rate-limited "could not read … HTTP 429" reviews no longer keeps a PR parked. The verdict collect now sets those report files aside as missing, the way the relay already skips new ones, and withdraws a stored `requires_human` that only they produced. The head reads `incomplete`, and the planner asks the missing perspectives again under `review.pending_ttl` and `max_redispatch`. A head with a set-aside security review is never resolved hands-free to approve while that review is owed. A real hold still parks the PR: a diff too large to read, a security finding, a decision question or the high-risk marker. Before, the pre-upgrade files on projectbluefin/testsuite#1116 kept head `ae22434` at `stage/085-needs-decision` hours after the relay fix was live. diff --git a/src/cmd/hive/main_sweeps.go b/src/cmd/hive/main_sweeps.go index 9bf87b3cc9..b77563f449 100644 --- a/src/cmd/hive/main_sweeps.go +++ b/src/cmd/hive/main_sweeps.go @@ -2859,7 +2859,8 @@ func refreshReviewVerdictsForPRs(cfg *config.Config, prs []review.PullRequest, l } logger.Info("review verdict artifact refreshed", "aggregates", len(artifact.Items), "elapsed", time.Since(started), "report_files_read", artifact.Collection.FilesRead, - "new_report_receipts", artifact.Collection.NewReports, "max_new_report_lag", artifact.Collection.MaxReportLag) + "new_report_receipts", artifact.Collection.NewReports, "max_new_report_lag", artifact.Collection.MaxReportLag, + "unreadable_reports_skipped", artifact.Collection.UnreadableSkipped) } // reviewModelStamps reads the dispatch state so a verdict whose report diff --git a/src/docs/review-swarm.md b/src/docs/review-swarm.md index 4dd1411121..b3f1466e80 100644 --- a/src/docs/review-swarm.md +++ b/src/docs/review-swarm.md @@ -97,7 +97,7 @@ Aggregation rules are deterministic: 3. Any explicit `requires_human` yields `requires_human`. 4. Any `changes_requested` enters a fix cycle while below the fix cap. 5. Every perspective in the hive's configured set approving (or, under `max_perspectives_per_pr`, every perspective the PR was eligible to receive) yields a merge-eligible aggregate. -6. Every delivered perspective approving while some the PR is owed have not reported yields `incomplete`: not merge-eligible, not a human hold, with a reason naming the missing perspectives (`review coverage incomplete: missing docs-currency`). An incomplete head stays in the dispatcher: only the missing perspectives are re-asked (under `pending_ttl` / `max_redispatch`), never the ones whose reports already exist. The exception is a head that already holds a settled non-approve verdict (`requires_human`, `changes_requested` or `reject`): an incomplete re-review of the same head does not replace it. The artifact keeps the prior decision, its findings or question and its `recorded_at`, so the dispatcher never sees `incomplete` for that head — a `requires_human` head keeps taking the human-hold path and stays at 085, and a `changes_requested` or `reject` head stays on the fix path (rather than dropping back to 050 to re-ask the missing perspectives) until a complete replacement review lands. When the stale entries expire under `pending_ttl`, the whole perspective set is asked again for that head, not just the missing ones (hivecommons/hive#12878). Before this state existed a reviewer kick that died after its peers had reported left the head on a `requires_human` hold that no re-dispatch could clear (hivecommons/hive#12045). +6. Every delivered perspective approving while some the PR is owed have not reported yields `incomplete`: not merge-eligible, not a human hold, with a reason naming the missing perspectives (`review coverage incomplete: missing docs-currency`). An incomplete head stays in the dispatcher: only the missing perspectives are re-asked (under `pending_ttl` / `max_redispatch`), never the ones whose reports already exist. The exception is a head that already holds a settled non-approve verdict (`requires_human`, `changes_requested` or `reject`): an incomplete re-review of the same head does not replace it. The artifact keeps the prior decision, its findings or question and its `recorded_at`, so the dispatcher never sees `incomplete` for that head — a `requires_human` head keeps taking the human-hold path and stays at 085, and a `changes_requested` or `reject` head stays on the fix path (rather than dropping back to 050 to re-ask the missing perspectives) until a complete replacement review lands. When the stale entries expire under `pending_ttl`, the whole perspective set is asked again for that head, not just the missing ones (hivecommons/hive#12878). Before this state existed a reviewer kick that died after its peers had reported left the head on a `requires_human` hold that no re-dispatch could clear (hivecommons/hive#12045). The one stored `requires_human` an incomplete collect does withdraw is one that rate-limited "could not read" reports alone produced (see *A review the reviewer could not read because of a rate limit is no verdict* below). 7. Any other non-unanimous result requires human review. One exception to rules 2–3: on a **hive superseding PR** (the App's own PR declaring `Supersedes #N`, `PerspectiveReport.hive_superseding`) a `requires_human` — security-sensitive included — becomes `changes_requested` and enters the fix cycle, because the owner's rule is that such PRs auto-merge with no operator decision; only the fix cap, a `reject`, or the stage engine's hard stops (sentinel, hold, red CI, unresolvable conflict) hand it to a person. `auto_merge.hive_superseding.auto_merge: false` turns this off. See [hive-merge.md](hive-merge.md#hive-authored-superseding-prs-auto-merge). @@ -585,6 +585,8 @@ Expiry also covers a head whose aggregate is `incomplete`: the delivered perspec **A review the reviewer could not read because of a rate limit is no verdict.** The read instruction tells a reviewer that could not read the diff to return `requires_human` and say so. When the only reason is a rate limit — an HTTP 429, GitHub's rate limit, or the hive proxy's per-agent cap (`agents.github_api_hourly_cap`) — the relay does not record that report (`review.UnreadableReviewReason`, `src/pkg/review/unreadable.go`). It must be `requires_human`, carry no in-scope finding, no `hive-decision-risk: high` marker and no explicit decision question (``, since a reviewer that asks the owner something read enough to ask it), and its summary must say both that the change was not read and that a rate limit was the cause. The perspective's pending entry stays open, expires after `pending_ttl` like any kick that posted nothing, and is asked again, so no person is asked and no security hold is claimed. Other perspectives in the same combined review that did read the PR are recorded as usual, and the head reads as `incomplete` until the re-asked ones report. The review-request watcher logs `unreadable review not recorded as a verdict`. Any review comment the reviewer already posted saying it could not read the PR stays on the PR with no verdict behind it, and the re-asked reviewer may post another. A `requires_human` for any other reason, such as a diff too large to read, still parks the PR. Before this, the five `could not read … HTTP 429` reports on projectbluefin/testsuite#1116 parked it at 085 as "the security review withheld approval". +**The same rule applies to report files already on disk.** A report written before the relay learned to skip these is still read by every verdict collect, so the collect applies the same check (`UnreadableReviewReason`) and sets such a file aside as missing; the refresh log line counts them as `unreadable_reports_skipped`. A stored `requires_human` for that head is withdrawn when those set-aside reports alone produced it: every perspective it holds at `requires_human`, and every human summary and security hold it quotes, must come from one of them, and it must carry no decision question, no in-scope finding and no exhausted fix cycle. The head then reads `incomplete` (or has no verdict yet), and the planner asks the missing perspectives again on the normal path, under `pending_ttl` and `max_redispatch`, not on every sweep. While a set-aside perspective is owed, the head is never `approve`: the contributor rule's hands-free approve waits for full coverage instead. A stored hold that anything else backs, including a real report no longer on disk, stays as it is (rule 6 above). On projectbluefin/testsuite#1116 the five pre-upgrade files kept the head at 085 hours after the upgrade until this check. + The PR review queue (`GET /api/review/queue`) shows this state on unreviewed items: `pending_since` (first dispatch for the current head still pending) and `redispatch_count`, plus a reason such as `review dispatched 32m ago, no verdict yet (re-dispatched 2x after expiry)` or `review dispatch exhausted: …; waiting on a human` after the generic `not reviewed at this head` line. When an aggregate verdict is `changes_requested`, Hive builds a review-fix kick containing the aggregate findings and sends it to `review.fixer_agent`, the classified PR lane, or `scanner`. diff --git a/src/pkg/review/review.go b/src/pkg/review/review.go index ae9a21ba91..72cdaab219 100644 --- a/src/pkg/review/review.go +++ b/src/pkg/review/review.go @@ -365,6 +365,26 @@ type CollectionStats struct { FilesRead int NewReports int MaxReportLag time.Duration + // UnreadableSkipped counts report files set aside because they only + // say a rate limit kept the reviewer from reading the PR + // (UnreadableReviewReason). They are not verdicts. + UnreadableSkipped int + // unreadable are those reports, kept so the merge can withdraw a stored + // verdict that rested on them alone. + unreadable []unreadableReport +} + +// unreadableReport is a report file the collect set aside as no verdict. +type unreadableReport struct { + Repo string + Number int + HeadSHA string + Perspective Perspective + // HumanSummary is how an aggregate built from the report quotes it + // (humanSummaryReason); Finding is the security hold it would claim + // (IsSecuritySensitiveFinding), if any. + HumanSummary string + Finding string } type Artifact struct { @@ -1454,6 +1474,19 @@ func collectForPRs(dir string, opts AggregateOptions, prs []PullRequest) (Artifa if prs != nil && !targets[strings.ToLower(fmt.Sprintf("%s#%d", report.Repo, report.Number))] { continue } + if _, unread := UnreadableReviewReason(*report); unread { + // A rate limit kept this reviewer from reading the PR: that is + // no verdict. The relay no longer records such a report, but + // files written before it learned to skip them are still on + // disk, and read as a verdict they park the head at 085 + // (projectbluefin/testsuite#1116). Set it aside as missing, so + // the head reads incomplete and the planner asks again. + stats.UnreadableSkipped++ + stats.unreadable = append(stats.unreadable, unreadableReport{Repo: report.Repo, Number: report.Number, + HeadSHA: report.HeadSHA, Perspective: report.Perspective, HumanSummary: humanSummaryReason(*report), + Finding: unreadableFinding(*report, opts.SensitivePaths)}) + continue + } opts.ModelStamps.apply(report) key := reviewKey(report.Repo, report.Number, report.HeadSHA, report.BodyDigest) groups[key] = append(groups[key], *report) @@ -1465,6 +1498,18 @@ func collectForPRs(dir string, opts AggregateOptions, prs []PullRequest) (Artifa artifact := Artifact{GeneratedAt: time.Now().UTC(), Collection: stats} for _, key := range sortedKeys(groups) { agg := AggregateReports(groups[key], opts) + if agg.Verdict == VerdictApprove && len(unreadableForHead(stats.unreadable, agg.Repo, agg.Number, agg.HeadSHA)) > 0 && + !hasAllRequiredPerspectives(agg.Perspectives, opts.MaxPerspectivesPerPR, opts.Perspectives.Len()) { + // A perspective set aside as unread is owed, not approved. The + // contributor rule resolves a requires_human with no security + // finding to approve whatever the coverage; with the security + // review the one that could not read, that would mark an + // unread head merge-eligible. Wait for the re-ask instead. + agg.Verdict = VerdictIncomplete + agg.MergeEligible = false + agg.Reasons = append(agg.Reasons, incompleteCoverageReason(agg.Perspectives, opts)) + sortReasons(agg.Reasons) + } agg.ReportReceipts = receipts[key] artifact.Items = append(artifact.Items, agg) } @@ -1630,6 +1675,22 @@ func mergeCollected(fresh Artifact, dir, path string, now time.Time) (Artifact, } } + if len(fresh.Collection.unreadable) > 0 { + kept := order[:0] + for _, key := range order { + if prior := merged[key]; !prior.RecordedAt.After(now) && verdictRestsOnUnreadable(prior, fresh.Collection.unreadable) { + // The stored requires_human came only from reports that + // were never verdicts. Withdraw it, so the fresh collect + // (incomplete, or nothing yet) stands for this head and + // the missing perspectives are asked again. + delete(merged, key) + continue + } + kept = append(kept, key) + } + order = kept + } + for _, item := range fresh.Items { item.RecordedAt = now item.FirstRecordedAt = now @@ -1690,6 +1751,70 @@ func mergeCollected(fresh Artifact, dir, path string, now time.Time) (Artifact, return artifact, nil } +// verdictRestsOnUnreadable reports whether a stored aggregate is a +// requires_human that only the unreadable reports set aside by this collect +// produced: every perspective it holds at requires_human, and every human +// summary it carries, belongs to one of them. Anything else a person may +// need to see keeps it: a decision question, an in-scope finding, a fix +// cycle that ran out, or a DecisionFinding that is not one of those reports' +// own (on #1116 it was "the security review withheld approval: Delta not +// readable (GitHub API 429 cap …)", the unread security report). So does any requires_human perspective or summary the unreadable +// reports do not account for, such as one from a report no longer on disk. +func verdictRestsOnUnreadable(prior Aggregate, unread []unreadableReport) bool { + if prior.Verdict != VerdictRequiresHuman || prior.DecisionRequested || FixCycleCapReached(prior) || len(prior.HumanSummaries) == 0 { + return false + } + for _, f := range prior.Findings { + if InScopeFinding(f.Finding) { + return false + } + } + perspectives := map[Perspective]bool{} + summaries := map[string]bool{} + finding := strings.TrimSpace(prior.DecisionFinding) + findingExplained := finding == "" + for _, u := range unreadableForHead(unread, prior.Repo, prior.Number, prior.HeadSHA) { + perspectives[u.Perspective] = true + summaries[u.HumanSummary] = true + if u.Finding != "" && u.Finding == finding { + findingExplained = true + } + } + if len(perspectives) == 0 || !findingExplained { + return false + } + for p, v := range prior.Perspectives { + if v == VerdictRequiresHuman && !perspectives[p] { + return false + } + } + for _, s := range prior.HumanSummaries { + if !summaries[s] { + return false + } + } + return true +} + +// unreadableFinding is the security hold an aggregate would have claimed from +// r: the security perspective withholding approval reads as a finding, +// whatever it said. +func unreadableFinding(r PerspectiveReport, sensitivePaths []string) string { + finding, _ := IsSecuritySensitiveFinding(r, SecuritySensitivePaths(sensitivePaths)) + return strings.TrimSpace(finding) +} + +// unreadableForHead returns the set-aside reports for one PR head. +func unreadableForHead(unread []unreadableReport, repo string, number int, headSHA string) []unreadableReport { + var out []unreadableReport + for _, u := range unread { + if u.Number == number && strings.EqualFold(strings.TrimSpace(u.Repo), strings.TrimSpace(repo)) && SameHeadSHA(u.HeadSHA, headSHA) { + out = append(out, u) + } + } + return out +} + // PruneReports removes review-report-*.json files in dir whose modification // time is older than olderThan. Reports live on the durable data dir now // (DefaultReportDir), so without this they would accumulate for the life of diff --git a/src/pkg/review/unreadable_stored_test.go b/src/pkg/review/unreadable_stored_test.go new file mode 100644 index 0000000000..469f06d1c8 --- /dev/null +++ b/src/pkg/review/unreadable_stored_test.go @@ -0,0 +1,325 @@ +package review + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/outputschema" +) + +// Report files written before the relay learned to skip rate-limited +// "could not read" reports (#16994) are still on disk on a live hive. Read as +// verdicts they keep the head parked: projectbluefin/testsuite#1116 at head +// ae22434 still read requires_human → 085 hours after the upgrade. The collect +// treats them as missing, and the merge withdraws the requires_human they +// alone produced, so the head is re-asked through the normal pending path. + +const unreadableStoredRepo = "projectbluefin/testsuite" +const unreadableStoredNumber = 1116 +const unreadableStoredHead = "ae22434cfe05ce9c119f2e508634d599201ab327" + +func writeStoredReport(t *testing.T, dir string, r PerspectiveReport) { + t.Helper() + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + raw, err := json.Marshal(r) + if err != nil { + t.Fatal(err) + } + name := ReportFilePrefix(r.Repo, r.Number) + string(r.Perspective) + ReviewReportFileSuffix + if err := os.WriteFile(filepath.Join(dir, name), raw, 0o600); err != nil { + t.Fatal(err) + } +} + +func storedReport(p Perspective, v Verdict, summary string, findings ...outputschema.Finding) PerspectiveReport { + r := baseReport(p, v, findings...) + r.Repo, r.Number, r.HeadSHA = unreadableStoredRepo, unreadableStoredNumber, unreadableStoredHead + r.Summary = summary + return r +} + +// seedPreUpgradeVerdict writes the reports and builds the artifact the way the +// hive did before #16994: every report is aggregated, so the stored verdict +// for the head is requires_human. The pre-upgrade artifact is reproduced by +// aggregating the reports directly, since the current collect skips them. +func seedPreUpgradeVerdict(t *testing.T, reportDir, artifactPath string, reports []PerspectiveReport, at time.Time) { + t.Helper() + for _, r := range reports { + writeStoredReport(t, reportDir, r) + } + agg := AggregateReports(reports, AggregateOptions{}) + if agg.Verdict != VerdictRequiresHuman { + t.Fatalf("seed: pre-upgrade aggregate = %s, want requires_human (%v)", agg.Verdict, agg.Reasons) + } + agg.RecordedAt, agg.FirstRecordedAt = at, at + if err := WriteArtifact(artifactPath, Artifact{GeneratedAt: at, Items: []Aggregate{agg}}); err != nil { + t.Fatal(err) + } +} + +func testsuite1116Reports() []PerspectiveReport { + var reports []PerspectiveReport + for _, p := range DefaultPerspectives { + reports = append(reports, storedReport(p, VerdictRequiresHuman, testsuite1116Summaries[p])) + } + return reports +} + +func storedAggregateFor(a Artifact) (Aggregate, bool) { + return a.AggregateFor(unreadableStoredRepo, unreadableStoredNumber, unreadableStoredHead) +} + +func unreadableStoredPR() PullRequest { + return PullRequest{Repo: unreadableStoredRepo, Number: unreadableStoredNumber, HeadSHA: unreadableStoredHead} +} + +func unreadableStoredDispatchOpts(now time.Time) DispatchOptions { + return DispatchOptions{ + RequireApproval: true, FanOut: true, CombinedPerspectives: true, Now: now, + Agents: []AgentCapability{reviewer("reviewer")}, + } +} + +func TestStoredRateLimitedRequiresHumanIsWithdrawnAndReasked(t *testing.T) { + root := t.TempDir() + reportDir := filepath.Join(root, "reports") + artifactPath := filepath.Join(root, "data", ReviewVerdictsFile) + recorded := time.Date(2026, 11, 7, 10, 59, 0, 0, time.UTC) + now := recorded.Add(8 * time.Hour) + seedPreUpgradeVerdict(t, reportDir, artifactPath, testsuite1116Reports(), recorded) + + artifact, err := CollectAndMerge(reportDir, artifactPath, AggregateOptions{}, now) + if err != nil { + t.Fatal(err) + } + if artifact.Collection.UnreadableSkipped != len(DefaultPerspectives) { + t.Errorf("UnreadableSkipped = %d, want %d", artifact.Collection.UnreadableSkipped, len(DefaultPerspectives)) + } + if agg, ok := storedAggregateFor(artifact); ok { + t.Fatalf("stored rate-limited requires_human still stands for the head: %s %v", agg.Verdict, agg.Reasons) + } + if hold, held := artifact.VerdictHoldFor(unreadableStoredRepo, unreadableStoredNumber); held { + t.Fatalf("head still held by the stored verdict: %+v", hold) + } + onDisk, err := LoadArtifact(artifactPath) + if err != nil { + t.Fatal(err) + } + if _, ok := storedAggregateFor(onDisk); ok { + t.Fatal("withdrawn verdict was written back to review-verdicts.json") + } + + // The planner holds nothing for a person and asks every perspective again. + plan := PlanDispatch([]PullRequest{unreadableStoredPR()}, artifact, DispatchState{}, unreadableStoredDispatchOpts(now)) + if len(plan.NewHuman) != 0 || len(plan.State.Human) != 0 { + t.Fatalf("planner parked the head for a person: %+v", plan.State.Human) + } + if len(plan.ReviewKicks) != 1 || len(plan.ReviewKicks[0].Perspectives) != len(DefaultPerspectives) { + t.Fatalf("want one combined re-review of every perspective, got %+v", plan.ReviewKicks) + } + + // No re-dispatch loop: the next sweep, still inside pending_ttl, kicks + // nothing; once the TTL passes the head is re-asked once and counted. + again := PlanDispatch([]PullRequest{unreadableStoredPR()}, artifact, plan.State, unreadableStoredDispatchOpts(now.Add(time.Minute))) + if len(again.ReviewKicks) != 0 { + t.Fatalf("head re-kicked inside pending_ttl: %+v", again.ReviewKicks) + } + artifact, err = CollectAndMerge(reportDir, artifactPath, AggregateOptions{}, now.Add(DefaultPendingReviewTTL+time.Minute)) + if err != nil { + t.Fatal(err) + } + expired := PlanDispatch([]PullRequest{unreadableStoredPR()}, artifact, again.State, unreadableStoredDispatchOpts(now.Add(DefaultPendingReviewTTL+time.Minute))) + if len(expired.ReviewKicks) != 1 { + t.Fatalf("want one re-ask after pending_ttl, got %+v", expired.ReviewKicks) + } + if got := expired.State.RedispatchCountFor(unreadableStoredRepo, unreadableStoredNumber, unreadableStoredHead); got != 1 { + t.Fatalf("redispatch count = %d, want 1 (the max_redispatch budget applies)", got) + } +} + +func TestStoredPartlyUnreadableHeadReadsIncomplete(t *testing.T) { + // Some perspectives read the PR and approved; the others hit the cap. + // The head is under-asked, not a human decision. + root := t.TempDir() + reportDir := filepath.Join(root, "reports") + artifactPath := filepath.Join(root, "data", ReviewVerdictsFile) + recorded := time.Date(2026, 11, 7, 10, 59, 0, 0, time.UTC) + now := recorded.Add(time.Hour) + reports := []PerspectiveReport{ + storedReport(PerspectiveCorrectness, VerdictApprove, "Reads correctly."), + storedReport(PerspectiveStyle, VerdictApprove, "Fine."), + storedReport(PerspectiveSecurity, VerdictRequiresHuman, testsuite1116Summaries[PerspectiveSecurity]), + storedReport(PerspectiveIntentAlignment, VerdictRequiresHuman, testsuite1116Summaries[PerspectiveIntentAlignment]), + storedReport(PerspectiveDocsCurrency, VerdictRequiresHuman, testsuite1116Summaries[PerspectiveDocsCurrency]), + } + seedPreUpgradeVerdict(t, reportDir, artifactPath, reports, recorded) + + artifact, err := CollectAndMerge(reportDir, artifactPath, AggregateOptions{}, now) + if err != nil { + t.Fatal(err) + } + agg, ok := storedAggregateFor(artifact) + if !ok || agg.Verdict != VerdictIncomplete { + t.Fatalf("want the head incomplete, got ok=%v %s %v", ok, agg.Verdict, agg.Reasons) + } + plan := PlanDispatch([]PullRequest{unreadableStoredPR()}, artifact, DispatchState{}, unreadableStoredDispatchOpts(now)) + if len(plan.State.Human) != 0 { + t.Fatalf("planner parked the head for a person: %+v", plan.State.Human) + } + if len(plan.ReviewKicks) != 1 { + t.Fatalf("want one re-review kick, got %+v", plan.ReviewKicks) + } + got := map[Perspective]bool{} + for _, p := range plan.ReviewKicks[0].Perspectives { + got[p] = true + } + if len(got) != 3 || !got[PerspectiveSecurity] || !got[PerspectiveIntentAlignment] || !got[PerspectiveDocsCurrency] { + t.Fatalf("want only the unread perspectives re-asked, got %v", plan.ReviewKicks[0].Perspectives) + } +} + +var leakFinding = outputschema.Finding{Title: "secret leaks into logs", Summary: "The token is logged at info level.", + Severity: outputschema.SeverityHigh, File: "a.go", Line: 3} + +func TestStoredRealRequiresHumanStillParks(t *testing.T) { + unread := testsuite1116Summaries[PerspectiveSecurity] + cases := []struct { + name string + report PerspectiveReport + }{ + {"security review: diff too large", storedReport(PerspectiveSecurity, VerdictRequiresHuman, + "Could not read the diff: it is too large (12k lines). A person needs to review it.")}, + {"hive-authored PR: diff too large", func() PerspectiveReport { + r := storedReport(PerspectiveCorrectness, VerdictRequiresHuman, + "Could not read the diff: it is too large (12k lines). A person needs to review it.") + r.HiveAuthored = true + return r + }()}, + {"security finding", storedReport(PerspectiveSecurity, VerdictRequiresHuman, + "Token is written to the log; a person must decide.", leakFinding)}, + {"unread text but an in-scope finding", storedReport(PerspectiveSecurity, VerdictRequiresHuman, unread, leakFinding)}, + {"unread text but a decision question", func() PerspectiveReport { + r := storedReport(PerspectiveSecurity, VerdictRequiresHuman, unread) + r.DecisionRequested, r.DecisionQuestion = true, "Should the retry policy back off on HTTP 429?" + return r + }()}, + {"unread text but the high-risk marker", func() PerspectiveReport { + r := storedReport(PerspectiveSecurity, VerdictRequiresHuman, unread) + r.DecisionRiskHigh = true + return r + }()}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + root := t.TempDir() + reportDir := filepath.Join(root, "reports") + artifactPath := filepath.Join(root, "data", ReviewVerdictsFile) + recorded := time.Date(2026, 11, 7, 10, 59, 0, 0, time.UTC) + now := recorded.Add(8 * time.Hour) + // The other perspectives hit the rate limit; the one under test + // is a real hold and must keep the head parked. + reports := []PerspectiveReport{tc.report} + for _, p := range DefaultPerspectives { + if p != tc.report.Perspective { + r := storedReport(p, VerdictRequiresHuman, testsuite1116Summaries[p]) + r.HiveAuthored = tc.report.HiveAuthored + reports = append(reports, r) + } + } + seedPreUpgradeVerdict(t, reportDir, artifactPath, reports, recorded) + + artifact, err := CollectAndMerge(reportDir, artifactPath, AggregateOptions{}, now) + if err != nil { + t.Fatal(err) + } + agg, ok := storedAggregateFor(artifact) + if !ok || agg.Verdict != VerdictRequiresHuman { + t.Fatalf("real hold lost: ok=%v verdict=%s reasons=%v", ok, agg.Verdict, agg.Reasons) + } + plan := PlanDispatch([]PullRequest{unreadableStoredPR()}, artifact, DispatchState{}, unreadableStoredDispatchOpts(now)) + if len(plan.State.Human) == 0 { + t.Fatal("real requires_human no longer parks the head for a person") + } + if len(plan.ReviewKicks) != 0 { + t.Fatalf("settled head re-asked: %+v", plan.ReviewKicks) + } + }) + } +} + +func TestStoredRequiresHumanFromLostReportIsKept(t *testing.T) { + // A stored requires_human whose own report is gone from disk is not + // withdrawn just because an unreadable report for another perspective + // of the same head is still there: the unreadable one cannot account + // for it. + root := t.TempDir() + reportDir := filepath.Join(root, "reports") + artifactPath := filepath.Join(root, "data", ReviewVerdictsFile) + recorded := time.Date(2026, 11, 7, 10, 59, 0, 0, time.UTC) + real := storedReport(PerspectiveCorrectness, VerdictRequiresHuman, "Could not read the diff: it is too large (12k lines).") + unread := storedReport(PerspectiveSecurity, VerdictRequiresHuman, testsuite1116Summaries[PerspectiveSecurity]) + seedPreUpgradeVerdict(t, reportDir, artifactPath, []PerspectiveReport{real, unread}, recorded) + name := ReportFilePrefix(real.Repo, real.Number) + string(real.Perspective) + ReviewReportFileSuffix + if err := os.Remove(filepath.Join(reportDir, name)); err != nil { + t.Fatal(err) + } + + artifact, err := CollectAndMerge(reportDir, artifactPath, AggregateOptions{}, recorded.Add(time.Hour)) + if err != nil { + t.Fatal(err) + } + agg, ok := storedAggregateFor(artifact) + if !ok || agg.Verdict != VerdictRequiresHuman { + t.Fatalf("stored hold from a lost report withdrawn: ok=%v %s", ok, agg.Verdict) + } + if !strings.Contains(strings.Join(agg.HumanSummaries, "\n"), "too large") { + t.Fatalf("kept the wrong verdict: %v", agg.HumanSummaries) + } +} + +func TestStoredUnreadSecurityDoesNotLetContributorHoldResolveToApprove(t *testing.T) { + // The contributor rule resolves a requires_human with no security + // finding behind it to approve. With the security review the one set + // aside as unread, that must not mark the head merge-eligible: the + // security perspective is owed, so the head waits for the re-ask. + // (With a stored requires_human from before the upgrade on the head, the + // stored hold stands: the correctness report is a real one the unread + // reports do not account for. This collect has no stored verdict.) + root := t.TempDir() + reportDir := filepath.Join(root, "reports") + artifactPath := filepath.Join(root, "data", ReviewVerdictsFile) + recorded := time.Date(2026, 11, 7, 10, 59, 0, 0, time.UTC) + now := recorded.Add(time.Hour) + reports := []PerspectiveReport{ + storedReport(PerspectiveCorrectness, VerdictRequiresHuman, "Could not read the diff: it is too large (12k lines)."), + storedReport(PerspectiveStyle, VerdictApprove, "Fine."), + storedReport(PerspectiveIntentAlignment, VerdictApprove, "Matches the issue."), + storedReport(PerspectiveDocsCurrency, VerdictApprove, "Docs current."), + storedReport(PerspectiveSecurity, VerdictRequiresHuman, testsuite1116Summaries[PerspectiveSecurity]), + } + for _, r := range reports { + writeStoredReport(t, reportDir, r) + } + + artifact, err := CollectAndMerge(reportDir, artifactPath, AggregateOptions{}, now) + if err != nil { + t.Fatal(err) + } + if artifact.HasAggregateApproval(unreadableStoredRepo, unreadableStoredNumber, unreadableStoredHead) { + t.Fatal("head with an unread security review marked merge-eligible") + } + agg, ok := storedAggregateFor(artifact) + if !ok || agg.Verdict != VerdictIncomplete { + t.Fatalf("want incomplete while security is owed, got ok=%v %s %v", ok, agg.Verdict, agg.Reasons) + } + plan := PlanDispatch([]PullRequest{unreadableStoredPR()}, artifact, DispatchState{}, unreadableStoredDispatchOpts(now)) + if len(plan.ReviewKicks) != 1 || len(plan.ReviewKicks[0].Perspectives) != 1 || plan.ReviewKicks[0].Perspectives[0] != PerspectiveSecurity { + t.Fatalf("want the security perspective re-asked, got %+v", plan.ReviewKicks) + } +}