From 7e2f90c688cfef97a13dd1acbe1be51ab5c7f529 Mon Sep 17 00:00:00 2001 From: Danathar Date: Sat, 7 Nov 2026 14:38:44 -0500 Subject: [PATCH] fix(review): the contributor hands-free rule approves only a fully reviewed head contributorHandsFreeAggregate resolved a bare non-security requires_human (or decision question) on a contributor PR to approve + merge_eligible with no coverage check. Since #16994 the relay drops a rate-limited "could not read" review, so a head whose security review was dropped became merge-eligible on a correctness product question alone. A dead reviewer kick left the same hole. The approve branch now needs every perspective the head is owed (the configured set, capped by max_perspectives_per_pr), the same count the all-approve path uses. Short of that the head is incomplete with "review coverage incomplete: missing ...", and the planner re-asks only the missing perspective. The fixer lane, fix-cycle cap, decision questions and security findings are unchanged. Tests that approved hands-free on one report under the five-perspective default now add approving siblings or collect with max_perspectives_per_pr: 1. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BQYnPGFJx81c8JXR3o1sg1 Signed-off-by: Danathar --- .../fixed-hands-free-needs-full-coverage.md | 1 + src/docs/review-swarm.md | 8 +- src/pkg/github/review_verdict_relay_test.go | 8 +- .../github/review_verdict_unreadable_test.go | 52 +++++ src/pkg/review/hands_free_coverage_test.go | 209 ++++++++++++++++++ src/pkg/review/review_test.go | 10 +- src/pkg/review/security_finding.go | 19 +- 7 files changed, 298 insertions(+), 9 deletions(-) create mode 100644 changelog.d/fixed-hands-free-needs-full-coverage.md create mode 100644 src/pkg/review/hands_free_coverage_test.go diff --git a/changelog.d/fixed-hands-free-needs-full-coverage.md b/changelog.d/fixed-hands-free-needs-full-coverage.md new file mode 100644 index 0000000000..aa86cee69e --- /dev/null +++ b/changelog.d/fixed-hands-free-needs-full-coverage.md @@ -0,0 +1 @@ +- The contributor rule's hands-free approve now needs every perspective the head is owed. A bare non-security `requires_human` (or decision question) on a contributor PR resolved to `approve` and merge-eligible even when another perspective had never reported. Since the relay stopped recording rate-limited "could not read" reviews (#16994), that included a head whose security review was dropped: correctness asked a product question, nothing backed a security finding, and the head became merge-eligible with no security review. With a perspective missing the head now reads `incomplete` (`review coverage incomplete: missing security`), and the planner asks for the missing perspective on the normal `pending_ttl` / `max_redispatch` path. A fully reviewed head still resolves hands-free to approve, and the fixer-lane, fix-cycle-cap and security-finding outcomes are unchanged. diff --git a/src/docs/review-swarm.md b/src/docs/review-swarm.md index 4dd1411121..5f1ba8f34f 100644 --- a/src/docs/review-swarm.md +++ b/src/docs/review-swarm.md @@ -402,7 +402,13 @@ naming, scope, sequencing, licensing, a failing optional check, a conflict clanker / adoption path as `changes_requested`; a bare question is advisory, the aggregate is `approve`, recorded as `no security finding — the PR keeps moving; a person has the last word at the merge gate; recorded as advisory: `, -and the PR moves on. At the merge gate the untrusted-author invariant +and the PR moves on. That approve needs the same coverage as any other: if a +perspective the head is owed has not reported (a reviewer kick that died, or +a rate-limited security review the relay did not record), the aggregate is +`incomplete` with `review coverage incomplete: missing `, and +the planner asks for the missing ones under `pending_ttl` / +`max_redispatch`. Without that, a head the security perspective never read +could be merge-eligible. At the merge gate the untrusted-author invariant holds: a contributor the lane cannot vouch for parks at `stage/090-ready-for-human-merge` (`contributor-lane:claim-tier-untrusted`, `untrusted-author-role`, the human-merge rule) and waits for the operator's diff --git a/src/pkg/github/review_verdict_relay_test.go b/src/pkg/github/review_verdict_relay_test.go index 8f4e719aef..25d0642ffc 100644 --- a/src/pkg/github/review_verdict_relay_test.go +++ b/src/pkg/github/review_verdict_relay_test.go @@ -113,7 +113,10 @@ func TestRecordReviewVerdictIsCollectable(t *testing.T) { if len(matches) != 1 { t.Fatalf("want exactly one collectable report, got %v", matches) } - artifact, err := review.Collect(dir, review.AggregateOptions{}) + // One perspective per PR (max_perspectives_per_pr: 1), so the one + // relayed report is full coverage: a hands-free approve needs every + // perspective the head is owed. + artifact, err := review.Collect(dir, review.AggregateOptions{MaxPerspectivesPerPR: 1}) if err != nil { t.Fatalf("Collect: %v", err) } @@ -201,7 +204,8 @@ func TestRecordReviewVerdictPromotesDecisionMarker(t *testing.T) { if len(recorded) != 1 || recorded[0].Verdict != review.VerdictRequiresHuman || !recorded[0].DecisionRequested { t.Fatalf("recorded = %+v, want promoted decision request", recorded) } - artifact, err := review.Collect(dir, review.AggregateOptions{}) + // One perspective per PR, so the one report is full coverage. + artifact, err := review.Collect(dir, review.AggregateOptions{MaxPerspectivesPerPR: 1}) if err != nil { t.Fatalf("Collect: %v", err) } diff --git a/src/pkg/github/review_verdict_unreadable_test.go b/src/pkg/github/review_verdict_unreadable_test.go index 9e432cd400..6fe9ce69da 100644 --- a/src/pkg/github/review_verdict_unreadable_test.go +++ b/src/pkg/github/review_verdict_unreadable_test.go @@ -160,3 +160,55 @@ func TestRecordReviewVerdictKeepsReadPerspectivesAndDropsRateLimitedOnes(t *test t.Fatalf("aggregate = %s (requires_human=%t merge_eligible=%t), want incomplete", got.Verdict, got.RequiresHuman, got.MergeEligible) } } + +// The rate-limited security report is dropped (no verdict), and correctness +// read the PR and asked a non-security question. On a contributor PR that +// question alone is hands-free, but the head has had no security review: +// it must read incomplete, never merge-eligible. Before the coverage check +// the hands-free rule approved it. +func TestRecordReviewVerdictDroppedSecurityLeavesHandsFreeHeadIncomplete(t *testing.T) { + const ( + repo = "o/r" + num = 8 + head = "def456" + ) + dir := t.TempDir() + c := &Client{logger: testLogger()} + var pending []review.PendingReview + var elems []string + for _, p := range review.DefaultPerspectives { + pending = append(pending, review.PendingReview{Repo: repo, Number: num, HeadSHA: head, Perspective: p, Agent: "reviewer", Dispatched: time.Now().UTC()}) + switch p { + case review.PerspectiveSecurity: + elems = append(elems, verdictJSONWithSummary(t, repo, num, p, "requires_human", head, rateLimitedSummaries1116[p])) + case review.PerspectiveCorrectness: + elems = append(elems, verdictJSONWithSummary(t, repo, num, p, "requires_human", head, "Read the diff; unclear whether this groundwork should land before its consumer.")) + default: + elems = append(elems, verdictJSONWithSummary(t, repo, num, p, "approve", head, "Read the diff; fine.")) + } + } + withVerdictDispatchState(t, review.DispatchState{Pending: pending}) + + recorded := c.recordReviewVerdict(context.Background(), ReviewRequest{Repo: repo, Number: num, Agent: "reviewer", Report: "[" + strings.Join(elems, ",") + "]"}, dir) + + if len(recorded) != len(review.DefaultPerspectives)-1 { + t.Fatalf("recorded %d verdicts, want every perspective but the unread security one", len(recorded)) + } + artifact, err := review.Collect(dir, review.AggregateOptions{}) + if err != nil { + t.Fatalf("Collect: %v", err) + } + if len(artifact.Items) != 1 { + t.Fatalf("want one aggregate, got %+v", artifact.Items) + } + got := artifact.Items[0] + if _, ok := got.Perspectives[review.PerspectiveSecurity]; ok { + t.Fatalf("unread security perspective was counted: %v", got.Perspectives) + } + if got.MergeEligible || got.Verdict != review.VerdictIncomplete || got.RequiresHuman { + t.Fatalf("aggregate = %s (merge_eligible=%t requires_human=%t), want incomplete: the head has had no security review", got.Verdict, got.MergeEligible, got.RequiresHuman) + } + if !reasonsContain(got.Reasons, "review coverage incomplete: missing security") { + t.Fatalf("Reasons = %v, want the missing security perspective named", got.Reasons) + } +} diff --git a/src/pkg/review/hands_free_coverage_test.go b/src/pkg/review/hands_free_coverage_test.go new file mode 100644 index 0000000000..cb6b9c9d5b --- /dev/null +++ b/src/pkg/review/hands_free_coverage_test.go @@ -0,0 +1,209 @@ +package review + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/outputschema" +) + +// The contributor hands-free rule (hivecommons/hive#15640) lets a bare +// non-security requires_human on a contributor PR resolve to approve. It is +// an approve like any other, so it needs the same coverage the all-approve +// path needs: a head the security perspective never reported on has had no +// security review, and must read incomplete, not merge-eligible. Since +// hivecommons/hive#16994 the relay drops a rate-limited "could not read" +// security report instead of recording it, so a missing security report is +// a real case, not only a dead reviewer kick. + +func handsFreeReports(skip Perspective, ask func(*PerspectiveReport)) []PerspectiveReport { + var out []PerspectiveReport + for _, p := range DefaultPerspectives { + if p == skip { + continue + } + if p == PerspectiveCorrectness { + r := baseReport(p, VerdictRequiresHuman) + r.Summary = "Unclear whether the groundwork should land before its consumer." + if ask != nil { + ask(&r) + } + out = append(out, r) + continue + } + out = append(out, baseReport(p, VerdictApprove)) + } + return out +} + +// withApprovingSiblings is r plus an approve from every other default +// perspective: a fully reviewed head. +func withApprovingSiblings(r PerspectiveReport) []PerspectiveReport { + out := []PerspectiveReport{r} + for _, p := range DefaultPerspectives { + if p != r.Perspective { + out = append(out, baseReport(p, VerdictApprove)) + } + } + return out +} + +func askDecision(r *PerspectiveReport) { + r.DecisionRequested = true + r.DecisionQuestion = "@clubanderson should decide whether this groundwork lands before its consumer." +} + +func TestContributorHandsFreeWithMissingPerspectiveIsIncomplete(t *testing.T) { + cases := []struct { + name string + ask func(*PerspectiveReport) + }{ + {name: "bare requires_human"}, + {name: "decision question", ask: askDecision}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := AggregateReports(handsFreeReports(PerspectiveSecurity, tc.ask), AggregateOptions{}) + if got.Verdict != VerdictIncomplete || got.MergeEligible || got.RequiresHuman || got.DecisionRequested || got.FixCycle { + t.Fatalf("aggregate = %s (merge_eligible=%t requires_human=%t decision=%t fix=%t), want incomplete and not merge-eligible", + got.Verdict, got.MergeEligible, got.RequiresHuman, got.DecisionRequested, got.FixCycle) + } + want := "review coverage incomplete: missing security" + if len(got.Reasons) != 1 || got.Reasons[0] != want { + t.Fatalf("Reasons = %q, want [%q] (the all-approve path's reason)", got.Reasons, want) + } + if _, ok := got.Perspectives[PerspectiveSecurity]; ok { + t.Fatalf("perspectives = %v, the missing security perspective was counted", got.Perspectives) + } + }) + } +} + +// The same report set with every perspective present still resolves hands- +// free to approve: the coverage check only adds the missing-perspective case. +func TestContributorHandsFreeWithFullCoverageStillApproves(t *testing.T) { + cases := []struct { + name string + ask func(*PerspectiveReport) + }{ + {name: "bare requires_human"}, + {name: "decision question", ask: askDecision}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := AggregateReports(handsFreeReports("", tc.ask), AggregateOptions{}) + if got.Verdict != VerdictApprove || !got.MergeEligible || got.RequiresHuman || got.DecisionRequested { + t.Fatalf("aggregate = %+v, want hands-free advisory approve", got) + } + if got.Perspectives[PerspectiveCorrectness] != VerdictApprove { + t.Fatalf("perspectives = %v, want the requires_human read as approve", got.Perspectives) + } + if len(got.Reasons) != 1 || !reasonsHave(got.Reasons, "no security finding") { + t.Fatalf("Reasons = %v, want the hands-free advisory", got.Reasons) + } + }) + } + // Under a per-PR cap the head is owed only that many perspectives. + capped := AggregateReports(handsFreeReports(PerspectiveSecurity, nil), AggregateOptions{MaxPerspectivesPerPR: 3}) + if capped.Verdict != VerdictApprove || !capped.MergeEligible { + t.Fatalf("capped aggregate = %+v, want approve: four of the three owed perspectives reported", capped) + } +} + +// The other hands-free outcomes do not wait on coverage: a fix the review +// asked for goes to the fixer now, and the fix-cycle cap still hands the +// head to a person. +func TestContributorHandsFreeFixOutcomesIgnoreCoverage(t *testing.T) { + withFinding := func(r *PerspectiveReport) { + r.Findings = append(r.Findings, outputschema.Finding{Title: "Get-before-Clear is redundant", Severity: outputschema.SeverityLow, ReviewScope: "in-scope"}) + } + got := AggregateReports(handsFreeReports(PerspectiveSecurity, withFinding), AggregateOptions{}) + if got.Verdict != VerdictChangesRequested || !got.FixCycle || got.MergeEligible { + t.Fatalf("aggregate = %+v, want the fixer lane", got) + } + got = AggregateReports(handsFreeReports(PerspectiveSecurity, withFinding), AggregateOptions{FixAttempts: 3, MaxFixAttempts: 3}) + if got.Verdict != VerdictRequiresHuman || !got.RequiresHuman || got.MergeEligible { + t.Fatalf("aggregate = %+v, want the fix-cycle cap to hold for a person", got) + } +} + +// The planner re-asks only the missing perspective, asks no person, and +// once the security report arrives the head approves: no loop. +func TestContributorHandsFreeIncompleteReasksOnlyMissingPerspective(t *testing.T) { + now := time.Date(2026, 11, 7, 12, 0, 0, 0, time.UTC) + agg := AggregateReports(handsFreeReports(PerspectiveSecurity, nil), AggregateOptions{}) + agg.RecordedAt = now + pr := PullRequest{Repo: agg.Repo, Number: agg.Number, Title: "groundwork", Author: "contributor", HeadSHA: testSHA} + for perspective := range agg.Perspectives { + agg.ReportReceipts = append(agg.ReportReceipts, ReportReceipt{Perspective: perspective, Agent: "r1", ReceivedAt: now.Add(-time.Minute)}) + } + opts := DispatchOptions{RequireApproval: true, FanOut: true, ProjectOrg: "hivecommons", Now: now, + MaxParallelReviews: 5, Agents: []AgentCapability{reviewer("r1"), reviewer("r2")}} + plan := PlanDispatch([]PullRequest{pr}, Artifact{Items: []Aggregate{agg}}, DispatchState{}, opts) + if len(plan.NewHuman) != 0 || len(plan.State.Human) != 0 { + t.Fatalf("an incomplete head asked a person: %+v", plan.NewHuman) + } + var asked []Perspective + for _, k := range plan.ReviewKicks { + asked = append(asked, k.Perspective) + } + if len(asked) != 1 || asked[0] != PerspectiveSecurity { + t.Fatalf("re-asked %v, want only the missing security perspective", asked) + } + + done := AggregateReports(append(handsFreeReports(PerspectiveSecurity, nil), baseReport(PerspectiveSecurity, VerdictApprove)), AggregateOptions{}) + if done.Verdict != VerdictApprove || !done.MergeEligible { + t.Fatalf("aggregate after the security report = %+v, want approve", done) + } +} + +// A verdict artifact written before this fix may hold a hands-free approve +// for a partly reviewed head. The fresh incomplete replaces it (approve is +// not a settled non-approve verdict that retainsOverIncomplete keeps). +func TestContributorHandsFreeIncompleteReplacesStoredPartialApprove(t *testing.T) { + root := t.TempDir() + reportDir := filepath.Join(root, "reports") + artifactPath := filepath.Join(root, "data", ReviewVerdictsFile) + now := time.Date(2026, 11, 7, 12, 0, 0, 0, time.UTC) + if err := os.MkdirAll(reportDir, 0o755); err != nil { + t.Fatal(err) + } + for _, r := range handsFreeReports(PerspectiveSecurity, nil) { + raw, err := json.Marshal(r) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(reportDir, ReviewReportFilePrefix+string(r.Perspective)+ReviewReportFileSuffix), raw, 0o600); err != nil { + t.Fatal(err) + } + } + stale := AggregateReports(handsFreeReports(PerspectiveSecurity, nil), AggregateOptions{}) + stale.Verdict, stale.MergeEligible, stale.Reasons = VerdictApprove, true, []string{"no security finding — the PR keeps moving"} + stale.RecordedAt = now.Add(-time.Hour) + if err := WriteArtifact(artifactPath, Artifact{GeneratedAt: stale.RecordedAt, Items: []Aggregate{stale}}); err != nil { + t.Fatal(err) + } + merged, err := CollectAndMerge(reportDir, artifactPath, AggregateOptions{}, now) + if err != nil { + t.Fatal(err) + } + if len(merged.Items) != 1 { + t.Fatalf("items = %+v, want one", merged.Items) + } + if got := merged.Items[0]; got.Verdict != VerdictIncomplete || got.MergeEligible { + t.Fatalf("merged = %s merge_eligible=%t, want the stored partial approve replaced by incomplete", got.Verdict, got.MergeEligible) + } +} + +func reasonsHave(reasons []string, sub string) bool { + for _, r := range reasons { + if strings.Contains(r, sub) { + return true + } + } + return false +} diff --git a/src/pkg/review/review_test.go b/src/pkg/review/review_test.go index 9f101ff838..1fa49e4ec0 100644 --- a/src/pkg/review/review_test.go +++ b/src/pkg/review/review_test.go @@ -64,12 +64,12 @@ func TestAggregateReportsMatrix(t *testing.T) { r.DecisionQuestion = "@clubanderson should decide whether this workflow trigger change is acceptable." return r }()}, want: VerdictRequiresHuman, human: true}, - {name: "unbacked decision request on a contributor PR is advisory", reps: []PerspectiveReport{func() PerspectiveReport { + {name: "unbacked decision request on a fully reviewed contributor PR is advisory", reps: withApprovingSiblings(func() PerspectiveReport { r := baseReport(PerspectiveCorrectness, VerdictApprove) r.DecisionRequested = true r.DecisionQuestion = "@clubanderson should decide whether this groundwork lands before its consumer." return r - }()}, want: VerdictApprove, merge: true}, + }()), want: VerdictApprove, merge: true}, {name: "reject recommends close", reps: []PerspectiveReport{baseReport(PerspectiveCorrectness, VerdictReject)}, want: VerdictReject, close: true}, {name: "fix cap requires human", reps: []PerspectiveReport{baseReport(PerspectiveCorrectness, VerdictChangesRequested)}, opts: AggregateOptions{FixAttempts: DefaultFixCycleCap}, want: VerdictRequiresHuman, human: true}, {name: "missing perspectives is incomplete, not a human hold", reps: []PerspectiveReport{baseReport(PerspectiveCorrectness, VerdictApprove)}, want: VerdictIncomplete}, @@ -739,7 +739,9 @@ func TestAggregateReportsContributorDecisionNeedsBacking(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got := AggregateReports([]PerspectiveReport{tt.rep}, tt.opts) + // The siblings approve so the head is fully reviewed: a hands- + // free approve needs the same coverage as any approve. + got := AggregateReports(withApprovingSiblings(tt.rep), tt.opts) if got.Verdict != tt.want { t.Fatalf("aggregate = %+v, want %s", got, tt.want) } @@ -775,7 +777,7 @@ func TestAggregateReportsContributorDecisionNeedsBacking(t *testing.T) { func TestAggregateReportsContributorRequiresHumanNeedsSecurityFinding(t *testing.T) { plain := baseReport(PerspectiveIntentAlignment, VerdictRequiresHuman) plain.Summary = "Unclear whether the groundwork should land before its consumer." - got := AggregateReports([]PerspectiveReport{plain}, AggregateOptions{}) + got := AggregateReports(withApprovingSiblings(plain), AggregateOptions{}) if got.Verdict != VerdictApprove || got.RequiresHuman || !got.MergeEligible { t.Fatalf("aggregate = %+v, want advisory approve", got) } diff --git a/src/pkg/review/security_finding.go b/src/pkg/review/security_finding.go index 284c75ae53..539a9d1949 100644 --- a/src/pkg/review/security_finding.go +++ b/src/pkg/review/security_finding.go @@ -195,7 +195,9 @@ func contributorDecisionAdvisory(questions []string) string { // no security finding backs: to the fixer lane when the review left // something to fix (the clanker / adoption path is the mover), otherwise // to approve so the PR reaches the merge gate, where a person decides. -// Either way nobody is asked while the PR is in CI/Review. +// Approve needs every required perspective to have reported; short of that +// the head is incomplete. Either way nobody is asked while the PR is in +// CI/Review. func contributorHandsFreeAggregate(agg Aggregate, opts AggregateOptions, needsFix bool, maxFix int, advisory []string) Aggregate { if needsFix { if opts.FixAttempts >= maxFix { @@ -213,9 +215,22 @@ func contributorHandsFreeAggregate(agg Aggregate, opts AggregateOptions, needsFi agg.FixCycle = true markRequiresHumanPerspectivesAsChangesRequested(&agg) } else { + markRequiresHumanPerspectivesAsApprove(&agg) + if len(agg.Perspectives) < requiredPerspectiveCount(opts.MaxPerspectivesPerPR, opts.Perspectives.Len()) { + // An approve needs the coverage the all-approve path needs. A + // perspective that never reported (a dead kick, or a rate- + // limited security review the relay dropped, #16994) judged + // nothing, so the head is still under review: incomplete, and + // the planner asks for the missing perspectives. + agg.Verdict = VerdictIncomplete + agg.RequiresHuman = false + agg.DecisionRequested = false + agg.DecisionQuestion = "" + agg.Reasons = []string{incompleteCoverageReason(agg.Perspectives, opts)} + return agg + } agg.Verdict = VerdictApprove agg.MergeEligible = true - markRequiresHumanPerspectivesAsApprove(&agg) agg.Reasons = nil } agg.RequiresHuman = false