From 4e4bd1980c2dedabf1f02b78ff8460d421b2b172 Mon Sep 17 00:00:00 2001 From: Danathar Date: Sat, 7 Nov 2026 14:47:22 -0500 Subject: [PATCH] fix(review): a reject is never resolved to approve, and every approve path needs full coverage On a contributor PR, a perspective's reject next to another perspective's non-security decision question went to the hands-free rule, which made the question advisory and dropped the reject with it: approve and merge-eligible (or the fixer lane with a change request). The head now gets what the reject alone gives it, reject, and the question stays in reasons. On a hive-authored PR, the advisory approves for a non-risk decision question and for a bare non-risk requires_human never checked coverage, so a head whose security review never arrived (a dead kick, or a rate-limited review the relay no longer records, #16994) was merge-eligible. With a perspective missing the head now reads incomplete and the planner re-asks only that perspective. A fully reviewed head still resolves as advisory. Refs #16994 Refs #17082 Refs #17087 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BQYnPGFJx81c8JXR3o1sg1 Signed-off-by: Danathar --- .../fixed-review-reject-and-coverage.md | 1 + src/docs/review-swarm.md | 16 +- src/pkg/review/approve_needs_coverage_test.go | 222 ++++++++++++++++++ src/pkg/review/permissions_delta_test.go | 4 +- src/pkg/review/review.go | 43 ++++ src/pkg/review/review_test.go | 6 +- 6 files changed, 287 insertions(+), 5 deletions(-) create mode 100644 changelog.d/fixed-review-reject-and-coverage.md create mode 100644 src/pkg/review/approve_needs_coverage_test.go diff --git a/changelog.d/fixed-review-reject-and-coverage.md b/changelog.d/fixed-review-reject-and-coverage.md new file mode 100644 index 0000000000..5837ae3ded --- /dev/null +++ b/changelog.d/fixed-review-reject-and-coverage.md @@ -0,0 +1 @@ +- Two review-aggregate paths no longer approve heads they should not. On a contributor PR, a perspective's `reject` next to another perspective's non-security decision question resolved hands-free to `approve` and merge-eligible (with a `changes_requested` too, to the fixer lane): the contributor rule made the question advisory and dropped the reject with it. The head now gets the verdict the reject alone gives it, `reject`, which holds the PR for a person; the question stays in the reasons. On a hive-authored PR, the advisory approves for a non-risk decision question and for a bare non-risk `requires_human` never checked review coverage, so a head whose security review never arrived (a dead kick, or since #16994 a rate-limited review the relay does not record) was merge-eligible. With a perspective missing the head now reads `incomplete` (`review coverage incomplete: missing security`), and the planner re-asks only that perspective under `pending_ttl` / `max_redispatch`. A fully reviewed hive PR's non-risk question is still advisory. diff --git a/src/docs/review-swarm.md b/src/docs/review-swarm.md index b3f1466e80..8377f720e9 100644 --- a/src/docs/review-swarm.md +++ b/src/docs/review-swarm.md @@ -316,7 +316,14 @@ recorded on the approving aggregate as (Before [#13020](https://github.com/hivecommons/hive/pull/13020) the policy only resolved questions matching a short allowlist of non-risk words, so a green agent PR whose linked issue had already specified the default still -parked for a person.) The risk classes are: +parked for a person.) That approve, like the advisory approve of a bare +non-risk `requires_human` (`self-authored non-risk requires_human recorded +as advisory`), needs every perspective the head is owed to have reported. +If one has not (a reviewer kick that died, or a rate-limited security +review the relay did not record), the aggregate is `incomplete` with +`review coverage incomplete: missing `, and the planner asks +for the missing ones under `pending_ttl` / `max_redispatch`. Once they +report, the same question resolves as advisory. The risk classes are: | Class | Escalates when the question names … | |---|---| @@ -396,6 +403,13 @@ went to its author at `stage/072-returned-to-contributor` for a PR-body edit while the security review's App-credential question waited on a maintainer). +The rule makes a question advisory, not another perspective's `reject`. A +contributor PR where one perspective rejects and another asks a +non-security question gets the verdict the reject alone gives it: `reject`, +which holds the PR for a person (the question stays in the aggregate's +reasons). Before, the question's advisory approve also swallowed the +reject, and the head was merge-eligible. + Everything else the review raises — ordinary nits and findings, a duplicate, naming, scope, sequencing, licensing, a failing optional check, a conflict — never asks a person during CI/Review: a finding goes back to the diff --git a/src/pkg/review/approve_needs_coverage_test.go b/src/pkg/review/approve_needs_coverage_test.go new file mode 100644 index 0000000000..c38308d9dc --- /dev/null +++ b/src/pkg/review/approve_needs_coverage_test.go @@ -0,0 +1,222 @@ +package review + +import ( + "strings" + "testing" + "time" +) + +// Two aggregate paths approved heads they should not have: +// +// - On a contributor PR the hands-free rule (hivecommons/hive#15640) made a +// non-security decision question advisory, and with it a sibling +// perspective's reject: reject + question resolved to approve and +// merge-eligible. +// - The hive-authored advisory approves (a non-risk decision question, +// hivecommons/hive#12834, or a bare non-risk requires_human, +// hivecommons/hive#12888) never checked coverage. A head the security +// perspective never reported on (a dead kick, or since +// hivecommons/hive#16994 a rate-limited review the relay does not record) +// was merge-eligible with no security review. + +const p33Question = "@clubanderson should decide whether this groundwork lands before its consumer." + +// alongsideApprovals is reports plus an approve, with a neutral summary, from +// every default perspective none of them covers: a fully reviewed head. +// skip leaves those perspectives out, as a review that never arrived. +func alongsideApprovals(reports []PerspectiveReport, skip ...Perspective) []PerspectiveReport { + have := map[Perspective]bool{} + for _, p := range skip { + have[p] = true + } + hive := false + for _, r := range reports { + have[r.Perspective] = true + hive = hive || r.HiveAuthored + } + out := append([]PerspectiveReport(nil), reports...) + for _, p := range DefaultPerspectives { + if have[p] { + continue + } + r := baseReport(p, VerdictApprove) + r.Summary = "Read the diff; fine." + r.HiveAuthored = hive + out = append(out, r) + } + return out +} + +func p33DecisionQuestion(p Perspective, hive bool) PerspectiveReport { + r := baseReport(p, VerdictRequiresHuman) + r.Summary = "Read the diff; the sequencing is a product call." + r.DecisionRequested = true + r.DecisionQuestion = p33Question + r.HiveAuthored = hive + return r +} + +func p33BareRequiresHuman(p Perspective, hive bool) PerspectiveReport { + r := baseReport(p, VerdictRequiresHuman) + r.Summary = "Unclear whether the groundwork should land before its consumer." + r.HiveAuthored = hive + return r +} + +func TestContributorRejectWithDecisionQuestionStaysReject(t *testing.T) { + reject := baseReport(PerspectiveIntentAlignment, VerdictReject) + reject.Summary = "This duplicates an open PR; it should not land." + changes := baseReport(PerspectiveStyle, VerdictChangesRequested) + changes.Summary = "Rename the helper." + cases := []struct { + name string + reps []PerspectiveReport + }{ + {name: "reject + decision question", reps: []PerspectiveReport{p33DecisionQuestion(PerspectiveCorrectness, false), reject}}, + // Without the reject check the fixer lane took it: the reject was + // dropped just the same. + {name: "reject + changes_requested + decision question", reps: []PerspectiveReport{p33DecisionQuestion(PerspectiveCorrectness, false), reject, changes}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + reps := alongsideApprovals(tc.reps) + if len(reps) != len(DefaultPerspectives) { + t.Fatalf("fixture has %d perspectives, want all %d", len(reps), len(DefaultPerspectives)) + } + got := AggregateReports(reps, AggregateOptions{}) + if got.Verdict != VerdictReject || got.MergeEligible || got.FixCycle || !got.CloseRecommended { + t.Fatalf("aggregate = %s (merge_eligible=%t fix=%t close=%t), want reject: a reject is never resolved to approve", + got.Verdict, got.MergeEligible, got.FixCycle, got.CloseRecommended) + } + // Same outcome as the reject without the question. + var noQuestion []PerspectiveReport + for _, r := range reps { + if r.DecisionRequested { + r = baseReport(r.Perspective, VerdictApprove) + r.Summary = "Read the diff; fine." + } + noQuestion = append(noQuestion, r) + } + if want := AggregateReports(noQuestion, AggregateOptions{}); want.Verdict != got.Verdict || want.CloseRecommended != got.CloseRecommended { + t.Fatalf("with the question: %s close=%t; without it: %s close=%t, want the same", got.Verdict, got.CloseRecommended, want.Verdict, want.CloseRecommended) + } + if got.DecisionRequested || got.RequiresHuman { + t.Fatalf("aggregate = %+v, the contributor question must stay advisory", got) + } + if !reasonsInclude(got.Reasons, p33Question) { + t.Fatalf("Reasons = %q, want the decision question recorded", got.Reasons) + } + if BlocksSelfMerge(got) != true { + t.Fatalf("BlocksSelfMerge = false for %+v, want the reject to hold the PR", got) + } + }) + } +} + +// The bare requires_human and changes_requested paths already keep a reject +// and a change request on a contributor PR (the reject check and the fix +// cycle run before the hands-free branch); these pin it. +func TestContributorBareRequiresHumanKeepsRejectAndChanges(t *testing.T) { + reject := baseReport(PerspectiveIntentAlignment, VerdictReject) + reject.Summary = "This duplicates an open PR; it should not land." + changes := baseReport(PerspectiveStyle, VerdictChangesRequested) + changes.Summary = "Rename the helper." + got := AggregateReports(alongsideApprovals([]PerspectiveReport{p33BareRequiresHuman(PerspectiveCorrectness, false), reject}), AggregateOptions{}) + if got.Verdict != VerdictReject || got.MergeEligible { + t.Fatalf("reject + bare requires_human = %+v, want reject", got) + } + got = AggregateReports(alongsideApprovals([]PerspectiveReport{p33BareRequiresHuman(PerspectiveCorrectness, false), changes}), AggregateOptions{}) + if got.Verdict != VerdictChangesRequested || !got.FixCycle || got.MergeEligible { + t.Fatalf("changes_requested + bare requires_human = %+v, want the fixer lane", got) + } +} + +// Control: the decision question alone, the other four perspectives +// approving, resolves hands-free to approve as before. +func TestContributorDecisionQuestionAloneStillApprovesHandsFree(t *testing.T) { + got := AggregateReports(alongsideApprovals([]PerspectiveReport{p33DecisionQuestion(PerspectiveCorrectness, false)}), AggregateOptions{}) + if got.Verdict != VerdictApprove || !got.MergeEligible || got.RequiresHuman || got.DecisionRequested { + t.Fatalf("aggregate = %+v, want the hands-free advisory approve", got) + } + if len(got.Reasons) != 1 || !strings.HasPrefix(got.Reasons[0], "no security finding") || !strings.Contains(got.Reasons[0], p33Question) { + t.Fatalf("Reasons = %q, want the contributor advisory with the question", got.Reasons) + } +} + +func TestSelfAuthoredAdvisoryApproveNeedsFullCoverage(t *testing.T) { + cases := []struct { + name string + rep PerspectiveReport + wantReason string + }{ + {name: "bare non-risk requires_human", rep: p33BareRequiresHuman(PerspectiveCorrectness, true), wantReason: "self-authored non-risk requires_human recorded as advisory"}, + {name: "non-risk decision question", rep: p33DecisionQuestion(PerspectiveCorrectness, true), wantReason: "decision resolved by policy:"}, + } + for _, tc := range cases { + t.Run(tc.name+"/security missing", func(t *testing.T) { + got := AggregateReports(alongsideApprovals([]PerspectiveReport{tc.rep}, PerspectiveSecurity), AggregateOptions{}) + if got.Verdict != VerdictIncomplete || got.MergeEligible || got.RequiresHuman || got.DecisionRequested || got.FixCycle { + t.Fatalf("aggregate = %s (merge_eligible=%t requires_human=%t decision=%t fix=%t), want incomplete: the head had no security review", + got.Verdict, got.MergeEligible, got.RequiresHuman, got.DecisionRequested, got.FixCycle) + } + if want := "review coverage incomplete: missing security"; len(got.Reasons) != 1 || got.Reasons[0] != want { + t.Fatalf("Reasons = %q, want [%q]", got.Reasons, want) + } + if len(got.HumanSummaries) != 0 { + t.Fatalf("HumanSummaries = %q, an incomplete head asks nobody", got.HumanSummaries) + } + if BlocksSelfMerge(got) { + t.Fatalf("incomplete head blocks self-merge as a person's hold: %+v", got) + } + }) + // Control: the same set with every perspective present stays the + // advisory approve the self-authored rule is for. + t.Run(tc.name+"/full coverage", func(t *testing.T) { + got := AggregateReports(alongsideApprovals([]PerspectiveReport{tc.rep}), AggregateOptions{}) + if got.Verdict != VerdictApprove || !got.MergeEligible || got.RequiresHuman || got.DecisionRequested { + t.Fatalf("aggregate = %+v, want advisory approve", got) + } + if !reasonsInclude(got.Reasons, tc.wantReason) { + t.Fatalf("Reasons = %q, want %q", got.Reasons, tc.wantReason) + } + }) + } + // Under a per-PR cap the head is owed only that many perspectives. + capped := AggregateReports(alongsideApprovals([]PerspectiveReport{p33BareRequiresHuman(PerspectiveCorrectness, true)}, PerspectiveSecurity), AggregateOptions{MaxPerspectivesPerPR: 3}) + if capped.Verdict != VerdictApprove || !capped.MergeEligible { + t.Fatalf("capped aggregate = %+v, want approve: four reported, three owed", capped) + } +} + +// The planner re-asks only the missing perspective and asks no person. +func TestSelfAuthoredIncompleteReasksOnlyMissingPerspective(t *testing.T) { + now := time.Date(2026, 11, 7, 12, 0, 0, 0, time.UTC) + agg := AggregateReports(alongsideApprovals([]PerspectiveReport{p33BareRequiresHuman(PerspectiveCorrectness, true)}, PerspectiveSecurity), AggregateOptions{}) + agg.RecordedAt = now + for perspective := range agg.Perspectives { + agg.ReportReceipts = append(agg.ReportReceipts, ReportReceipt{Perspective: perspective, Agent: "r1", ReceivedAt: now.Add(-time.Minute)}) + } + pr := PullRequest{Repo: agg.Repo, Number: agg.Number, Title: "groundwork", Author: "hive-bot", HeadSHA: testSHA} + opts := DispatchOptions{RequireApproval: true, FanOut: true, ProjectOrg: "hivecommons", Now: now, + MaxParallelReviews: 5, Agents: []AgentCapability{reviewer("r1"), reviewer("r2")}} + plan := PlanDispatch([]PullRequest{pr}, Artifact{Items: []Aggregate{agg}}, DispatchState{}, opts) + if len(plan.NewHuman) != 0 || len(plan.State.Human) != 0 { + t.Fatalf("an incomplete head asked a person: %+v", plan.NewHuman) + } + var asked []Perspective + for _, k := range plan.ReviewKicks { + asked = append(asked, k.Perspective) + } + if len(asked) != 1 || asked[0] != PerspectiveSecurity { + t.Fatalf("re-asked %v, want only the missing security perspective", asked) + } +} + +func reasonsInclude(reasons []string, sub string) bool { + for _, r := range reasons { + if strings.Contains(r, sub) { + return true + } + } + return false +} diff --git a/src/pkg/review/permissions_delta_test.go b/src/pkg/review/permissions_delta_test.go index 59980db258..69ec8d2492 100644 --- a/src/pkg/review/permissions_delta_test.go +++ b/src/pkg/review/permissions_delta_test.go @@ -278,7 +278,7 @@ func TestAggregateReportsResolvesPermissionsTighteningDecision(t *testing.T) { r.DecisionQuestion = "please confirm the workflow permissions narrowing (id-token: write moved to the job) is acceptable to merge" r.Summary = "Permissions narrowed at workflow level; looks correct." - got := AggregateReports([]PerspectiveReport{r}, AggregateOptions{}) + got := AggregateReports(alongsideApprovals([]PerspectiveReport{r}), AggregateOptions{}) if got.Verdict != VerdictApprove || got.RequiresHuman || got.DecisionRequested || !got.MergeEligible { t.Fatalf("aggregate = %+v, want approve resolved by policy", got) @@ -328,7 +328,7 @@ func TestAggregateReportsPermissionsTighteningSummaryIsAdvisory(t *testing.T) { r.PermissionsTighteningOnly = true r.Summary = "Looks right; workflow permissions narrowed, nothing widened." - got := AggregateReports([]PerspectiveReport{r}, AggregateOptions{}) + got := AggregateReports(alongsideApprovals([]PerspectiveReport{r}), AggregateOptions{}) if got.Verdict != VerdictApprove || got.RequiresHuman { t.Fatalf("aggregate = %+v, want advisory approve", got) diff --git a/src/pkg/review/review.go b/src/pkg/review/review.go index 72cdaab219..597b445e08 100644 --- a/src/pkg/review/review.go +++ b/src/pkg/review/review.go @@ -623,6 +623,15 @@ func aggregateReports(reports []PerspectiveReport, opts AggregateOptions) Aggreg // person has the last word at the merge gate (090), where the // untrusted-author merge is theirs. Twelve of hive's own PRs // sat 3–21h on such questions. + if hasReject { + // The rule makes the question advisory, not a sibling + // perspective's reject: the head gets what the reject + // alone would give it (the reject check below). The + // question stays in Reasons. + agg.Verdict = VerdictReject + agg.CloseRecommended = true + return agg + } return contributorHandsFreeAggregate(agg, opts, hasChanges || reportsHaveInScopeFinding(reports), maxFix, decisionQuestions) } else if backed { agg.DecisionFinding = finding @@ -645,6 +654,9 @@ func aggregateReports(reports []PerspectiveReport, opts AggregateOptions) Aggreg sortReasons(agg.Reasons) return agg } + if !hasRequiredCoverage(agg.Perspectives, opts) { + return incompleteForMissingCoverage(agg, opts) + } agg.Verdict = VerdictApprove agg.MergeEligible = true agg.RequiresHuman = false @@ -717,6 +729,9 @@ func aggregateReports(reports []PerspectiveReport, opts AggregateOptions) Aggreg return agg } if !riskEscalation && !hasChanges && selfAuthoredHumanWithoutRemediation && !selfAuthoredHumanWithConcreteRemediation && selfAuthoredResolvableRequiresHumanOnly(reports, opts) { + if !hasRequiredCoverage(agg.Perspectives, opts) { + return incompleteForMissingCoverage(agg, opts) + } agg.Verdict = VerdictApprove agg.MergeEligible = true agg.RequiresHuman = false @@ -1955,6 +1970,34 @@ func hasAllRequiredPerspectives(got map[Perspective]Verdict, maxPerPR, configure return true } +// hasRequiredCoverage reports whether every perspective the head is owed has +// reported, whatever it said. hasAllRequiredPerspectives also needs each +// verdict to be approve; the paths that read a requires_human as advisory +// need only this half of it. +func hasRequiredCoverage(got map[Perspective]Verdict, opts AggregateOptions) bool { + return len(got) >= requiredPerspectiveCount(opts.MaxPerspectivesPerPR, opts.Perspectives.Len()) +} + +// incompleteForMissingCoverage is the verdict of a head that would resolve +// to an advisory approve but that a perspective it is owed never reported +// on: a reviewer kick that died, or a rate-limited security review the relay +// did not record (hivecommons/hive#16994). That perspective judged nothing, +// so the head is still under review, as on the all-approve path: incomplete, +// nobody is asked, and the planner re-asks only the missing perspectives +// under pending_ttl / max_redispatch. Once they report, the same reports +// resolve to the advisory approve. +func incompleteForMissingCoverage(agg Aggregate, opts AggregateOptions) Aggregate { + agg.Verdict = VerdictIncomplete + agg.MergeEligible = false + agg.RequiresHuman = false + agg.FixCycle = false + agg.DecisionRequested = false + agg.DecisionQuestion = "" + agg.HumanSummaries = nil + agg.Reasons = []string{incompleteCoverageReason(agg.Perspectives, opts)} + return agg +} + // requiredPerspectiveCount is how many perspectives a head must hear from // before its approvals count as unanimous: the configured set, capped by // max_perspectives_per_pr. diff --git a/src/pkg/review/review_test.go b/src/pkg/review/review_test.go index 9f101ff838..34cfe100a5 100644 --- a/src/pkg/review/review_test.go +++ b/src/pkg/review/review_test.go @@ -123,7 +123,9 @@ func TestAggregateReportsResolvesNonRiskHiveAuthoredDecisionQuestions(t *testing r.DecisionRequested = true r.DecisionQuestion = question - got := AggregateReports([]PerspectiveReport{r}, AggregateOptions{}) + // Every perspective reported: the advisory approve needs full + // coverage (approve_needs_coverage_test.go). + got := AggregateReports(alongsideApprovals([]PerspectiveReport{r}), AggregateOptions{}) if got.Verdict != VerdictApprove || !got.MergeEligible || got.RequiresHuman || got.DecisionRequested { t.Fatalf("aggregate = %+v, want resolved approve", got) @@ -437,7 +439,7 @@ func TestAggregateReportsAdvisesSelfAuthoredNonRiskHumanWithoutRemediation(t *te r.HiveAuthored = true r.Summary = "This sequencing question is low stakes but worth noting for maintainers." - got := AggregateReports([]PerspectiveReport{r}, AggregateOptions{}) + got := AggregateReports(alongsideApprovals([]PerspectiveReport{r}), AggregateOptions{}) if got.Verdict != VerdictApprove || !got.MergeEligible || got.RequiresHuman { t.Fatalf("aggregate = %+v, want advisory approval", got)