Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/fixed-hands-free-needs-full-coverage.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- The contributor rule's hands-free approve now needs every perspective the head is owed. A bare non-security `requires_human` (or decision question) on a contributor PR resolved to `approve` and merge-eligible even when another perspective had never reported. Since the relay stopped recording rate-limited "could not read" reviews (#16994), that included a head whose security review was dropped: correctness asked a product question, nothing backed a security finding, and the head became merge-eligible with no security review. With a perspective missing the head now reads `incomplete` (`review coverage incomplete: missing security`), and the planner asks for the missing perspective on the normal `pending_ttl` / `max_redispatch` path. A fully reviewed head still resolves hands-free to approve, and the fixer-lane, fix-cycle-cap and security-finding outcomes are unchanged.
8 changes: 7 additions & 1 deletion src/docs/review-swarm.md
Original file line number Diff line number Diff line change
Expand Up @@ -402,7 +402,13 @@ naming, scope, sequencing, licensing, a failing optional check, a conflict
clanker / adoption path as `changes_requested`; a bare question is
advisory, the aggregate is `approve`, recorded as
`no security finding — the PR keeps moving; a person has the last word at the merge gate; recorded as advisory: <question>`,
and the PR moves on. At the merge gate the untrusted-author invariant
and the PR moves on. That approve needs the same coverage as any other: if a
perspective the head is owed has not reported (a reviewer kick that died, or
a rate-limited security review the relay did not record), the aggregate is
`incomplete` with `review coverage incomplete: missing <perspectives>`, and
the planner asks for the missing ones under `pending_ttl` /
`max_redispatch`. Without that, a head the security perspective never read
could be merge-eligible. At the merge gate the untrusted-author invariant
holds: a contributor the lane cannot vouch for parks at
`stage/090-ready-for-human-merge` (`contributor-lane:claim-tier-untrusted`,
`untrusted-author-role`, the human-merge rule) and waits for the operator's
Expand Down
8 changes: 6 additions & 2 deletions src/pkg/github/review_verdict_relay_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -113,7 +113,10 @@ func TestRecordReviewVerdictIsCollectable(t *testing.T) {
if len(matches) != 1 {
t.Fatalf("want exactly one collectable report, got %v", matches)
}
artifact, err := review.Collect(dir, review.AggregateOptions{})
// One perspective per PR (max_perspectives_per_pr: 1), so the one
// relayed report is full coverage: a hands-free approve needs every
// perspective the head is owed.
artifact, err := review.Collect(dir, review.AggregateOptions{MaxPerspectivesPerPR: 1})
if err != nil {
t.Fatalf("Collect: %v", err)
}
Expand Down Expand Up @@ -201,7 +204,8 @@ func TestRecordReviewVerdictPromotesDecisionMarker(t *testing.T) {
if len(recorded) != 1 || recorded[0].Verdict != review.VerdictRequiresHuman || !recorded[0].DecisionRequested {
t.Fatalf("recorded = %+v, want promoted decision request", recorded)
}
artifact, err := review.Collect(dir, review.AggregateOptions{})
// One perspective per PR, so the one report is full coverage.
artifact, err := review.Collect(dir, review.AggregateOptions{MaxPerspectivesPerPR: 1})
if err != nil {
t.Fatalf("Collect: %v", err)
}
Expand Down
52 changes: 52 additions & 0 deletions src/pkg/github/review_verdict_unreadable_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -160,3 +160,55 @@ func TestRecordReviewVerdictKeepsReadPerspectivesAndDropsRateLimitedOnes(t *test
t.Fatalf("aggregate = %s (requires_human=%t merge_eligible=%t), want incomplete", got.Verdict, got.RequiresHuman, got.MergeEligible)
}
}

// The rate-limited security report is dropped (no verdict), and correctness
// read the PR and asked a non-security question. On a contributor PR that
// question alone is hands-free, but the head has had no security review:
// it must read incomplete, never merge-eligible. Before the coverage check
// the hands-free rule approved it.
func TestRecordReviewVerdictDroppedSecurityLeavesHandsFreeHeadIncomplete(t *testing.T) {
const (
repo = "o/r"
num = 8
head = "def456"
)
dir := t.TempDir()
c := &Client{logger: testLogger()}
var pending []review.PendingReview
var elems []string
for _, p := range review.DefaultPerspectives {
pending = append(pending, review.PendingReview{Repo: repo, Number: num, HeadSHA: head, Perspective: p, Agent: "reviewer", Dispatched: time.Now().UTC()})
switch p {
case review.PerspectiveSecurity:
elems = append(elems, verdictJSONWithSummary(t, repo, num, p, "requires_human", head, rateLimitedSummaries1116[p]))
case review.PerspectiveCorrectness:
elems = append(elems, verdictJSONWithSummary(t, repo, num, p, "requires_human", head, "Read the diff; unclear whether this groundwork should land before its consumer."))
default:
elems = append(elems, verdictJSONWithSummary(t, repo, num, p, "approve", head, "Read the diff; fine."))
}
}
withVerdictDispatchState(t, review.DispatchState{Pending: pending})

recorded := c.recordReviewVerdict(context.Background(), ReviewRequest{Repo: repo, Number: num, Agent: "reviewer", Report: "[" + strings.Join(elems, ",") + "]"}, dir)

if len(recorded) != len(review.DefaultPerspectives)-1 {
t.Fatalf("recorded %d verdicts, want every perspective but the unread security one", len(recorded))
}
artifact, err := review.Collect(dir, review.AggregateOptions{})
if err != nil {
t.Fatalf("Collect: %v", err)
}
if len(artifact.Items) != 1 {
t.Fatalf("want one aggregate, got %+v", artifact.Items)
}
got := artifact.Items[0]
if _, ok := got.Perspectives[review.PerspectiveSecurity]; ok {
t.Fatalf("unread security perspective was counted: %v", got.Perspectives)
}
if got.MergeEligible || got.Verdict != review.VerdictIncomplete || got.RequiresHuman {
t.Fatalf("aggregate = %s (merge_eligible=%t requires_human=%t), want incomplete: the head has had no security review", got.Verdict, got.MergeEligible, got.RequiresHuman)
}
if !reasonsContain(got.Reasons, "review coverage incomplete: missing security") {
t.Fatalf("Reasons = %v, want the missing security perspective named", got.Reasons)
}
}
209 changes: 209 additions & 0 deletions src/pkg/review/hands_free_coverage_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,209 @@
package review

import (
"encoding/json"
"os"
"path/filepath"
"strings"
"testing"
"time"

"github.com/hivecommons/hive/pkg/outputschema"
)

// The contributor hands-free rule (hivecommons/hive#15640) lets a bare
// non-security requires_human on a contributor PR resolve to approve. It is
// an approve like any other, so it needs the same coverage the all-approve
// path needs: a head the security perspective never reported on has had no
// security review, and must read incomplete, not merge-eligible. Since
// hivecommons/hive#16994 the relay drops a rate-limited "could not read"
// security report instead of recording it, so a missing security report is
// a real case, not only a dead reviewer kick.

func handsFreeReports(skip Perspective, ask func(*PerspectiveReport)) []PerspectiveReport {
var out []PerspectiveReport
for _, p := range DefaultPerspectives {
if p == skip {
continue
}
if p == PerspectiveCorrectness {
r := baseReport(p, VerdictRequiresHuman)
r.Summary = "Unclear whether the groundwork should land before its consumer."
if ask != nil {
ask(&r)
}
out = append(out, r)
continue
}
out = append(out, baseReport(p, VerdictApprove))
}
return out
}

// withApprovingSiblings is r plus an approve from every other default
// perspective: a fully reviewed head.
func withApprovingSiblings(r PerspectiveReport) []PerspectiveReport {
out := []PerspectiveReport{r}
for _, p := range DefaultPerspectives {
if p != r.Perspective {
out = append(out, baseReport(p, VerdictApprove))
}
}
return out
}

func askDecision(r *PerspectiveReport) {
r.DecisionRequested = true
r.DecisionQuestion = "@clubanderson should decide whether this groundwork lands before its consumer."
}

func TestContributorHandsFreeWithMissingPerspectiveIsIncomplete(t *testing.T) {
cases := []struct {
name string
ask func(*PerspectiveReport)
}{
{name: "bare requires_human"},
{name: "decision question", ask: askDecision},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
got := AggregateReports(handsFreeReports(PerspectiveSecurity, tc.ask), AggregateOptions{})
if got.Verdict != VerdictIncomplete || got.MergeEligible || got.RequiresHuman || got.DecisionRequested || got.FixCycle {
t.Fatalf("aggregate = %s (merge_eligible=%t requires_human=%t decision=%t fix=%t), want incomplete and not merge-eligible",
got.Verdict, got.MergeEligible, got.RequiresHuman, got.DecisionRequested, got.FixCycle)
}
want := "review coverage incomplete: missing security"
if len(got.Reasons) != 1 || got.Reasons[0] != want {
t.Fatalf("Reasons = %q, want [%q] (the all-approve path's reason)", got.Reasons, want)
}
if _, ok := got.Perspectives[PerspectiveSecurity]; ok {
t.Fatalf("perspectives = %v, the missing security perspective was counted", got.Perspectives)
}
})
}
}

// The same report set with every perspective present still resolves hands-
// free to approve: the coverage check only adds the missing-perspective case.
func TestContributorHandsFreeWithFullCoverageStillApproves(t *testing.T) {
cases := []struct {
name string
ask func(*PerspectiveReport)
}{
{name: "bare requires_human"},
{name: "decision question", ask: askDecision},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
got := AggregateReports(handsFreeReports("", tc.ask), AggregateOptions{})
if got.Verdict != VerdictApprove || !got.MergeEligible || got.RequiresHuman || got.DecisionRequested {
t.Fatalf("aggregate = %+v, want hands-free advisory approve", got)
}
if got.Perspectives[PerspectiveCorrectness] != VerdictApprove {
t.Fatalf("perspectives = %v, want the requires_human read as approve", got.Perspectives)
}
if len(got.Reasons) != 1 || !reasonsHave(got.Reasons, "no security finding") {
t.Fatalf("Reasons = %v, want the hands-free advisory", got.Reasons)
}
})
}
// Under a per-PR cap the head is owed only that many perspectives.
capped := AggregateReports(handsFreeReports(PerspectiveSecurity, nil), AggregateOptions{MaxPerspectivesPerPR: 3})
if capped.Verdict != VerdictApprove || !capped.MergeEligible {
t.Fatalf("capped aggregate = %+v, want approve: four of the three owed perspectives reported", capped)
}
}

// The other hands-free outcomes do not wait on coverage: a fix the review
// asked for goes to the fixer now, and the fix-cycle cap still hands the
// head to a person.
func TestContributorHandsFreeFixOutcomesIgnoreCoverage(t *testing.T) {
withFinding := func(r *PerspectiveReport) {
r.Findings = append(r.Findings, outputschema.Finding{Title: "Get-before-Clear is redundant", Severity: outputschema.SeverityLow, ReviewScope: "in-scope"})
}
got := AggregateReports(handsFreeReports(PerspectiveSecurity, withFinding), AggregateOptions{})
if got.Verdict != VerdictChangesRequested || !got.FixCycle || got.MergeEligible {
t.Fatalf("aggregate = %+v, want the fixer lane", got)
}
got = AggregateReports(handsFreeReports(PerspectiveSecurity, withFinding), AggregateOptions{FixAttempts: 3, MaxFixAttempts: 3})
if got.Verdict != VerdictRequiresHuman || !got.RequiresHuman || got.MergeEligible {
t.Fatalf("aggregate = %+v, want the fix-cycle cap to hold for a person", got)
}
}

// The planner re-asks only the missing perspective, asks no person, and
// once the security report arrives the head approves: no loop.
func TestContributorHandsFreeIncompleteReasksOnlyMissingPerspective(t *testing.T) {
now := time.Date(2026, 11, 7, 12, 0, 0, 0, time.UTC)
agg := AggregateReports(handsFreeReports(PerspectiveSecurity, nil), AggregateOptions{})
agg.RecordedAt = now
pr := PullRequest{Repo: agg.Repo, Number: agg.Number, Title: "groundwork", Author: "contributor", HeadSHA: testSHA}
for perspective := range agg.Perspectives {
agg.ReportReceipts = append(agg.ReportReceipts, ReportReceipt{Perspective: perspective, Agent: "r1", ReceivedAt: now.Add(-time.Minute)})
}
opts := DispatchOptions{RequireApproval: true, FanOut: true, ProjectOrg: "hivecommons", Now: now,
MaxParallelReviews: 5, Agents: []AgentCapability{reviewer("r1"), reviewer("r2")}}
plan := PlanDispatch([]PullRequest{pr}, Artifact{Items: []Aggregate{agg}}, DispatchState{}, opts)
if len(plan.NewHuman) != 0 || len(plan.State.Human) != 0 {
t.Fatalf("an incomplete head asked a person: %+v", plan.NewHuman)
}
var asked []Perspective
for _, k := range plan.ReviewKicks {
asked = append(asked, k.Perspective)
}
if len(asked) != 1 || asked[0] != PerspectiveSecurity {
t.Fatalf("re-asked %v, want only the missing security perspective", asked)
}

done := AggregateReports(append(handsFreeReports(PerspectiveSecurity, nil), baseReport(PerspectiveSecurity, VerdictApprove)), AggregateOptions{})
if done.Verdict != VerdictApprove || !done.MergeEligible {
t.Fatalf("aggregate after the security report = %+v, want approve", done)
}
}

// A verdict artifact written before this fix may hold a hands-free approve
// for a partly reviewed head. The fresh incomplete replaces it (approve is
// not a settled non-approve verdict that retainsOverIncomplete keeps).
func TestContributorHandsFreeIncompleteReplacesStoredPartialApprove(t *testing.T) {
root := t.TempDir()
reportDir := filepath.Join(root, "reports")
artifactPath := filepath.Join(root, "data", ReviewVerdictsFile)
now := time.Date(2026, 11, 7, 12, 0, 0, 0, time.UTC)
if err := os.MkdirAll(reportDir, 0o755); err != nil {
t.Fatal(err)
}
for _, r := range handsFreeReports(PerspectiveSecurity, nil) {
raw, err := json.Marshal(r)
if err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(reportDir, ReviewReportFilePrefix+string(r.Perspective)+ReviewReportFileSuffix), raw, 0o600); err != nil {
t.Fatal(err)
}
}
stale := AggregateReports(handsFreeReports(PerspectiveSecurity, nil), AggregateOptions{})
stale.Verdict, stale.MergeEligible, stale.Reasons = VerdictApprove, true, []string{"no security finding — the PR keeps moving"}
stale.RecordedAt = now.Add(-time.Hour)
if err := WriteArtifact(artifactPath, Artifact{GeneratedAt: stale.RecordedAt, Items: []Aggregate{stale}}); err != nil {
t.Fatal(err)
}
merged, err := CollectAndMerge(reportDir, artifactPath, AggregateOptions{}, now)
if err != nil {
t.Fatal(err)
}
if len(merged.Items) != 1 {
t.Fatalf("items = %+v, want one", merged.Items)
}
if got := merged.Items[0]; got.Verdict != VerdictIncomplete || got.MergeEligible {
t.Fatalf("merged = %s merge_eligible=%t, want the stored partial approve replaced by incomplete", got.Verdict, got.MergeEligible)
}
}

func reasonsHave(reasons []string, sub string) bool {
for _, r := range reasons {
if strings.Contains(r, sub) {
return true
}
}
return false
}
10 changes: 6 additions & 4 deletions src/pkg/review/review_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -64,12 +64,12 @@ func TestAggregateReportsMatrix(t *testing.T) {
r.DecisionQuestion = "@clubanderson should decide whether this workflow trigger change is acceptable."
return r
}()}, want: VerdictRequiresHuman, human: true},
{name: "unbacked decision request on a contributor PR is advisory", reps: []PerspectiveReport{func() PerspectiveReport {
{name: "unbacked decision request on a fully reviewed contributor PR is advisory", reps: withApprovingSiblings(func() PerspectiveReport {
r := baseReport(PerspectiveCorrectness, VerdictApprove)
r.DecisionRequested = true
r.DecisionQuestion = "@clubanderson should decide whether this groundwork lands before its consumer."
return r
}()}, want: VerdictApprove, merge: true},
}()), want: VerdictApprove, merge: true},
{name: "reject recommends close", reps: []PerspectiveReport{baseReport(PerspectiveCorrectness, VerdictReject)}, want: VerdictReject, close: true},
{name: "fix cap requires human", reps: []PerspectiveReport{baseReport(PerspectiveCorrectness, VerdictChangesRequested)}, opts: AggregateOptions{FixAttempts: DefaultFixCycleCap}, want: VerdictRequiresHuman, human: true},
{name: "missing perspectives is incomplete, not a human hold", reps: []PerspectiveReport{baseReport(PerspectiveCorrectness, VerdictApprove)}, want: VerdictIncomplete},
Expand Down Expand Up @@ -739,7 +739,9 @@ func TestAggregateReportsContributorDecisionNeedsBacking(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := AggregateReports([]PerspectiveReport{tt.rep}, tt.opts)
// The siblings approve so the head is fully reviewed: a hands-
// free approve needs the same coverage as any approve.
got := AggregateReports(withApprovingSiblings(tt.rep), tt.opts)
if got.Verdict != tt.want {
t.Fatalf("aggregate = %+v, want %s", got, tt.want)
}
Expand Down Expand Up @@ -775,7 +777,7 @@ func TestAggregateReportsContributorDecisionNeedsBacking(t *testing.T) {
func TestAggregateReportsContributorRequiresHumanNeedsSecurityFinding(t *testing.T) {
plain := baseReport(PerspectiveIntentAlignment, VerdictRequiresHuman)
plain.Summary = "Unclear whether the groundwork should land before its consumer."
got := AggregateReports([]PerspectiveReport{plain}, AggregateOptions{})
got := AggregateReports(withApprovingSiblings(plain), AggregateOptions{})
if got.Verdict != VerdictApprove || got.RequiresHuman || !got.MergeEligible {
t.Fatalf("aggregate = %+v, want advisory approve", got)
}
Expand Down
19 changes: 17 additions & 2 deletions src/pkg/review/security_finding.go
Original file line number Diff line number Diff line change
Expand Up @@ -195,7 +195,9 @@ func contributorDecisionAdvisory(questions []string) string {
// no security finding backs: to the fixer lane when the review left
// something to fix (the clanker / adoption path is the mover), otherwise
// to approve so the PR reaches the merge gate, where a person decides.
// Either way nobody is asked while the PR is in CI/Review.
// Approve needs every required perspective to have reported; short of that
// the head is incomplete. Either way nobody is asked while the PR is in
// CI/Review.
func contributorHandsFreeAggregate(agg Aggregate, opts AggregateOptions, needsFix bool, maxFix int, advisory []string) Aggregate {
if needsFix {
if opts.FixAttempts >= maxFix {
Expand All @@ -213,9 +215,22 @@ func contributorHandsFreeAggregate(agg Aggregate, opts AggregateOptions, needsFi
agg.FixCycle = true
markRequiresHumanPerspectivesAsChangesRequested(&agg)
} else {
markRequiresHumanPerspectivesAsApprove(&agg)
if len(agg.Perspectives) < requiredPerspectiveCount(opts.MaxPerspectivesPerPR, opts.Perspectives.Len()) {
// An approve needs the coverage the all-approve path needs. A
// perspective that never reported (a dead kick, or a rate-
// limited security review the relay dropped, #16994) judged
// nothing, so the head is still under review: incomplete, and
// the planner asks for the missing perspectives.
agg.Verdict = VerdictIncomplete
agg.RequiresHuman = false
agg.DecisionRequested = false
agg.DecisionQuestion = ""
agg.Reasons = []string{incompleteCoverageReason(agg.Perspectives, opts)}
return agg
}
agg.Verdict = VerdictApprove
agg.MergeEligible = true
markRequiresHumanPerspectivesAsApprove(&agg)
agg.Reasons = nil
}
agg.RequiresHuman = false
Expand Down
Loading