Skip to content
1 change: 1 addition & 0 deletions changelog.d/fixed-15559-decision-verdict-advisory.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- A review-heads decision verdict no longer records as an advisory, undecided report when other review kicks in the same pass are delivered first, so the decision is not left to expire and run again; a verdict that falls back to advisory despite its agent having dispatch rows for the PR now logs the rows it was compared with ([#15559](https://github.com/hivecommons/hive/issues/15559)).
21 changes: 19 additions & 2 deletions src/cmd/hive/main.go
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
package main

import (
Expand Down Expand Up @@ -7841,7 +7841,7 @@
onReviewDelivered: func(msg scheduler.KickMessage) {
if k, ok := reviewKickByMessage[msg.Agent+"\x00"+msg.Message]; ok {
deliveredReviewKicks = append(deliveredReviewKicks, k)
persistReviewDispatchState(reviewPlan, deliveredReviewKicks, logger)
persistReviewDispatchStateInProgress(reviewPlan, deliveredReviewKicks, logger)
}
},
onFailed: func(msg scheduler.KickMessage, err error) {
Expand Down Expand Up @@ -11411,11 +11411,28 @@
}

func persistReviewDispatchState(plan review.DispatchPlan, delivered []review.DispatchKick, logger *slog.Logger) {
writeReviewDispatchState(plan, func(planned []review.DispatchKick) review.DispatchState {
return review.ConfirmDelivered(plan.State, planned, delivered)
}, logger)
}

// persistReviewDispatchStateInProgress is the persist made while the kick pass
// is still delivering: it keeps the pending rows of planned kicks not yet
// confirmed, so a verdict from a kick still in flight binds to its dispatch
// (hivecommons/hive#15559). The end-of-pass persistReviewDispatchState drops
// the kicks that never landed.
func persistReviewDispatchStateInProgress(plan review.DispatchPlan, delivered []review.DispatchKick, logger *slog.Logger) {
writeReviewDispatchState(plan, func([]review.DispatchKick) review.DispatchState {
return review.ConfirmDeliveredInProgress(plan.State, delivered)
}, logger)
}

func writeReviewDispatchState(plan review.DispatchPlan, confirm func(planned []review.DispatchKick) review.DispatchState, logger *slog.Logger) {
planned := append(append([]review.DispatchKick(nil), plan.ReviewKicks...), plan.FixKicks...)
if plan.State.GeneratedAt.IsZero() && len(planned) == 0 {
return
}
state := review.ConfirmDelivered(plan.State, planned, delivered)
state := confirm(planned)
if err := review.WriteDispatchState("", state); err != nil {
logger.Warn("failed to persist review dispatch state", "error", err)
}
Expand Down
47 changes: 47 additions & 0 deletions src/cmd/hive/review_dispatch_wiring_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -357,6 +357,53 @@ func TestPersistReviewDispatchState_KeepsOnlyDeliveredKicks(t *testing.T) {
}
}

// The persist made from onReviewDelivered runs while later kicks of the same
// pass are still being sent. It must keep their pending rows on disk, or a
// verdict from one of them finds no dispatch row and binds as advisory
// (hivecommons/hive#15559).
func TestPersistReviewDispatchStateInProgress_KeepsKicksStillDelivering(t *testing.T) {
redirectReviewPaths(t)
now := time.Now().UTC()
delivered := review.DispatchKick{
Kind: "review", Agent: "reviewer", Repo: "hivecommons/hive",
Number: 4321, HeadSHA: reviewTestSHA, Perspective: review.PerspectiveSecurity,
}
inFlight := review.DispatchKick{
Kind: "review", Agent: "decider", Repo: "hivecommons/hive",
Number: 4321, HeadSHA: reviewTestSHA, Perspective: review.PerspectiveStyle, Decision: true,
}
plan := review.DispatchPlan{
ReviewKicks: []review.DispatchKick{delivered, inFlight},
State: review.DispatchState{
GeneratedAt: now,
Pending: []review.PendingReview{
{Repo: delivered.Repo, Number: delivered.Number, HeadSHA: delivered.HeadSHA, Perspective: delivered.Perspective, Agent: delivered.Agent, Dispatched: now},
{Repo: inFlight.Repo, Number: inFlight.Number, HeadSHA: inFlight.HeadSHA, Perspective: inFlight.Perspective, Agent: inFlight.Agent, Decision: true, Dispatched: now},
},
},
}

persistReviewDispatchStateInProgress(plan, []review.DispatchKick{delivered}, restoreTestLogger())

state, err := review.LoadDispatchState("")
if err != nil {
t.Fatalf("load persisted state: %v", err)
}
if len(state.Pending) != 2 {
t.Fatalf("persisted pending = %+v, want both the delivered and the in-flight kick", state.Pending)
}

persistReviewDispatchState(plan, []review.DispatchKick{delivered}, restoreTestLogger())

state, err = review.LoadDispatchState("")
if err != nil {
t.Fatalf("load persisted state: %v", err)
}
if len(state.Pending) != 1 || state.Pending[0].Agent != delivered.Agent {
t.Fatalf("end-of-pass pending = %+v, want only the delivered kick", state.Pending)
}
}

func TestPersistReviewDispatchState_WriteFailureIsNonFatal(t *testing.T) {
dir := redirectReviewPaths(t)
// Point the state file inside a plain file so MkdirAll/rename must fail.
Expand Down
36 changes: 36 additions & 0 deletions src/pkg/github/review_verdict_relay.go
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
package github

import (
Expand Down Expand Up @@ -545,9 +545,45 @@
// "awaiting review approval", and combined verdict arrays from the
// reviewer recorded only the one perspective whose dispatch happened to be
// assigned to that same agent.
c.logAdvisoryFallbackCandidates(state, report, req)
return c.advisoryVerdictAuthorized(report, req)
}

// logAdvisoryFallbackCandidates records why a verdict from an agent this hive
// dispatched to for the PR bound to no dispatch row and fell back to advisory:
// what the verdict claimed, and every row for that repo and number it was
// compared with (hivecommons/hive#15559). A verdict from an agent with no row
// for the PR is an ordinary advisory review and logs nothing.
func (c *Client) logAdvisoryFallbackCandidates(state review.DispatchState, report review.PerspectiveReport, req ReviewRequest) {
agent := strings.TrimSpace(req.Agent)
var candidates []string
agentHasRow := false
add := func(kind, repo string, number int, rowAgent string, perspective review.Perspective, headSHA string, decision bool) {
if number != report.Number || !strings.EqualFold(strings.TrimSpace(repo), strings.TrimSpace(report.Repo)) {
return
}
if strings.TrimSpace(rowAgent) == agent {
agentHasRow = true
}
candidates = append(candidates, fmt.Sprintf("%s agent=%s perspective=%s head=%s decision=%t", kind, strings.TrimSpace(rowAgent), perspective, strings.TrimSpace(headSHA), decision))
}
for _, p := range state.Pending {
add("pending", p.Repo, p.Number, p.Agent, p.Perspective, p.HeadSHA, p.Decision)
}
for _, r := range state.Recent {
add("recent", r.Repo, r.Number, r.Agent, r.Perspective, r.HeadSHA, r.Decision)
}
if !agentHasRow || c.logger == nil {
return
}
c.logger.Info("review-request watcher: verdict matched no dispatch row for its agent, falling back to advisory",
slog.String("reviewed", fmt.Sprintf("%s#%d", req.Repo, req.Number)),
slog.String("agent", agent),
slog.String("perspective", string(report.Perspective)),
slog.String("reported_head", strings.TrimSpace(report.HeadSHA)),
slog.String("candidates", strings.Join(candidates, "; ")))
}

// advisoryVerdictAuthorized admits a verdict no dispatch row binds. The
// verdict feeds the same aggregate that gates merge eligibility and
// recommends closing a PR, so the submitting lane must be one this hive
Expand Down
45 changes: 45 additions & 0 deletions src/pkg/github/review_verdict_relay_test.go
Original file line number Diff line number Diff line change
@@ -1,9 +1,11 @@
package github

import (
"bytes"
"compress/gzip"
"context"
"encoding/json"
"log/slog"
"os"
"path/filepath"
"strings"
Expand Down Expand Up @@ -313,6 +315,49 @@ func TestRecordReviewVerdictAcceptsAdvisoryVerdictWhenDispatchStateHasNoMatch(t
}
}

// A verdict from an agent that does hold dispatch rows for the PR, but binds to
// none of them, logs what it was compared with before it falls back to
// advisory, so a lost decision row can be traced (hivecommons/hive#15559).
func TestRecordReviewVerdictLogsCandidatesOnAdvisoryFallback(t *testing.T) {
dir := t.TempDir()
reportDir := filepath.Join(dir, "reports")
var logBuf bytes.Buffer
c := &Client{logger: slog.New(slog.NewTextHandler(&logBuf, nil))}
raw := validVerdictJSONWithHead(t, "o/r", 1, "security", "approve", "abc123")
row := pendingVerdict("o/r", 1, review.PerspectiveCorrectness, "reviewer")
row.HeadSHA = "abc123"
row.Decision = true
withVerdictDispatchState(t, review.DispatchState{Pending: []review.PendingReview{row}})

c.recordReviewVerdict(context.Background(), ReviewRequest{Repo: "o/r", Number: 1, Agent: "reviewer", Report: raw}, reportDir)

out := logBuf.String()
for _, want := range []string{"falling back to advisory", "agent=reviewer", "perspective=security", "reported_head=abc123", "pending agent=reviewer perspective=correctness head=abc123 decision=true"} {
if !strings.Contains(out, want) {
t.Errorf("advisory fallback log missing %q:\n%s", want, out)
}
}
}

// An agent with no dispatch row for the PR is an ordinary advisory reviewer;
// its fallback is not worth a diagnostic line.
func TestRecordReviewVerdictAdvisoryFallbackQuietWithoutAgentRows(t *testing.T) {
dir := t.TempDir()
reportDir := filepath.Join(dir, "reports")
var logBuf bytes.Buffer
c := &Client{logger: slog.New(slog.NewTextHandler(&logBuf, nil))}
raw := validVerdictJSONWithHead(t, "o/r", 1, "security", "approve", "abc123")
withVerdictDispatchState(t, review.DispatchState{Pending: []review.PendingReview{
pendingVerdict("o/r", 1, review.PerspectiveSecurity, "other-reviewer"),
}})

c.recordReviewVerdict(context.Background(), ReviewRequest{Repo: "o/r", Number: 1, Agent: "reviewer", Report: raw}, reportDir)

if strings.Contains(logBuf.String(), "falling back to advisory") {
t.Fatalf("advisory fallback logged for an agent with no dispatch rows:\n%s", logBuf.String())
}
}

// A lane this hive does not review with must not be able to record a verdict
// that no dispatch row binds: the verdict feeds the merge gate and the
// close recommendation, so a non-reviewer that can reach the relay could
Expand Down
11 changes: 11 additions & 0 deletions src/pkg/review/dispatch.go
Original file line number Diff line number Diff line change
Expand Up @@ -1638,6 +1638,17 @@ func selectFixerAgent(pr PullRequest, opts DispatchOptions) string {
return ""
}

// ConfirmDeliveredInProgress is ConfirmDelivered for a kick pass that is still
// delivering: it records the kicks delivered so far and drops nothing for the
// planned kicks not yet delivered. A kick still being sent may already have
// reached its agent, and the verdict relay binds that agent's verdict to the
// pending rows on disk; dropping them at an intermediate persist bound a
// decision verdict as advisory (hivecommons/hive#15559). The end-of-pass
// persist uses ConfirmDelivered to release the kicks that never landed.
func ConfirmDeliveredInProgress(state DispatchState, delivered []DispatchKick) DispatchState {
return ConfirmDelivered(state, delivered, delivered)
}

func ConfirmDelivered(state DispatchState, planned, delivered []DispatchKick) DispatchState {
now := time.Now().UTC()
deliveredSet := map[string]bool{}
Expand Down
26 changes: 26 additions & 0 deletions src/pkg/review/dispatch_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -135,6 +135,32 @@ func TestConfirmDeliveredRollsBackFailedKicks(t *testing.T) {
}
}

// A persist made while the kick pass is still delivering must keep the pending
// rows of a planned kick that has not been confirmed yet: it may already have
// reached its agent, and its verdict binds to those rows (hivecommons/hive#15559).
func TestConfirmDeliveredInProgressKeepsKicksStillDelivering(t *testing.T) {
first := DispatchKick{Kind: "review", Agent: "r1", Repo: "acme/hive", Number: 7, HeadSHA: "sha1", Perspective: PerspectiveCorrectness}
decision := DispatchKick{Kind: "review", Agent: "base", Repo: "acme/hive", Number: 7, HeadSHA: "sha1", Perspective: PerspectiveSecurity, Perspectives: []Perspective{PerspectiveSecurity, PerspectiveStyle}, Decision: true}
state := DispatchState{Pending: []PendingReview{
{Repo: "acme/hive", Number: 7, HeadSHA: "sha1", Perspective: PerspectiveCorrectness, Agent: "r1"},
{Repo: "acme/hive", Number: 7, HeadSHA: "sha1", Perspective: PerspectiveSecurity, Agent: "base", Decision: true},
{Repo: "acme/hive", Number: 7, HeadSHA: "sha1", Perspective: PerspectiveStyle, Agent: "base", Decision: true},
}}

got := ConfirmDeliveredInProgress(state, []DispatchKick{first})
if len(got.Pending) != 3 {
t.Fatalf("in-progress persist dropped pending rows of a kick still delivering: %+v", got.Pending)
}
if len(got.Recent) != 1 || got.Recent[0].Agent != "r1" {
t.Fatalf("in-progress persist did not record the delivered kick: %+v", got.Recent)
}

final := ConfirmDelivered(got, []DispatchKick{first, decision}, []DispatchKick{first})
if len(final.Pending) != 1 || final.Pending[0].Agent != "r1" {
t.Fatalf("end-of-pass persist kept rows of an undelivered kick: %+v", final.Pending)
}
}

func TestBuildFixPromptAndCapExhaustion(t *testing.T) {
finding := PerspectiveFinding{Perspective: PerspectiveCorrectness, Finding: outputschema.Finding{Title: "nil panic", Severity: outputschema.SeverityMedium, Summary: "guard nil input"}}
agg := Aggregate{Repo: "acme/hive", Number: 7, HeadSHA: "sha1", Verdict: VerdictChangesRequested, Findings: []PerspectiveFinding{finding}}
Expand Down
Loading