diff --git a/changelog.d/fixed-15559-decision-verdict-advisory.md b/changelog.d/fixed-15559-decision-verdict-advisory.md new file mode 100644 index 0000000000..f0670865e9 --- /dev/null +++ b/changelog.d/fixed-15559-decision-verdict-advisory.md @@ -0,0 +1 @@ +- A review-heads decision verdict no longer records as an advisory, undecided report when other review kicks in the same pass are delivered first, so the decision is not left to expire and run again; a verdict that falls back to advisory despite its agent having dispatch rows for the PR now logs the rows it was compared with ([#15559](https://github.com/hivecommons/hive/issues/15559)). diff --git a/src/cmd/hive/main.go b/src/cmd/hive/main.go index 3c639a2843..173452600f 100644 --- a/src/cmd/hive/main.go +++ b/src/cmd/hive/main.go @@ -7841,7 +7841,7 @@ func runEvalCycle( onReviewDelivered: func(msg scheduler.KickMessage) { if k, ok := reviewKickByMessage[msg.Agent+"\x00"+msg.Message]; ok { deliveredReviewKicks = append(deliveredReviewKicks, k) - persistReviewDispatchState(reviewPlan, deliveredReviewKicks, logger) + persistReviewDispatchStateInProgress(reviewPlan, deliveredReviewKicks, logger) } }, onFailed: func(msg scheduler.KickMessage, err error) { @@ -11411,11 +11411,28 @@ func postReviewAcknowledgements(ctx context.Context, poster reviewCommentPoster, } func persistReviewDispatchState(plan review.DispatchPlan, delivered []review.DispatchKick, logger *slog.Logger) { + writeReviewDispatchState(plan, func(planned []review.DispatchKick) review.DispatchState { + return review.ConfirmDelivered(plan.State, planned, delivered) + }, logger) +} + +// persistReviewDispatchStateInProgress is the persist made while the kick pass +// is still delivering: it keeps the pending rows of planned kicks not yet +// confirmed, so a verdict from a kick still in flight binds to its dispatch +// (hivecommons/hive#15559). The end-of-pass persistReviewDispatchState drops +// the kicks that never landed. +func persistReviewDispatchStateInProgress(plan review.DispatchPlan, delivered []review.DispatchKick, logger *slog.Logger) { + writeReviewDispatchState(plan, func([]review.DispatchKick) review.DispatchState { + return review.ConfirmDeliveredInProgress(plan.State, delivered) + }, logger) +} + +func writeReviewDispatchState(plan review.DispatchPlan, confirm func(planned []review.DispatchKick) review.DispatchState, logger *slog.Logger) { planned := append(append([]review.DispatchKick(nil), plan.ReviewKicks...), plan.FixKicks...) if plan.State.GeneratedAt.IsZero() && len(planned) == 0 { return } - state := review.ConfirmDelivered(plan.State, planned, delivered) + state := confirm(planned) if err := review.WriteDispatchState("", state); err != nil { logger.Warn("failed to persist review dispatch state", "error", err) } diff --git a/src/cmd/hive/review_dispatch_wiring_test.go b/src/cmd/hive/review_dispatch_wiring_test.go index 7ae4c8be11..5f83ee32f7 100644 --- a/src/cmd/hive/review_dispatch_wiring_test.go +++ b/src/cmd/hive/review_dispatch_wiring_test.go @@ -357,6 +357,53 @@ func TestPersistReviewDispatchState_KeepsOnlyDeliveredKicks(t *testing.T) { } } +// The persist made from onReviewDelivered runs while later kicks of the same +// pass are still being sent. It must keep their pending rows on disk, or a +// verdict from one of them finds no dispatch row and binds as advisory +// (hivecommons/hive#15559). +func TestPersistReviewDispatchStateInProgress_KeepsKicksStillDelivering(t *testing.T) { + redirectReviewPaths(t) + now := time.Now().UTC() + delivered := review.DispatchKick{ + Kind: "review", Agent: "reviewer", Repo: "hivecommons/hive", + Number: 4321, HeadSHA: reviewTestSHA, Perspective: review.PerspectiveSecurity, + } + inFlight := review.DispatchKick{ + Kind: "review", Agent: "decider", Repo: "hivecommons/hive", + Number: 4321, HeadSHA: reviewTestSHA, Perspective: review.PerspectiveStyle, Decision: true, + } + plan := review.DispatchPlan{ + ReviewKicks: []review.DispatchKick{delivered, inFlight}, + State: review.DispatchState{ + GeneratedAt: now, + Pending: []review.PendingReview{ + {Repo: delivered.Repo, Number: delivered.Number, HeadSHA: delivered.HeadSHA, Perspective: delivered.Perspective, Agent: delivered.Agent, Dispatched: now}, + {Repo: inFlight.Repo, Number: inFlight.Number, HeadSHA: inFlight.HeadSHA, Perspective: inFlight.Perspective, Agent: inFlight.Agent, Decision: true, Dispatched: now}, + }, + }, + } + + persistReviewDispatchStateInProgress(plan, []review.DispatchKick{delivered}, restoreTestLogger()) + + state, err := review.LoadDispatchState("") + if err != nil { + t.Fatalf("load persisted state: %v", err) + } + if len(state.Pending) != 2 { + t.Fatalf("persisted pending = %+v, want both the delivered and the in-flight kick", state.Pending) + } + + persistReviewDispatchState(plan, []review.DispatchKick{delivered}, restoreTestLogger()) + + state, err = review.LoadDispatchState("") + if err != nil { + t.Fatalf("load persisted state: %v", err) + } + if len(state.Pending) != 1 || state.Pending[0].Agent != delivered.Agent { + t.Fatalf("end-of-pass pending = %+v, want only the delivered kick", state.Pending) + } +} + func TestPersistReviewDispatchState_WriteFailureIsNonFatal(t *testing.T) { dir := redirectReviewPaths(t) // Point the state file inside a plain file so MkdirAll/rename must fail. diff --git a/src/pkg/github/review_verdict_relay.go b/src/pkg/github/review_verdict_relay.go index f04c724e96..5ecfe6f017 100644 --- a/src/pkg/github/review_verdict_relay.go +++ b/src/pkg/github/review_verdict_relay.go @@ -545,9 +545,45 @@ func (c *Client) verdictDispatchAuthorized(report review.PerspectiveReport, req // "awaiting review approval", and combined verdict arrays from the // reviewer recorded only the one perspective whose dispatch happened to be // assigned to that same agent. + c.logAdvisoryFallbackCandidates(state, report, req) return c.advisoryVerdictAuthorized(report, req) } +// logAdvisoryFallbackCandidates records why a verdict from an agent this hive +// dispatched to for the PR bound to no dispatch row and fell back to advisory: +// what the verdict claimed, and every row for that repo and number it was +// compared with (hivecommons/hive#15559). A verdict from an agent with no row +// for the PR is an ordinary advisory review and logs nothing. +func (c *Client) logAdvisoryFallbackCandidates(state review.DispatchState, report review.PerspectiveReport, req ReviewRequest) { + agent := strings.TrimSpace(req.Agent) + var candidates []string + agentHasRow := false + add := func(kind, repo string, number int, rowAgent string, perspective review.Perspective, headSHA string, decision bool) { + if number != report.Number || !strings.EqualFold(strings.TrimSpace(repo), strings.TrimSpace(report.Repo)) { + return + } + if strings.TrimSpace(rowAgent) == agent { + agentHasRow = true + } + candidates = append(candidates, fmt.Sprintf("%s agent=%s perspective=%s head=%s decision=%t", kind, strings.TrimSpace(rowAgent), perspective, strings.TrimSpace(headSHA), decision)) + } + for _, p := range state.Pending { + add("pending", p.Repo, p.Number, p.Agent, p.Perspective, p.HeadSHA, p.Decision) + } + for _, r := range state.Recent { + add("recent", r.Repo, r.Number, r.Agent, r.Perspective, r.HeadSHA, r.Decision) + } + if !agentHasRow || c.logger == nil { + return + } + c.logger.Info("review-request watcher: verdict matched no dispatch row for its agent, falling back to advisory", + slog.String("reviewed", fmt.Sprintf("%s#%d", req.Repo, req.Number)), + slog.String("agent", agent), + slog.String("perspective", string(report.Perspective)), + slog.String("reported_head", strings.TrimSpace(report.HeadSHA)), + slog.String("candidates", strings.Join(candidates, "; "))) +} + // advisoryVerdictAuthorized admits a verdict no dispatch row binds. The // verdict feeds the same aggregate that gates merge eligibility and // recommends closing a PR, so the submitting lane must be one this hive diff --git a/src/pkg/github/review_verdict_relay_test.go b/src/pkg/github/review_verdict_relay_test.go index 8a304226ce..583efd8400 100644 --- a/src/pkg/github/review_verdict_relay_test.go +++ b/src/pkg/github/review_verdict_relay_test.go @@ -1,9 +1,11 @@ package github import ( + "bytes" "compress/gzip" "context" "encoding/json" + "log/slog" "os" "path/filepath" "strings" @@ -313,6 +315,49 @@ func TestRecordReviewVerdictAcceptsAdvisoryVerdictWhenDispatchStateHasNoMatch(t } } +// A verdict from an agent that does hold dispatch rows for the PR, but binds to +// none of them, logs what it was compared with before it falls back to +// advisory, so a lost decision row can be traced (hivecommons/hive#15559). +func TestRecordReviewVerdictLogsCandidatesOnAdvisoryFallback(t *testing.T) { + dir := t.TempDir() + reportDir := filepath.Join(dir, "reports") + var logBuf bytes.Buffer + c := &Client{logger: slog.New(slog.NewTextHandler(&logBuf, nil))} + raw := validVerdictJSONWithHead(t, "o/r", 1, "security", "approve", "abc123") + row := pendingVerdict("o/r", 1, review.PerspectiveCorrectness, "reviewer") + row.HeadSHA = "abc123" + row.Decision = true + withVerdictDispatchState(t, review.DispatchState{Pending: []review.PendingReview{row}}) + + c.recordReviewVerdict(context.Background(), ReviewRequest{Repo: "o/r", Number: 1, Agent: "reviewer", Report: raw}, reportDir) + + out := logBuf.String() + for _, want := range []string{"falling back to advisory", "agent=reviewer", "perspective=security", "reported_head=abc123", "pending agent=reviewer perspective=correctness head=abc123 decision=true"} { + if !strings.Contains(out, want) { + t.Errorf("advisory fallback log missing %q:\n%s", want, out) + } + } +} + +// An agent with no dispatch row for the PR is an ordinary advisory reviewer; +// its fallback is not worth a diagnostic line. +func TestRecordReviewVerdictAdvisoryFallbackQuietWithoutAgentRows(t *testing.T) { + dir := t.TempDir() + reportDir := filepath.Join(dir, "reports") + var logBuf bytes.Buffer + c := &Client{logger: slog.New(slog.NewTextHandler(&logBuf, nil))} + raw := validVerdictJSONWithHead(t, "o/r", 1, "security", "approve", "abc123") + withVerdictDispatchState(t, review.DispatchState{Pending: []review.PendingReview{ + pendingVerdict("o/r", 1, review.PerspectiveSecurity, "other-reviewer"), + }}) + + c.recordReviewVerdict(context.Background(), ReviewRequest{Repo: "o/r", Number: 1, Agent: "reviewer", Report: raw}, reportDir) + + if strings.Contains(logBuf.String(), "falling back to advisory") { + t.Fatalf("advisory fallback logged for an agent with no dispatch rows:\n%s", logBuf.String()) + } +} + // A lane this hive does not review with must not be able to record a verdict // that no dispatch row binds: the verdict feeds the merge gate and the // close recommendation, so a non-reviewer that can reach the relay could diff --git a/src/pkg/review/dispatch.go b/src/pkg/review/dispatch.go index 0a62a09747..5e3b3c75f4 100644 --- a/src/pkg/review/dispatch.go +++ b/src/pkg/review/dispatch.go @@ -1638,6 +1638,17 @@ func selectFixerAgent(pr PullRequest, opts DispatchOptions) string { return "" } +// ConfirmDeliveredInProgress is ConfirmDelivered for a kick pass that is still +// delivering: it records the kicks delivered so far and drops nothing for the +// planned kicks not yet delivered. A kick still being sent may already have +// reached its agent, and the verdict relay binds that agent's verdict to the +// pending rows on disk; dropping them at an intermediate persist bound a +// decision verdict as advisory (hivecommons/hive#15559). The end-of-pass +// persist uses ConfirmDelivered to release the kicks that never landed. +func ConfirmDeliveredInProgress(state DispatchState, delivered []DispatchKick) DispatchState { + return ConfirmDelivered(state, delivered, delivered) +} + func ConfirmDelivered(state DispatchState, planned, delivered []DispatchKick) DispatchState { now := time.Now().UTC() deliveredSet := map[string]bool{} diff --git a/src/pkg/review/dispatch_test.go b/src/pkg/review/dispatch_test.go index a476929a7a..49f832cfc8 100644 --- a/src/pkg/review/dispatch_test.go +++ b/src/pkg/review/dispatch_test.go @@ -135,6 +135,32 @@ func TestConfirmDeliveredRollsBackFailedKicks(t *testing.T) { } } +// A persist made while the kick pass is still delivering must keep the pending +// rows of a planned kick that has not been confirmed yet: it may already have +// reached its agent, and its verdict binds to those rows (hivecommons/hive#15559). +func TestConfirmDeliveredInProgressKeepsKicksStillDelivering(t *testing.T) { + first := DispatchKick{Kind: "review", Agent: "r1", Repo: "acme/hive", Number: 7, HeadSHA: "sha1", Perspective: PerspectiveCorrectness} + decision := DispatchKick{Kind: "review", Agent: "base", Repo: "acme/hive", Number: 7, HeadSHA: "sha1", Perspective: PerspectiveSecurity, Perspectives: []Perspective{PerspectiveSecurity, PerspectiveStyle}, Decision: true} + state := DispatchState{Pending: []PendingReview{ + {Repo: "acme/hive", Number: 7, HeadSHA: "sha1", Perspective: PerspectiveCorrectness, Agent: "r1"}, + {Repo: "acme/hive", Number: 7, HeadSHA: "sha1", Perspective: PerspectiveSecurity, Agent: "base", Decision: true}, + {Repo: "acme/hive", Number: 7, HeadSHA: "sha1", Perspective: PerspectiveStyle, Agent: "base", Decision: true}, + }} + + got := ConfirmDeliveredInProgress(state, []DispatchKick{first}) + if len(got.Pending) != 3 { + t.Fatalf("in-progress persist dropped pending rows of a kick still delivering: %+v", got.Pending) + } + if len(got.Recent) != 1 || got.Recent[0].Agent != "r1" { + t.Fatalf("in-progress persist did not record the delivered kick: %+v", got.Recent) + } + + final := ConfirmDelivered(got, []DispatchKick{first, decision}, []DispatchKick{first}) + if len(final.Pending) != 1 || final.Pending[0].Agent != "r1" { + t.Fatalf("end-of-pass persist kept rows of an undelivered kick: %+v", final.Pending) + } +} + func TestBuildFixPromptAndCapExhaustion(t *testing.T) { finding := PerspectiveFinding{Perspective: PerspectiveCorrectness, Finding: outputschema.Finding{Title: "nil panic", Severity: outputschema.SeverityMedium, Summary: "guard nil input"}} agg := Aggregate{Repo: "acme/hive", Number: 7, HeadSHA: "sha1", Verdict: VerdictChangesRequested, Findings: []PerspectiveFinding{finding}}