Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/fixed-review-wait-starvation.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- Review dispatch now remembers heads it passed over for want of a free reviewer, promotes them ahead of webhook-prioritized PRs after three consecutive cycles, and the review queue plus Activity Monitor say how far the review got and how long the head has waited instead of a bare `stalled` pill (#15229).
6 changes: 6 additions & 0 deletions src/cmd/hive/main.go
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
package main

import (
Expand Down Expand Up @@ -11027,6 +11027,12 @@
for _, f := range plan.ForcedFixes {
logger.Warn("review fix forced: changes_requested head starved by the revisit path", "repo", f.Repo, "pr", f.Number, "head", f.HeadSHA, "reason", f.Reason, "cycles", f.Cycles, "limit", review.RevisitStarvationCycles)
}
for _, w := range plan.Promoted {
logger.Warn("review dispatch promoted: head starved of a reviewer walked ahead of the queue", "repo", w.Repo, "pr", w.Number, "head", w.HeadSHA, "reason", w.Reason, "cycles", w.Cycles, "since", w.FirstAt, "limit", review.ReviewWaitStarvationCycles)
}
for _, w := range plan.Waiting {
logger.Info("review dispatch waiting: perspectives still owed, no reviewer this cycle", "repo", w.Repo, "pr", w.Number, "head", w.HeadSHA, "reason", w.Reason)
}
if len(plan.ReviewKicks)+len(plan.FixKicks)+len(plan.FixExclusions) > 0 {
logger.Info("review swarm dispatch planned", "review_kicks", len(plan.ReviewKicks), "fix_kicks", len(plan.FixKicks), "fix_excluded", len(plan.FixExclusions), "fix_excluded_reasons", fixExclusionReasonSummary(plan.FixExclusions))
}
Expand Down
47 changes: 47 additions & 0 deletions src/pkg/dashboard/stage_tracker.go
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
package dashboard

import (
Expand Down Expand Up @@ -1554,6 +1554,39 @@
return joinClause("No stage recorded yet", clause), ""
}

// stageTrackerReviewWaitClause explains a review-pending head the swarm has
// not finished: how many perspectives have reported, and, when the planner
// has been passing the head over for lack of a reviewer, for how many
// cycles and since when (hivecommons/hive#15229). "" when neither is known.
func stageTrackerReviewWaitClause(agg *review.Aggregate, state review.DispatchState, fullRepo, repo string, number int, head string, now time.Time) string {
var parts []string
if agg != nil && agg.Verdict == review.VerdictIncomplete {
coverage := fmt.Sprintf("%s reported so far", plural(len(agg.Perspectives), "perspective"))
for _, r := range agg.Reasons {
if strings.HasPrefix(r, "review coverage incomplete") {
coverage = r
break
}
}
parts = append(parts, coverage)
}
wait, ok := state.WaitFor(fullRepo, number, head)
if !ok && fullRepo != repo {
wait, ok = state.WaitFor(repo, number, head)
}
if ok {
w := fmt.Sprintf("waiting for a free reviewer for %s", plural(wait.Cycles, "cycle"))
if !wait.FirstAt.IsZero() {
w += " since " + wait.FirstAt.UTC().Format("15:04") + " UTC"
}
if wait.Cycles >= review.ReviewWaitStarvationCycles {
w += "; promoted ahead of the queue"
}
parts = append(parts, w)
}
return strings.Join(parts, "; ")
}

// stageTrackerRegression fills the item's regression narration from the
// record: the hop that opened the regression when History still has it,
// else the record's fall time and current stage.
Expand Down Expand Up @@ -1724,6 +1757,15 @@
verdictArtifact = &art
}
verdictFor := stageTrackerVerdictLookup(verdictArtifact)
// The planner's wait records (review.DispatchState.Waits): a review-
// pending head the swarm keeps passing over for lack of a reviewer
// says so in its caption instead of a bare "stalled" pill
// (hivecommons/hive#15229). Unreadable state just leaves the caption
// as it was.
var dispatchState review.DispatchState
if st, err := review.LoadDispatchState(""); err == nil {
dispatchState = st
}
sentinelLabel := s.sentinelLabel()
lookup := func(repo string, number int) (stagelabels.Record, bool) {
if store == nil {
Expand Down Expand Up @@ -1975,6 +2017,11 @@
if a, ok := verdictFor(repoName, pr.Number, pr.HeadSHA); ok {
agg = &a
}
if it.Stage == trackerNumberReviewPending && it.Superseded == nil {
if clause := stageTrackerReviewWaitClause(agg, dispatchState, qualifyRepo(repoName), repoName, pr.Number, pr.HeadSHA, now); clause != "" {
it.Caption = joinClause(it.Caption, clause)
}
}
it.DecisionNeeded = stageTrackerDecisionNeededAt(it, agg, sentinelLabel, now)
it.StageActor = stageTrackerStageActor(it, workers, now)
add(it)
Expand Down
44 changes: 44 additions & 0 deletions src/pkg/dashboard/stage_tracker_review_wait_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
package dashboard

import (
"strings"
"testing"
"time"

"github.com/hivecommons/hive/pkg/review"
)

// hivecommons/hive#15229: a review-pending row whose head the swarm keeps
// passing over says how far the review got and how long it has waited,
// instead of a bare "stalled" pill.
func TestStageTrackerReviewWaitClauseNamesCoverageAndWait(t *testing.T) {
now := time.Date(2026, 10, 29, 1, 30, 0, 0, time.UTC)
agg := &review.Aggregate{
Verdict: review.VerdictIncomplete,
Perspectives: map[review.Perspective]review.Verdict{review.PerspectiveSecurity: review.VerdictApprove, review.PerspectiveIntentAlignment: review.VerdictApprove},
Reasons: []string{"review coverage incomplete: missing correctness, style, docs-currency"},
}
state := review.DispatchState{Waits: []review.ReviewWaitRecord{{
Repo: "hivecommons/hive", Number: 15139, HeadSHA: "cf5adb194", Reason: review.ReviewWaitNoFreeReviewer,
Cycles: review.ReviewWaitStarvationCycles + 1, FirstAt: now.Add(-40 * time.Minute),
}}}
got := stageTrackerReviewWaitClause(agg, state, "hivecommons/hive", "hive", 15139, "cf5adb194", now)
want := "review coverage incomplete: missing correctness, style, docs-currency; waiting for a free reviewer for 4 cycles since 00:50 UTC; promoted ahead of the queue"
if got != want {
t.Fatalf("clause = %q, want %q", got, want)
}

// The record is keyed by the short repo name on an older state file.
state.Waits[0].Repo = "hive"
if got := stageTrackerReviewWaitClause(nil, state, "hivecommons/hive", "hive", 15139, "cf5adb194", now); !strings.HasPrefix(got, "waiting for a free reviewer for 4 cycles") {
t.Fatalf("short-repo clause = %q", got)
}

if got := stageTrackerReviewWaitClause(nil, review.DispatchState{}, "hivecommons/hive", "hive", 15139, "cf5adb194", now); got != "" {
t.Fatalf("nothing known: clause = %q, want empty", got)
}
settled := &review.Aggregate{Verdict: review.VerdictApprove}
if got := stageTrackerReviewWaitClause(settled, review.DispatchState{}, "hivecommons/hive", "hive", 15139, "cf5adb194", now); got != "" {
t.Fatalf("settled verdict: clause = %q, want empty", got)
}
}
33 changes: 33 additions & 0 deletions src/pkg/github/review_queue.go
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
package github

import (
Expand Down Expand Up @@ -535,13 +535,46 @@
if exhausted != "" {
return "review dispatch exhausted: " + strings.TrimPrefix(exhausted, review.HoldReasonDispatchExhausted+": ") + "; waiting on a human"
}
wait, waiting := opts.Dispatch.WaitFor(full, pr.Number, pr.HeadSHA)
if !waiting && full != pr.Repo {
wait, waiting = opts.Dispatch.WaitFor(pr.Repo, pr.Number, pr.HeadSHA)
}
if since.IsZero() {
if waiting {
return reviewQueueWaitReason(wait, now)
}
return ""
}
reason := fmt.Sprintf("review dispatched %s ago, no verdict yet", reviewQueueDispatchAge(now.Sub(since)))
if count > 0 {
reason += fmt.Sprintf(" (re-dispatched %dx after expiry)", count)
}
if waiting {
reason += "; " + reviewQueueWaitReason(wait, now)
}
return reason
}

// reviewQueueWaitReason explains a head the planner keeps passing over: the
// perspectives it still owes have had no reviewer for this many consecutive
// cycles (hivecommons/hive#15229). Past review.ReviewWaitStarvationCycles the
// planner walks the head ahead of the queue, and the line says so.
func reviewQueueWaitReason(w review.ReviewWaitRecord, now time.Time) string {
cause := "every eligible reviewer busy"
if w.Reason == review.ReviewWaitNoParallelSlot {
cause = "max_parallel_reviews spent"
}
reason := fmt.Sprintf("review waiting for a free reviewer: %d consecutive cycle", w.Cycles)
if w.Cycles != 1 {
reason += "s"
}
if !w.FirstAt.IsZero() {
reason += fmt.Sprintf(" over %s", reviewQueueDispatchAge(now.Sub(w.FirstAt)))
}
reason += " (" + cause + ")"
if w.Cycles >= review.ReviewWaitStarvationCycles {
reason += "; promoted ahead of the queue, takes the next free reviewer"
}
return reason
}

Expand Down
26 changes: 26 additions & 0 deletions src/pkg/github/review_queue_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -690,3 +690,29 @@ func TestReviewQueueStageFollowsStagePolicy(t *testing.T) {
t.Fatalf("require_quiet=false: 2h since entry stalls, quiet 30m is not past dwell; got stalled=%v past_dwell=%v", st.Stalled, st.PastDwell)
}
}

// hivecommons/hive#15229: a head the planner keeps passing over for lack of
// a reviewer says so, instead of showing no dispatch reason at all.
func TestReviewQueueExplainsAHeadWaitingForAFreeReviewer(t *testing.T) {
opened := reviewQueueTestNow.Add(-2 * time.Hour)
prs := []PullRequest{
{Repo: "acme/a", Number: 1, Title: "fix: starved", HeadSHA: "h1", CIStatus: "success", CreatedAt: opened},
{Repo: "acme/a", Number: 2, Title: "fix: first wait", HeadSHA: "h2", CIStatus: "success", CreatedAt: opened},
}
q := BuildReviewQueue(prs, ReviewQueueOptions{
Org: "acme",
Now: reviewQueueTestNow,
Dispatch: review.DispatchState{Waits: []review.ReviewWaitRecord{
{Repo: "acme/a", Number: 1, HeadSHA: "h1", Reason: review.ReviewWaitNoFreeReviewer, Cycles: review.ReviewWaitStarvationCycles + 2, FirstAt: reviewQueueTestNow.Add(-50 * time.Minute)},
{Repo: "acme/a", Number: 2, HeadSHA: "h2", Reason: review.ReviewWaitNoParallelSlot, Cycles: 1, FirstAt: reviewQueueTestNow.Add(-time.Minute)},
}},
})
starved := findEntry(t, q, "acme/a", 1)
if !hasReason(starved, "review waiting for a free reviewer: 5 consecutive cycles over 50m (every eligible reviewer busy); promoted ahead of the queue, takes the next free reviewer") {
t.Fatalf("starved reasons = %v", starved.Reasons)
}
first := findEntry(t, q, "acme/a", 2)
if !hasReason(first, "review waiting for a free reviewer: 1 consecutive cycle over 1m (max_parallel_reviews spent)") || hasReason(first, "promoted") {
t.Fatalf("first-wait reasons = %v", first.Reasons)
}
}
135 changes: 134 additions & 1 deletion src/pkg/review/dispatch.go
Original file line number Diff line number Diff line change
Expand Up @@ -285,8 +285,37 @@ type DispatchState struct {
// the fix path. At RevisitStarvationCycles the planner forces the fix
// (hivecommons/hive#14665). Cleared when the head is settled normally.
RevisitStarved []RevisitStarvation `json:"revisit_starved,omitempty"`
// Waits records, per head, the consecutive cycles in which the head
// still owed perspectives but got no reviewer (plan.Waiting). Without
// it a half-reviewed head could starve indefinitely behind the webhook
// PRs the caller moves to the front of every cycle, and the stall was
// invisible: nothing recorded that the head had been passed over. At
// ReviewWaitStarvationCycles the planner moves the head ahead of the
// rest of the list; the record is dropped once it is dispatched or
// settled.
Waits []ReviewWaitRecord `json:"review_waits,omitempty"`
}

// ReviewWaitRecord is the durable form of ReviewWait: the per-head count of
// consecutive cycles a head with perspectives still owed got no reviewer.
type ReviewWaitRecord struct {
Repo string `json:"repo"`
Number int `json:"number"`
HeadSHA string `json:"head_sha,omitempty"`
Reason string `json:"reason"`
Cycles int `json:"cycles"`
FirstAt time.Time `json:"first_at"`
LastAt time.Time `json:"last_at"`
}

// ReviewWaitStarvationCycles is how many consecutive cycles a head may wait
// for a reviewer before the planner walks it ahead of every other PR,
// including the webhook-event PRs the caller promoted. Three cycles is long
// enough that an ordinary busy cycle never reorders the queue, and short
// enough that a head is not left half-reviewed for an hour while fresher PRs
// keep taking the free reviewers (hivecommons/hive#15229).
const ReviewWaitStarvationCycles = 3

// RevisitStarvation is the per-head counter behind RevisitStarvationCycles.
type RevisitStarvation struct {
Repo string `json:"repo"`
Expand Down Expand Up @@ -510,7 +539,11 @@ type DispatchPlan struct {
// spent. They are the reviewer autoscaler's demand signal
// (hivecommons/hive#14685): one entry per PR, on its current head.
Waiting []ReviewWait
State DispatchState
// Promoted are the heads this cycle walked ahead of the rest of the
// list because they had waited ReviewWaitStarvationCycles or more
// consecutive cycles for a reviewer. The caller logs each one.
Promoted []ReviewWaitRecord
State DispatchState
}

// ReviewWait is one PR left waiting for a reviewer this cycle.
Expand Down Expand Up @@ -633,6 +666,7 @@ func PlanDispatch(prs []PullRequest, artifact Artifact, state DispatchState, opt
maxParallel = DefaultMaxParallelReviews
}
availableSlots := maxParallel
prs, plan.Promoted = promoteStarvedWaits(prs, plan.State.Waits, opts.ProjectOrg)
for _, pr := range prs {
pr.Repo = fullRepoName(pr.Repo, opts.ProjectOrg)
if pr.Number <= 0 || pr.Repo == "" {
Expand Down Expand Up @@ -934,9 +968,101 @@ func PlanDispatch(prs []PullRequest, artifact Artifact, state DispatchState, opt
availableSlots--
}
}
plan.State.Waits = reconcileWaits(plan.State.Waits, plan.Waiting, opts.ProjectOrg, now)
return plan
}

// promoteStarvedWaits returns prs with every head whose wait record has
// reached ReviewWaitStarvationCycles moved to the front, in their existing
// relative order, and the records it promoted. The caller has usually
// already moved webhook-event PRs to the front; a head that has been passed
// over for that many cycles goes ahead of them, so a steady stream of fresh
// events cannot leave a half-reviewed head waiting indefinitely
// (hivecommons/hive#15229). Every gate in PlanDispatch still applies.
func promoteStarvedWaits(prs []PullRequest, waits []ReviewWaitRecord, org string) ([]PullRequest, []ReviewWaitRecord) {
if len(waits) == 0 || len(prs) < 2 {
return prs, nil
}
starved := make([]bool, len(prs))
var promoted []ReviewWaitRecord
for i, pr := range prs {
repo := fullRepoName(pr.Repo, org)
for _, w := range waits {
if w.Cycles >= ReviewWaitStarvationCycles && samePRHead(w.Repo, w.Number, w.HeadSHA, repo, pr.Number, pr.HeadSHA) {
starved[i] = true
promoted = append(promoted, w)
break
}
}
}
if len(promoted) == 0 {
return prs, nil
}
out := make([]PullRequest, 0, len(prs))
for i, pr := range prs {
if starved[i] {
out = append(out, pr)
}
}
for i, pr := range prs {
if !starved[i] {
out = append(out, pr)
}
}
return out, promoted
}

// reconcileWaits folds this cycle's Waiting into the durable wait records:
// a head still waiting gains a cycle; a head that was waiting and now is
// not (dispatched or settled) loses its record. Records for PRs no longer
// open were already dropped by pruneState.
func reconcileWaits(records []ReviewWaitRecord, waiting []ReviewWait, org string, now time.Time) []ReviewWaitRecord {
stillWaiting := make(map[string]ReviewWait, len(waiting))
for _, w := range waiting {
stillWaiting[reviewKey(fullRepoName(w.Repo, org), w.Number, w.HeadSHA, "")] = w
}
out := make([]ReviewWaitRecord, 0, len(waiting))
seen := map[string]bool{}
for _, r := range records {
key := reviewKey(fullRepoName(r.Repo, org), r.Number, r.HeadSHA, "")
w, waits := stillWaiting[key]
if !waits {
continue
}
r.Cycles++
r.Reason = w.Reason
r.LastAt = now
if r.FirstAt.IsZero() {
r.FirstAt = now
}
seen[key] = true
out = append(out, r)
}
for _, w := range waiting {
key := reviewKey(fullRepoName(w.Repo, org), w.Number, w.HeadSHA, "")
if seen[key] {
continue
}
seen[key] = true
out = append(out, ReviewWaitRecord{Repo: fullRepoName(w.Repo, org), Number: w.Number, HeadSHA: w.HeadSHA, Reason: w.Reason, Cycles: 1, FirstAt: now, LastAt: now})
}
if len(out) == 0 {
return nil
}
return out
}

// WaitFor returns the wait record for this PR head, if the planner has been
// passing it over for lack of a reviewer.
func (s DispatchState) WaitFor(repo string, number int, headSHA string) (ReviewWaitRecord, bool) {
for _, w := range s.Waits {
if samePRHead(w.Repo, w.Number, w.HeadSHA, repo, number, headSHA) {
return w, true
}
}
return ReviewWaitRecord{}, false
}

// deltaPreviousHead is the head a delta kick is judged against, "" for a
// full review.
func deltaPreviousHead(d *DeltaContext) string {
Expand Down Expand Up @@ -1905,6 +2031,13 @@ func pruneState(state DispatchState, prs []PullRequest, org string) DispatchStat
}
}
state.RevisitStarved = starved
var waits []ReviewWaitRecord
for _, w := range state.Waits {
if openHeads[reviewKey(w.Repo, w.Number, w.HeadSHA, "")] {
waits = append(waits, w)
}
}
state.Waits = waits
state = pruneReviewHeadState(state, openHeads)
return state
}
Expand Down
Loading
Loading