diff --git a/changelog.d/fixed-15744-fixer-exhausted-needs-decision.md b/changelog.d/fixed-15744-fixer-exhausted-needs-decision.md new file mode 100644 index 0000000000..511d241be8 --- /dev/null +++ b/changelog.d/fixed-15744-fixer-exhausted-needs-decision.md @@ -0,0 +1 @@ +- Hive-authored PRs whose review fix dispatch is refused with `fix-attempts-exhausted` now move from `stage/070-fixing` to `stage/085-needs-decision` (reason `fixer-exhausted:`) with a `fixer gave up after N attempts — ` caption, a once-per-head decision comment and **Re-author** as the tracker's lead action; a 070 row no longer claims `Fixing · @bot` while no fix job is dispatched, a stale answered decision no longer clears `stage/stalled` while the fixer is excluded and the head is unchanged, and the `review fix dispatch excluded` warning is logged once per PR head instead of every cycle ([#15744](https://github.com/hivecommons/hive/issues/15744)). diff --git a/src/cmd/hive/main.go b/src/cmd/hive/main.go index 5a6088e168..dc9cf334f9 100644 --- a/src/cmd/hive/main.go +++ b/src/cmd/hive/main.go @@ -2202,6 +2202,7 @@ func (b *boot) bootAgentsWith(deps bootAgentsDeps) { } return automerge.RebaseEpisode{Attempt: a.Count(), MaxAttempts: rebasemerge.MaxRebaseAttempts, Exhausted: a.Exhausted(), RequestedBy: a.RequestedBy}, true } + autoMergeOpts.FixDispatchStatus = fixDispatchStatusReader() autoMergeOpts.MergeTrain = func() automerge.MergeTrainPolicy { if b.cfg == nil { return automerge.DefaultMergeTrainPolicy diff --git a/src/cmd/hive/main_sweeps.go b/src/cmd/hive/main_sweeps.go index 394c7fb005..7583a59f19 100644 --- a/src/cmd/hive/main_sweeps.go +++ b/src/cmd/hive/main_sweeps.go @@ -2832,12 +2832,19 @@ type reviewFixAuditor interface { func auditWithheldReviewFixes(plan review.DispatchPlan, auditor reviewFixAuditor, logger *slog.Logger) { // One WARN per changes_requested head that ends the cycle without a fix // kick: a PR shown as "Fixing" that nobody is fixing must be loud - // (hivecommons/hive#14665). + // (hivecommons/hive#14665) — once per (pr, head, reason), Debug while + // the same exclusion persists (hivecommons/hive#15744). + active := make(map[string]struct{}, len(plan.FixExclusions)) + defer func() { reviewFixExcludedWarn.retain(active) }() for _, x := range plan.FixExclusions { - level := slog.LevelWarn + key := reviewFixExcludedWarnKey(x) + active[key] = struct{}{} + level := reviewFixExcludedWarn.level(key) if x.Reason == review.FixDispatchExcludedNotHiveAuthored || x.Reason == review.FixDispatchExcludedNotHiveAuthoredDCO { // Policy skip (the hive never pushes to PRs it did not author), not a fault. - level = slog.LevelInfo + if level == slog.LevelWarn { + level = slog.LevelInfo + } } logger.Log(context.Background(), level, "review fix dispatch excluded", "repo", x.Repo, "pr", x.Number, "head_sha", x.HeadSHA, "author", x.Author, "reason", x.Reason, "attempts", x.Attempts) @@ -2853,6 +2860,41 @@ func auditWithheldReviewFixes(plan review.DispatchPlan, auditor reviewFixAuditor } } +func reviewFixExcludedWarnKey(x review.FixDispatchExclusion) string { + return prWarnKey(x.Repo, x.Number) + "@" + strings.ToLower(strings.TrimSpace(x.HeadSHA)) + ":" + x.Reason +} + +// fixDispatchStatusReader is the automerge engine's view of the review +// fixer's standing on a PR head (hivecommons/hive#15744): the persisted +// dispatch state, re-read per call so a stage pass sees the fixer's latest +// verdict. No state (fresh hive, unreadable file) reads as unknown. +func fixDispatchStatusReader() func(repo string, number int, head string) (review.FixDispatchStatus, bool) { + var ( + mu sync.Mutex + state review.DispatchState + modTime time.Time + size int64 + loaded bool + ) + return func(repo string, number int, head string) (review.FixDispatchStatus, bool) { + mu.Lock() + defer mu.Unlock() + // A stage pass asks once per PR; re-parse only when the file moved. + if fi, err := os.Stat(review.ReviewDispatchStatePath); err == nil && loaded && fi.ModTime().Equal(modTime) && fi.Size() == size { + return review.FixDispatchStatusFor(state, repo, number, head), true + } else if err == nil { + modTime, size = fi.ModTime(), fi.Size() + } + st, err := review.LoadDispatchState("") + if err != nil { + loaded = false + return review.FixDispatchStatus{}, false + } + state, loaded = st, true + return review.FixDispatchStatusFor(state, repo, number, head), true + } +} + // reviewMissingHeads reads the review heads that ran out of retries while // another head answered (hivecommons/hive#13542), so their perspectives count // as settled for the decision step. No state is none. diff --git a/src/cmd/hive/review_fix_withheld_audit_test.go b/src/cmd/hive/review_fix_withheld_audit_test.go index 1531948cd7..b5758328bf 100644 --- a/src/cmd/hive/review_fix_withheld_audit_test.go +++ b/src/cmd/hive/review_fix_withheld_audit_test.go @@ -1,8 +1,10 @@ package main import ( + "context" "io" "log/slog" + "slices" "strings" "testing" "time" @@ -82,3 +84,63 @@ func TestAuditWithheldReviewFixesNotHiveAuthoredExclusionIsNotWarn(t *testing.T) t.Fatalf("not-hive-authored exclusion must log below WARN, got: %s", out) } } + +type levelRecorder struct { + levels []slog.Level +} + +func (r *levelRecorder) Enabled(context.Context, slog.Level) bool { return true } +func (r *levelRecorder) Handle(_ context.Context, rec slog.Record) error { + if rec.Message == "review fix dispatch excluded" { + r.levels = append(r.levels, rec.Level) + } + return nil +} +func (r *levelRecorder) WithAttrs([]slog.Attr) slog.Handler { return r } +func (r *levelRecorder) WithGroup(string) slog.Handler { return r } + +// TestAuditWithheldReviewFixesWarnsOncePerHead: the dispatcher re-derives +// its exclusions every cycle, so the same fix-attempts-exhausted line +// fired every ~5 min for 13h on #15308 (hivecommons/hive#15744). WARN +// once per (pr, head, reason); Debug while it persists; WARN again on a +// new head or after the exclusion cleared and came back. +func TestAuditWithheldReviewFixesWarnsOncePerHead(t *testing.T) { + reviewFixExcludedWarn = prWarnOnce{} + t.Cleanup(func() { reviewFixExcludedWarn = prWarnOnce{} }) + rec := &levelRecorder{} + logger := slog.New(rec) + exhausted := review.FixDispatchExclusion{Repo: "hivecommons/hive", Number: 15308, HeadSHA: "5b62304", Author: "hivecommons-hive[bot]", Reason: review.FixDispatchExcludedAttemptsExhausted, Attempts: 6} + plan := review.DispatchPlan{FixExclusions: []review.FixDispatchExclusion{exhausted}} + + auditWithheldReviewFixes(plan, nil, logger) + auditWithheldReviewFixes(plan, nil, logger) + auditWithheldReviewFixes(plan, nil, logger) + if want := []slog.Level{slog.LevelWarn, slog.LevelDebug, slog.LevelDebug}; !slices.Equal(rec.levels, want) { + t.Fatalf("levels across three cycles = %v, want %v", rec.levels, want) + } + + moved := exhausted + moved.HeadSHA = "9f0e1d2" + rec.levels = nil + auditWithheldReviewFixes(review.DispatchPlan{FixExclusions: []review.FixDispatchExclusion{moved}}, nil, logger) + if want := []slog.Level{slog.LevelWarn}; !slices.Equal(rec.levels, want) { + t.Fatalf("new head levels = %v, want %v", rec.levels, want) + } + + // Cleared for a cycle, then back: the old key was retained away. + rec.levels = nil + auditWithheldReviewFixes(review.DispatchPlan{}, nil, logger) + auditWithheldReviewFixes(plan, nil, logger) + if want := []slog.Level{slog.LevelWarn}; !slices.Equal(rec.levels, want) { + t.Fatalf("recurrence levels = %v, want %v", rec.levels, want) + } + + // Policy skips stay Info the first time, Debug after. + rec.levels = nil + policy := review.DispatchPlan{FixExclusions: []review.FixDispatchExclusion{{Repo: "acme/x", Number: 1, HeadSHA: "a", Reason: review.FixDispatchExcludedNotHiveAuthored}}} + auditWithheldReviewFixes(policy, nil, logger) + auditWithheldReviewFixes(policy, nil, logger) + if want := []slog.Level{slog.LevelInfo, slog.LevelDebug}; !slices.Equal(rec.levels, want) { + t.Fatalf("policy skip levels = %v, want %v", rec.levels, want) + } +} diff --git a/src/cmd/hive/warn_once_pr.go b/src/cmd/hive/warn_once_pr.go index 7651d2f9c1..d2b556ec21 100644 --- a/src/cmd/hive/warn_once_pr.go +++ b/src/cmd/hive/warn_once_pr.go @@ -47,4 +47,9 @@ var ( humanLabelWarn prWarnOnce reviewExpiredWarn prWarnOnce reviewPromotedWarn prWarnOnce + // reviewFixExcludedWarn keys on (repo, pr, head, reason): the fix + // dispatcher re-derives its exclusions every cycle, so without it the + // same "fix-attempts-exhausted" line fired every ~5 min for 13h on + // hive's own #15308 (hivecommons/hive#15744). + reviewFixExcludedWarn prWarnOnce ) diff --git a/src/docs/hive-merge.md b/src/docs/hive-merge.md index be395258d7..702961a2eb 100644 --- a/src/docs/hive-merge.md +++ b/src/docs/hive-merge.md @@ -860,7 +860,11 @@ human-authored PR → `stage/071-needs-author`, on a clanker PR → review or `/hive decision ` on a hive-authored PR's current head → `stage/070-fixing` with reason `human-changes-requested` and one fix kick carrying the text ([#13041](https://github.com/hivecommons/hive/issues/13041)), -hold / sentinel alert → `stage/085-needs-decision`, untrusted author with an approve verdict at the +hold / sentinel alert → `stage/085-needs-decision`, hive-authored PR whose fix +dispatch is refused with `fix-attempts-exhausted` → `stage/085-needs-decision` +with reason `fixer-exhausted:` and a once-per-head decision comment (re-author / +hand to a person / close as not planned; a stale `decision-answered` cannot pull it +back to 070 while the head is unchanged, [#15744](https://github.com/hivecommons/hive/issues/15744)), untrusted author with an approve verdict at the current head, green checks, DCO and soak done → `stage/090-ready-for-human-merge` (only a maintainer's `/lgtm` `/approve` is missing; the dashboard derives a compact **human-merge reason code** from the diff --git a/src/docs/review-swarm.md b/src/docs/review-swarm.md index b53beea683..341d71cf0b 100644 --- a/src/docs/review-swarm.md +++ b/src/docs/review-swarm.md @@ -553,6 +553,13 @@ The PR review queue (`GET /api/review/queue`) shows this state on unreviewed ite When an aggregate verdict is `changes_requested`, Hive builds a review-fix kick containing the aggregate findings and sends it to `review.fixer_agent`, the classified PR lane, or `scanner`. Fix dispatches are capped by `escalation.MaxReEngagements`; once exhausted, dispatch state records a `requires_human` hold for the PR head so the automated loop stops. +Once that cap is hit the hive is no longer fixing the PR, and the stage must say so ([#15744](https://github.com/hivecommons/hive/issues/15744)): + +- The stage engine moves a hive-authored PR the dispatcher refuses with `fix-attempts-exhausted` from `stage/070-fixing` to `stage/085-needs-decision` with reason `fixer-exhausted:`. The tracker captions it `fixer gave up after N attempts — `, and the lead action is **Re-author** (`decide:re-author`); *Hand it to a person* (`/hive decision `) and *Close as not planned* sit in the Decide… submenu. One decision comment per head lists the same three options; it goes through the normal write-back gate. +- A `070-fixing` row never claims `Fixing · @bot` when no fix job is dispatched or running. If the dispatcher excludes the head (attempts exhausted, no enabled fixer, budget), the caption reads `Fixing paused — ; no fix job is dispatched` (plus `conflicting with the base branch` when the PR is dirty) and the stage actor is *nobody*. +- A stale answered decision (`decision-answered`) does not clear `stage/stalled` or move the PR back to 070 while the fixer is excluded and the head has not changed; the 085 park is preserved. +- The `review fix dispatch excluded … reason=fix-attempts-exhausted` WARN is logged once per (PR, head, reason); later cycles log it at debug. + ### Who may be pushed to: `all_authors` and the retired `fix_human_prs` push A review-fix kick tells the fixer agent to check out the PR branch and push a commit to it. That is fine on a PR one of the hive's own agents opened; on a contributor's fork (reachable through GitHub's "allow edits by maintainers") or a maintainer's branch it is a bot rewriting someone else's work before any person has looked at it. Reviewing and pushing are therefore separate: diff --git a/src/pkg/dashboard/stage_tracker_build.go b/src/pkg/dashboard/stage_tracker_build.go index 01de767567..a21069a083 100644 --- a/src/pkg/dashboard/stage_tracker_build.go +++ b/src/pkg/dashboard/stage_tracker_build.go @@ -437,8 +437,12 @@ func (s *Server) buildStageTracker(status *StatusPayload, store *stagelabels.Sto it.Caption = joinClause(it.Caption, clause) } } + if n, ok := stageTrackerFixerExhaustedAttempts(it.Reason); ok && it.Stage == trackerNumberNeedsDecision { + it.Caption, it.Actor = stageTrackerFixerExhaustedCaption(n, stageTrackerFixerExhaustedSummary(agg)), "maintainer" + } it.DecisionNeeded = stageTrackerDecisionNeededWith(it, agg, sentinelLabel, now, bctx) it.StageActor = stageTrackerStageActor(it, workers, now) + stageTrackerApplyFixerStatus(&it, review.FixDispatchStatusFor(dispatchState, qualifyRepo(repoName), pr.Number, pr.HeadSHA), workers) if stageTrackerClaimantStage(it.Kind, it.Stage) { var worker *stageTrackerWorker if w, ok := stageTrackerWorkerOn(workers, repoName, pr.Number); ok { diff --git a/src/pkg/dashboard/stage_tracker_decision_pr.go b/src/pkg/dashboard/stage_tracker_decision_pr.go index 7d09211323..9e45b3eb9f 100644 --- a/src/pkg/dashboard/stage_tracker_decision_pr.go +++ b/src/pkg/dashboard/stage_tracker_decision_pr.go @@ -38,6 +38,9 @@ func decisionForPR(it StageTrackerItem, verdict *review.Aggregate, sentinelLabel // supersedes) after the fixer's two rebase-resolve attempts failed. rebaseFailedTwice := lower == stagelabels.LaneReasonRebaseFailedTwice || (adoptionStalled && adoptionLane == stagelabels.LaneReasonRebaseFailedTwice) reviewerDeferred := lower == stageengine.StageReasonVerdictRequiresHuman || lower == stageengine.StageReasonDecisionRequested + // hivecommons/hive#15744: the hive's own PR after its fixer spent the + // re-engagement cap; nobody is fixing it any more. + fixerAttempts, fixerExhausted := stagelabels.ParseFixerExhausted(base) hiveSupersederStop := stagelabels.HiveSupersederHardStopReason(base) // WHAT it is waiting on: the question, most specific source first. A @@ -82,6 +85,13 @@ func decisionForPR(it StageTrackerItem, verdict *review.Aggregate, sentinelLabel } d.Blocker = "The hive set `hold` on its own: " + decisionHoldReason(selfHold) + ". It does nothing more until the hold is released." d.BlockerReason = decisionBlockerHoldHive + case fixerExhausted: + d.AskedBy = decisionAskedByReviewer + d.Question = "The hive's fixer gave up on this PR after " + strconv.Itoa(fixerAttempts) + " attempts" + if finding := decisionFixerExhaustedFinding(it, verdict); finding != "" { + d.Question += ": " + finding + } + d.Question += ". Re-author it from scratch, hand it to a person, or close it as not planned?" case verdict != nil && decisionFirstNonEmpty(verdict.HumanSummaries) != "": d.AskedBy = decisionAskedByReviewer d.Question = "A reviewer handed this to a person: " + decisionFirstNonEmpty(verdict.HumanSummaries) @@ -141,6 +151,9 @@ func decisionForPR(it StageTrackerItem, verdict *review.Aggregate, sentinelLabel d.Blocker = "The hive never merges this by itself. After approval, a maintainer merges." case forkGuarded: d.Blocker = "Neither the author nor the hive can make CI run for this fork PR." + case fixerExhausted: + d.Blocker = "The fix dispatcher refuses another fix for this head (" + strconv.Itoa(fixerAttempts) + " attempts spent). No agent is on it until a person decides." + approveOK, approveWhy = false, "The head still carries the reviewer's unresolved findings; approving does not fix them." case reviewerDeferred: d.Blocker = "The hive will not merge until a person decides." if verdict != nil && strings.TrimSpace(verdict.DecisionFinding) != "" { @@ -176,6 +189,9 @@ func decisionForPR(it StageTrackerItem, verdict *review.Aggregate, sentinelLabel // for merge-now (rebaseMergeFailureOptions). d.Options = slices.Insert(d.Options, 0, decisionOption(decisionActionAdopt, rebasemerge.OptionAdoptFreshLabel, true, "")) } + if fixerExhausted { + d.Options = decisionLeadWithReauthor(d.Options) + } if selfHold != nil { // Release leads: it is the one action that answers the question. d.Options = append([]StageTrackerDecisionOption{decisionHoldOption(true)}, d.Options[:len(d.Options)-1]...) @@ -207,6 +223,49 @@ func decisionLeadWithCloseSuperseded(opts []StageTrackerDecisionOption, redundan return out } +// decisionLeadWithReauthor puts "Re-author" at the front of opts (the lead +// action for a fixer-exhausted PR, hivecommons/hive#15744) and relabels the +// won't-fix close as "Close as not planned" so it stays in the Decide… +// submenu rather than leading a destructive answer. +func decisionLeadWithReauthor(opts []StageTrackerDecisionOption) []StageTrackerDecisionOption { + lead := decisionOption(decisionActionReauthor, "Re-author", true, "") + lead.Consequence = "The hive opens a fresh PR for the same issue from the current base and closes this one as superseded." + lead.Lead = true + out := make([]StageTrackerDecisionOption, 0, len(opts)+1) + out = append(out, lead) + for _, o := range opts { + switch o.Action { + case decisionActionReauthor: + continue + case decisionActionCloseWontfix: + o.Label = "Close as not planned" + o.Consequence = "Closes this PR as not planned; the issue stays open for someone else." + case decisionActionDirection: + o.Consequence = "Hands the PR to a person: say who takes it or what to change; the hive steps back." + } + out = append(out, o) + } + return out +} + +// decisionFixerExhaustedFinding is the reviewer's last finding in one line +// for the fixer-exhausted question: the verdict's human summary or +// question, else the caption's dash clause. +func decisionFixerExhaustedFinding(it StageTrackerItem, verdict *review.Aggregate) string { + if verdict != nil { + if s := decisionFirstNonEmpty(verdict.HumanSummaries); s != "" { + return stageTrackerFirstLine(s) + } + if q := strings.TrimSpace(verdict.DecisionQuestion); q != "" { + return stageTrackerFirstLine(q) + } + } + if _, after, ok := strings.Cut(it.Caption, " — "); ok { + return strings.TrimSpace(after) + } + return "" +} + // decisionHoldReason is the hold's plain-words reason for the popover. func decisionHoldReason(h *StageTrackerHold) string { if h != nil { diff --git a/src/pkg/dashboard/stage_tracker_fixer.go b/src/pkg/dashboard/stage_tracker_fixer.go new file mode 100644 index 0000000000..7a1ec8da65 --- /dev/null +++ b/src/pkg/dashboard/stage_tracker_fixer.go @@ -0,0 +1,105 @@ +package dashboard + +import ( + "slices" + "strconv" + "strings" + + "github.com/hivecommons/hive/pkg/review" + "github.com/hivecommons/hive/pkg/stagelabels" +) + +// Fixer standing on a hive PR row (hivecommons/hive#15744). Hive's own +// #15308 read "Fixing · @hivecommons-hive[bot]" for 13h while the fix +// dispatcher refused it every cycle (fix-attempts-exhausted, 6 attempts): +// the row borrowed the author as the actor. A 070 row now says why nobody +// is fixing when the dispatcher bars a fix, and the 085 row the stage +// engine parks an exhausted PR on captions the reviewer's last finding. + +const stageTrackerFixerExhaustedSummaryMax = 140 + +// stageTrackerApplyFixerStatus rewrites the caption and stage actor of a +// hive-authored PR row when the fix dispatcher has no job on its head and +// refuses to start one. Rows with an agent on them, non-hive rows and rows +// off the fixing stage are left alone. +func stageTrackerApplyFixerStatus(it *StageTrackerItem, fixer review.FixDispatchStatus, workers map[string]stageTrackerWorker) { + if it == nil || it.Kind != "pr" || it.Stage != trackerNumberFixing || !fixer.Excluded() { + return + } + if it.StageActor == nil || it.StageActor.Kind != StageActorHive { + return + } + if _, busy := stageTrackerWorkerOn(workers, it.Repo, it.Number); busy { + return + } + why := review.FixDispatchExclusionText(fixer) + if why == "" { + return + } + if slices.Contains(it.Modifiers, modNeedsRebase) { + why += "; conflicting with the base branch" + } + it.Caption = "Fixing paused — " + why + "; no fix job is dispatched" + it.StageActor.Kind = StageActorNone + it.StageActor.Login = "" + it.StageActor.Why = "nobody — " + why + "; the hive will not re-kick it" + if fixer.Exhausted() { + it.Actor = "maintainer" + it.StageActor.Why += " — a person decides" + } +} + +// stageTrackerFixerExhaustedCaption is the 085 caption for a PR parked on +// stagelabels.FixerExhaustedReason: "fixer gave up after N attempts — ". summary "" drops the dash clause. +func stageTrackerFixerExhaustedCaption(attempts int, summary string) string { + c := "fixer gave up after " + strconv.Itoa(attempts) + " attempts" + if summary = stageTrackerFirstLine(summary); summary != "" { + c += " — " + summary + } + return c +} + +// stageTrackerFixerExhaustedSummary is the reviewer's one-line finding for +// the exhausted caption, read from the head's review aggregate. +func stageTrackerFixerExhaustedSummary(agg *review.Aggregate) string { + if agg == nil { + return "" + } + for _, s := range agg.HumanSummaries { + if s = strings.TrimSpace(s); s != "" { + return stageTrackerFirstLine(s) + } + } + if q := strings.TrimSpace(agg.DecisionQuestion); q != "" { + return stageTrackerFirstLine(q) + } + for _, f := range agg.Findings { + if t := strings.TrimSpace(f.Finding.Title); t != "" { + return stageTrackerFirstLine(t) + } + } + for _, r := range agg.Reasons { + if r = strings.TrimSpace(r); r != "" { + return stageTrackerFirstLine(r) + } + } + return "" +} + +func stageTrackerFirstLine(s string) string { + s = strings.TrimSpace(s) + if i := strings.IndexByte(s, '\n'); i >= 0 { + s = strings.TrimSpace(s[:i]) + } + if len(s) > stageTrackerFixerExhaustedSummaryMax { + s = strings.TrimSpace(s[:stageTrackerFixerExhaustedSummaryMax-1]) + "…" + } + return s +} + +// stageTrackerFixerExhaustedAttempts reads the attempt count from a row's +// 085 reason; ok is false for any other reason. +func stageTrackerFixerExhaustedAttempts(reason string) (int, bool) { + return stagelabels.ParseFixerExhausted(reason) +} diff --git a/src/pkg/dashboard/stage_tracker_fixer_test.go b/src/pkg/dashboard/stage_tracker_fixer_test.go new file mode 100644 index 0000000000..efa5457c5d --- /dev/null +++ b/src/pkg/dashboard/stage_tracker_fixer_test.go @@ -0,0 +1,95 @@ +package dashboard + +import ( + "strings" + "testing" + + "github.com/hivecommons/hive/pkg/review" + "github.com/hivecommons/hive/pkg/stagelabels" +) + +// hivecommons/hive#15744: a hive PR the fix dispatcher refuses must not +// read "Fixing · @bot" at 070, and at 085 its lead answer is Re-author. + +func TestStageTrackerApplyFixerStatusExhausted(t *testing.T) { + exhausted := review.FixDispatchStatus{Reason: review.FixDispatchExcludedAttemptsExhausted, Attempts: 6} + base := func() StageTrackerItem { + return StageTrackerItem{ + Repo: "hivecommons/hive", Number: 15308, Kind: "pr", Stage: trackerNumberFixing, + Caption: "Fixing", + Modifiers: []string{modNeedsRebase}, + StageActor: &StageTrackerActor{Kind: StageActorHive, Login: "hivecommons-hive[bot]"}, + } + } + + it := base() + stageTrackerApplyFixerStatus(&it, exhausted, nil) + if !strings.HasPrefix(it.Caption, "Fixing paused — fixer gave up after 6 attempts") || !strings.Contains(it.Caption, "conflicting with the base branch") { + t.Fatalf("caption = %q", it.Caption) + } + if it.StageActor.Kind != StageActorNone || it.StageActor.Login != "" || it.Actor != "maintainer" { + t.Fatalf("actor = %+v (row actor %q)", it.StageActor, it.Actor) + } + + // A worker on the PR means a fix really is running: leave the row alone. + busy := base() + workers := map[string]stageTrackerWorker{"hivecommons/hive#15308": {}} + stageTrackerApplyFixerStatus(&busy, exhausted, workers) + if busy.Caption != "Fixing" || busy.StageActor.Kind != StageActorHive { + t.Fatalf("busy row changed: %q %+v", busy.Caption, busy.StageActor) + } + + // Active / not-excluded dispatch: unchanged. + active := base() + stageTrackerApplyFixerStatus(&active, review.FixDispatchStatus{Active: true, Agent: "claude"}, nil) + if active.Caption != "Fixing" || active.StageActor.Kind != StageActorHive { + t.Fatalf("active row changed: %q %+v", active.Caption, active.StageActor) + } + + // Off the fixing stage: unchanged. + parked := base() + parked.Stage = trackerNumberNeedsDecision + stageTrackerApplyFixerStatus(&parked, exhausted, nil) + if parked.Caption != "Fixing" { + t.Fatalf("085 row changed: %q", parked.Caption) + } +} + +func TestStageTrackerFixerExhaustedCaption(t *testing.T) { + if got := stageTrackerFixerExhaustedCaption(6, "tests fail on arm64\nsecond line"); got != "fixer gave up after 6 attempts — tests fail on arm64" { + t.Fatalf("caption = %q", got) + } + if got := stageTrackerFixerExhaustedCaption(3, ""); got != "fixer gave up after 3 attempts" { + t.Fatalf("caption = %q", got) + } +} + +func TestDecisionForPRFixerExhaustedLeadsWithReauthor(t *testing.T) { + it := StageTrackerItem{ + Repo: "hivecommons/hive", Number: 15308, Kind: "pr", Stage: trackerNumberNeedsDecision, + AuthorKind: authorKindHive, + Reason: stagelabels.FixerExhaustedReason(6), + } + d := decisionForPR(it, &review.Aggregate{HumanSummaries: []string{"tests fail on arm64"}}, "", false) + if d == nil || !strings.Contains(d.Question, "6 attempts") || !strings.Contains(d.Question, "tests fail on arm64") { + t.Fatalf("decision = %+v", d) + } + if len(d.Options) == 0 || d.Options[0].Action != decisionActionReauthor || !d.Options[0].Lead || !d.Options[0].Enabled { + t.Fatalf("lead option = %+v", d.Options) + } + var sawClose bool + for i, o := range d.Options { + if o.Action == decisionActionCloseWontfix { + sawClose = true + if i == 0 || o.Lead || o.Label != "Close as not planned" { + t.Fatalf("close option = %+v at %d", o, i) + } + } + if o.Action == decisionActionReauthor && i != 0 { + t.Fatalf("duplicate re-author option at %d: %+v", i, d.Options) + } + } + if !sawClose { + t.Fatalf("no close-as-not-planned option: %+v", d.Options) + } +} diff --git a/src/pkg/github/automerge/automerge_fixer_exhausted_test.go b/src/pkg/github/automerge/automerge_fixer_exhausted_test.go new file mode 100644 index 0000000000..ed195c64ed --- /dev/null +++ b/src/pkg/github/automerge/automerge_fixer_exhausted_test.go @@ -0,0 +1,115 @@ +package automerge + +import ( + "strings" + "testing" + "time" + + gh "github.com/google/go-github/v72/github" + "github.com/hivecommons/hive/pkg/review" + "github.com/hivecommons/hive/pkg/stagelabels" +) + +// Fixer exhausted on the hive's own PR (hivecommons/hive#15744): the fix +// dispatcher refused another fix every cycle (fix-attempts-exhausted, 6 +// attempts) while #15308 sat at 070 reading "Fixing" for 13h. The stage +// pass now parks it at 085 with the attempt count in the reason and posts +// the decision comment once per head. + +func fixerExhaustedHarness(t *testing.T) (*offlineAdoptHarness, func(), map[int]review.FixDispatchStatus) { + t.Helper() + o, done := newOfflineAdoptHarness(t, true) + status := map[int]review.FixDispatchStatus{} + changes := ContributorLaneVerdict{Configured: true, Verdict: "changes_requested", HumanSummaries: []string{"handler drops the error from Close()\nmore detail"}} + o.e.contributorLaneVerdict = func(string, int, string) ContributorLaneVerdict { return changes } + o.e.fixDispatch = func(_ string, number int, _ string) (review.FixDispatchStatus, bool) { + st, ok := status[number] + return st, ok + } + return o, done, status +} + +func hiveFixingPR(number int, head string, labels ...string) *gh.PullRequest { + pr := adoptPR(number, head, labels...) + pr.User = &gh.User{Login: gh.Ptr(testHiveAppBotLogin), Type: gh.Ptr("Bot")} + pr.Head.Ref = gh.Ptr("hive/fix-" + head) + pr.Head.Repo = &gh.Repository{FullName: gh.Ptr("acme/widget")} + return pr +} + +func decisionComments(api *stageAPI) []string { + var out []string + for _, c := range api.comments { + if strings.HasPrefix(c, stagelabels.DecisionCommentMarker) { + out = append(out, c) + } + } + return out +} + +func TestFixerExhaustedParksHivePRAtNeedsDecisionOnce(t *testing.T) { + o, done, status := fixerExhaustedHarness(t) + defer done() + key := stageKey("acme", "widget", 90) + + // Attempts left: the PR stays on its fixing stage. + status[90] = review.FixDispatchStatus{Attempts: 3} + o.observeSelf(hiveFixingPR(90, "ccc", "stage/070-fixing"), contributorLaneReasonChangesRequested) + if rec, _ := o.e.stageStoreOrShared().Get(key); rec.Stage != stagelabels.Fixing { + t.Fatalf("with attempts left = %s (%q)", rec.Stage, rec.Reason) + } + + status[90] = review.FixDispatchStatus{Attempts: 6, Reason: review.FixDispatchExcludedAttemptsExhausted} + o.api.reset() + o.clock.advance(time.Minute) + o.observeSelf(hiveFixingPR(90, "ccc", "stage/070-fixing"), contributorLaneReasonChangesRequested) + rec, _ := o.e.stageStoreOrShared().Get(key) + if rec.Stage != stagelabels.NeedsDecision || rec.Reason != stagelabels.FixerExhaustedReason(6) { + t.Fatalf("after exhaustion = %s (%q)", rec.Stage, rec.Reason) + } + if n, ok := stagelabels.ParseFixerExhausted(rec.Reason); !ok || n != 6 { + t.Fatalf("reason %q does not parse to 6 attempts", rec.Reason) + } + comments := decisionComments(o.api) + if len(comments) != 1 { + t.Fatalf("decision comments = %d, want 1: %q", len(comments), comments) + } + for _, want := range []string{"gave up after 6 attempts", "last reviewer finding: handler drops the error from Close()", "Re-author from scratch", "Hand it to a person", "Close as not planned"} { + if !strings.Contains(comments[0], want) { + t.Fatalf("decision comment lacks %q:\n%s", want, comments[0]) + } + } + if strings.Contains(comments[0], "more detail") { + t.Fatalf("decision comment carries more than the finding's first line:\n%s", comments[0]) + } + + // Same head, next pass: no second comment. + o.clock.advance(5 * time.Minute) + o.observeSelf(hiveFixingPR(90, "ccc", "stage/070-fixing"), contributorLaneReasonChangesRequested) + if got := decisionComments(o.api); len(got) != 1 { + t.Fatalf("decision comments after second pass = %d, want 1", len(got)) + } + if rec, _ := o.e.stageStoreOrShared().Get(key); rec.Stage != stagelabels.NeedsDecision { + t.Fatalf("second pass moved the PR to %s", rec.Stage) + } + + // A fix in flight on a new head is a fixing stage again. + status[90] = review.FixDispatchStatus{Active: true, Agent: "fixer", Attempts: 1} + o.observeSelf(hiveFixingPR(90, "ddd", "stage/085-needs-decision"), contributorLaneReasonChangesRequested) + if rec, _ := o.e.stageStoreOrShared().Get(key); rec.Stage != stagelabels.Fixing { + t.Fatalf("new head with an active fix = %s (%q)", rec.Stage, rec.Reason) + } +} + +func TestFixerExhaustedIgnoresContributorPR(t *testing.T) { + o, done, status := fixerExhaustedHarness(t) + defer done() + status[7] = review.FixDispatchStatus{Attempts: 6, Reason: review.FixDispatchExcludedAttemptsExhausted} + o.observe(adoptPR(7, "aaa", "stage/072-returned-to-contributor", "role/contributor", "author/clanker"), contributorLaneReasonChangesRequested) + if rec, _ := o.e.stageStoreOrShared().Get(stageKey("acme", "widget", 7)); rec.Stage == stagelabels.NeedsDecision { + t.Fatalf("contributor PR parked at 085 on the hive's fixer status: %+v", rec) + } + if got := decisionComments(o.api); len(got) != 0 { + t.Fatalf("decision comments on a contributor PR = %d", len(got)) + } +} diff --git a/src/pkg/github/automerge/automerge_stage_human.go b/src/pkg/github/automerge/automerge_stage_human.go index 2213c5ab02..40e779c1af 100644 --- a/src/pkg/github/automerge/automerge_stage_human.go +++ b/src/pkg/github/automerge/automerge_stage_human.go @@ -666,13 +666,17 @@ func (c *Engine) postHumanEntry(ctx context.Context, store *stagelabels.Store, k default: return 0, nil } - if obs.Reason == autoMergeReasonHumanMergeRequired { + _, fixerExhausted := stagelabels.ParseFixerExhausted(obs.Reason) + if obs.Reason == autoMergeReasonHumanMergeRequired || fixerExhausted { // The "waiting for a person" note on a breaking-titled or // owner-flagged PR is an unsolicited write (hivecommons/hive#15442): // it goes through the write-back gate like every other hive // comment, and a refusal leaves the record unmarked so the next - // pass asks again. - if d := c.writebackAllowed(obs.Display, writebackItemForObservation(obs), writeback.KindNeedsDecisionComment, body); !d.Allowed { + // pass asks again. The fixer-exhausted note on the hive's own PR + // takes the same gate (hivecommons/hive#15744). + item := writebackItemForObservation(obs) + item.HiveAuthored = fixerExhausted + if d := c.writebackAllowed(obs.Display, item, writeback.KindNeedsDecisionComment, body); !d.Allowed { c.info("stage labels: entry comment withheld by the write-back gate", "repo", obs.Display, "pr", obs.Number, "stage", route.Stage.String(), "reason", d.Reason) return 0, nil } @@ -709,3 +713,55 @@ func (c *Engine) workflowApprovalSnippet(reason string) string { b.WriteString("\n```\n\n") return b.String() } + +// fixDispatchStatusFor reads the review fixer's standing on obs's head for +// a hive-authored PR; ok is false when the engine has no dispatch reader or +// the PR is not the hive's (the hive never fixes a PR it did not open). +func (c *Engine) fixDispatchStatusFor(obs stageObservation, pass *stageengine.PRPassFacts) (review.FixDispatchStatus, bool) { + if c == nil || c.fixDispatch == nil || obs.Number <= 0 { + return review.FixDispatchStatus{}, false + } + if obs.Lane != stageLaneSelf && !(pass != nil && pass.AuthorIsApp) && obs.AuthorKind != authorkind.Hive { + return review.FixDispatchStatus{}, false + } + return c.fixDispatch(obs.Owner+"/"+obs.Repo, obs.Number, obs.HeadSHA) +} + +// fixerExhaustedRoute is the 085 route (and so the entry comment, once per +// head) for a hive PR whose fixer gave up (hivecommons/hive#15744). +func (c *Engine) fixerExhaustedRoute(obs stageObservation, fixer review.FixDispatchStatus) stagelabels.Route { + v := obs.Detail.Verdict + if !obs.Detail.VerdictKnown { + v = c.contributorLaneVerdictFor(obs.Owner+"/"+obs.Repo, obs.Number, obs.HeadSHA) + } + return stagelabels.FixerExhaustedRoute(obs.HeadSHA, fixer.Attempts, fixerExhaustedSummary(v, obs.LaneReason)) +} + +// fixerExhaustedSummary is the reviewer's one-line reason the fixer kept +// failing to clear: the first human summary, else the decision question or +// directive, else the lane's reason in words. +func fixerExhaustedSummary(v ContributorLaneVerdict, laneReason string) string { + for _, s := range v.HumanSummaries { + if s = strings.TrimSpace(s); s != "" { + return firstLine(s) + } + } + if q := strings.TrimSpace(v.DecisionQuestion); q != "" { + return firstLine(q) + } + if d := strings.TrimSpace(v.DecisionDirective); d != "" { + return firstLine(d) + } + r := strings.TrimSpace(laneReason) + for _, prefix := range []string{"contributor-lane:", "reviewer-verdict:"} { + r = strings.TrimPrefix(r, prefix) + } + return strings.NewReplacer("-", " ", "_", " ").Replace(r) +} + +func firstLine(s string) string { + if i := strings.IndexByte(s, '\n'); i >= 0 { + s = s[:i] + } + return strings.TrimSpace(s) +} diff --git a/src/pkg/github/automerge/automerge_stage_labels.go b/src/pkg/github/automerge/automerge_stage_labels.go index d14cdee6e3..bbf411d37e 100644 --- a/src/pkg/github/automerge/automerge_stage_labels.go +++ b/src/pkg/github/automerge/automerge_stage_labels.go @@ -918,6 +918,12 @@ func (c *Engine) ApplyStageLabels(ctx context.Context) (StageLabelResult, error) verdict = replacement obs.Reason = replacement.Reason } + fixer, fixerKnown := c.fixDispatchStatusFor(obs, pass) + pass.FixerExcluded = fixerKnown && fixer.Excluded() + if replacement, ok := stageengine.PreserveStageOnStaleDecisionAnswer(prevRec, hadPrev, obs, verdict, pass.FixerExcluded); ok { + verdict = replacement + obs.Reason = replacement.Reason + } if hasLabelFold(obs.Labels, "hold") && !verdict.Terminal { // Who held it and why rides the row (hivecommons/hive#15020); // a reporter-trust hold the hive put on its own PR is lifted @@ -1111,6 +1117,19 @@ func (c *Engine) ApplyStageLabels(ctx context.Context) (StageLabelResult, error) obs.Reason = stagelabels.LaneReasonRebaseFailedTwice pass.RebaseFailedTwice = true } + if !verdict.Terminal && verdict.Stage == fixingStage(obs) && fixerKnown && fixer.Exhausted() { + // The hive's own PR whose fix dispatcher refuses another + // fix (re-engagement cap spent, hivecommons/hive#15744): + // a person decides (re-author / hand over / close as not + // planned) instead of the row reading "Fixing" with + // nobody on it for hours (#15308). + verdict.Stage = stagelabels.NeedsDecision + verdict.Rank = stageRankOwned + obs.Reason = stagelabels.FixerExhaustedReason(fixer.Attempts) + pass.FixerExhausted = fixer.Attempts + route := c.fixerExhaustedRoute(obs, fixer) + obs.Route = &route + } tr = store.Observe(key, verdict.Stage, obs.Reason) // hivecommons/hive#12502: remember the author so the dashboard can // list "PRs returned to this contributor" from the store alone. diff --git a/src/pkg/github/automerge/automerge_sweep.go b/src/pkg/github/automerge/automerge_sweep.go index 9849c56cb7..25c38b4653 100644 --- a/src/pkg/github/automerge/automerge_sweep.go +++ b/src/pkg/github/automerge/automerge_sweep.go @@ -241,6 +241,12 @@ type Options struct { // parks a copy whose episode is exhausted at 085 instead of leaving it // on its fixing stage (hivecommons/hive#15307). nil reads as no episode. RebaseEpisode func(repo string, number int) (RebaseEpisode, bool) + // FixDispatchStatus reads the review fixer's standing on a PR head from + // the dispatch state (review.FixDispatchStatusFor through cmd/hive): a + // hive PR whose fixer is exhausted parks at 085 instead of reading + // "Fixing" with nobody on it (hivecommons/hive#15744). nil reads as + // unknown and leaves the fixing stage alone. + FixDispatchStatus func(repo string, number int, head string) (review.FixDispatchStatus, bool) // HotspotRecorder receives the merge train's PR file lists and the // conflicts it asks the resolver to repair, so the merge-hotspot // advisor (pkg/hotspot) derives its evidence without polling GitHub @@ -396,6 +402,7 @@ type Engine struct { mergeTrainPolicy func() MergeTrainPolicy rebaseResolver RebaseResolver rebaseEpisode func(repo string, number int) (RebaseEpisode, bool) + fixDispatch func(repo string, number int, head string) (review.FixDispatchStatus, bool) hotspotRecorder HotspotRecorder mergeTrainState } @@ -456,6 +463,7 @@ func New(transport Transport, opts Options) *Engine { nativeMergeQueuePolicy: opts.NativeMergeQueue, rebaseResolver: opts.RebaseResolver, rebaseEpisode: opts.RebaseEpisode, + fixDispatch: opts.FixDispatchStatus, hotspotRecorder: opts.HotspotRecorder, evaluatedHeads: make(map[string]string), requiredChecksFallbackWarned: make(map[string]bool), diff --git a/src/pkg/review/fix_dispatch_status.go b/src/pkg/review/fix_dispatch_status.go new file mode 100644 index 0000000000..c5890b682b --- /dev/null +++ b/src/pkg/review/fix_dispatch_status.go @@ -0,0 +1,113 @@ +package review + +import ( + "strconv" + "strings" + + "github.com/hivecommons/hive/pkg/escalation" +) + +// FixDispatchStatus is the fixer's standing on one PR head, read from the +// persisted dispatch state: is a fix job in flight, and if not, why the +// dispatcher refuses to start one (hivecommons/hive#15744). The stage engine +// and the tracker consult it so a 070 row never claims "the hive is fixing +// it" when the fixer has in fact given up or is otherwise excluded. +type FixDispatchStatus struct { + // Active is true when a pending fix for this head is on the books. + Active bool + // Agent is the fixer holding the pending fix (Active only). + Agent string + // Reason is the FixDispatchExcluded* reason when no fix is active and the + // dispatcher would refuse to start one; "" when a fix could be kicked. + Reason string + // Attempts is the highest attempt number recorded for the PR. + Attempts int +} + +// Exhausted reports whether the fixer has hit the re-engagement cap. +func (s FixDispatchStatus) Exhausted() bool { + return !s.Active && s.Reason == FixDispatchExcludedAttemptsExhausted +} + +// Excluded reports whether no fix is active and a reason bars dispatch. +func (s FixDispatchStatus) Excluded() bool { + return !s.Active && s.Reason != "" +} + +// FixDispatchStatusFor derives the fixer's standing on repo#number at head +// from state. Repo names are matched loosely (bare vs owner/name) because the +// dispatch state records the org-qualified form while callers may hold +// either. A head of "" matches any head of the PR. +func FixDispatchStatusFor(state DispatchState, repo string, number int, head string) FixDispatchStatus { + var out FixDispatchStatus + if number <= 0 { + return out + } + headMatches := func(sha string) bool { + return strings.TrimSpace(head) == "" || SameHeadSHA(sha, head) + } + for _, f := range state.Fixes { + if f.Number != number || !sameRepoLoose(f.Repo, repo) { + continue + } + if f.Attempts > out.Attempts { + out.Attempts = f.Attempts + } + if headMatches(f.HeadSHA) { + out.Active, out.Agent = true, f.Agent + } + } + for _, f := range state.FixAttempts { + if f.Number == number && sameRepoLoose(f.Repo, repo) && f.Attempts > out.Attempts { + out.Attempts = f.Attempts + } + } + if out.Active { + return out + } + if out.Attempts >= escalation.MaxReEngagements { + out.Reason = FixDispatchExcludedAttemptsExhausted + return out + } + for _, h := range state.Human { + if h.Number != number || !sameRepoLoose(h.Repo, repo) || !headMatches(h.HeadSHA) { + continue + } + switch { + case h.Reason == FixAttemptsExhaustedHumanDecisionReason: + out.Reason = FixDispatchExcludedAttemptsExhausted + if out.Attempts < escalation.MaxReEngagements { + out.Attempts = escalation.MaxReEngagements + } + return out + case strings.Contains(h.Reason, "no enabled kick-capable fixer"): + out.Reason = FixDispatchExcludedNoEnabledFixer + return out + } + } + for _, w := range state.Withheld { + if w.Number == number && sameRepoLoose(w.Repo, repo) && headMatches(w.HeadSHA) { + out.Reason = FixDispatchExcludedNotHiveAuthored + return out + } + } + return out +} + +// FixDispatchExclusionText renders a dispatch exclusion reason as the short +// phrase the tracker caption and the 085 decision comment use. +func FixDispatchExclusionText(s FixDispatchStatus) string { + switch s.Reason { + case FixDispatchExcludedAttemptsExhausted: + return "fixer gave up after " + strconv.Itoa(s.Attempts) + " attempts" + case FixDispatchExcludedNoEnabledFixer: + return "no enabled fixer agent" + case FixDispatchExcludedNotHiveAuthored, FixDispatchExcludedNotHiveAuthoredDCO: + return "not hive-authored — the hive does not push to it" + case FixDispatchExcludedPending: + return "fix already dispatched" + case "": + return "" + } + return s.Reason +} diff --git a/src/pkg/review/fix_dispatch_status_test.go b/src/pkg/review/fix_dispatch_status_test.go new file mode 100644 index 0000000000..86cd411924 --- /dev/null +++ b/src/pkg/review/fix_dispatch_status_test.go @@ -0,0 +1,49 @@ +package review + +import ( + "testing" + "time" + + "github.com/hivecommons/hive/pkg/escalation" +) + +// hivecommons/hive#15744: the fixer's standing on a head, read back from +// the persisted dispatch state the way #15308 recorded it (six attempts, +// a fix-attempts-exhausted human hold, no pending fix). +func TestFixDispatchStatusFor(t *testing.T) { + now := time.Date(2026, 10, 21, 8, 0, 0, 0, time.UTC) + state := DispatchState{ + FixAttempts: []FixAttempt{ + {Repo: "hivecommons/hive", Number: 15308, HeadSHA: "5b62304aaaaaaaa", Agent: "fixer", Attempts: escalation.MaxReEngagements}, + {Repo: "hivecommons/hive", Number: 15310, HeadSHA: "abc", Agent: "fixer", Attempts: 2}, + }, + Human: []HumanReviewHold{ + {Repo: "hivecommons/hive", Number: 15308, HeadSHA: "5b62304aaaaaaaa", Reason: FixAttemptsExhaustedHumanDecisionReason, UpdatedAt: now}, + {Repo: "hivecommons/hive", Number: 15311, HeadSHA: "def", Reason: "no enabled kick-capable fixer agent is available", UpdatedAt: now}, + }, + Fixes: []PendingFix{{Repo: "hivecommons/hive", Number: 15312, HeadSHA: "fff", Agent: "fixer", Attempts: 1, Dispatched: now}}, + } + + got := FixDispatchStatusFor(state, "hive", 15308, "5b62304") + if !got.Exhausted() || got.Attempts != escalation.MaxReEngagements || got.Active { + t.Fatalf("exhausted PR (bare repo, short head) = %+v", got) + } + if text := FixDispatchExclusionText(got); text != "fixer gave up after 6 attempts" { + t.Fatalf("exclusion text = %q", text) + } + if got := FixDispatchStatusFor(state, "hivecommons/hive", 15310, "abc"); got.Excluded() || got.Attempts != 2 { + t.Fatalf("PR with attempts left = %+v, want not excluded", got) + } + if got := FixDispatchStatusFor(state, "hivecommons/hive", 15311, "def"); got.Reason != FixDispatchExcludedNoEnabledFixer || !got.Excluded() { + t.Fatalf("no-fixer hold = %+v", got) + } + if got := FixDispatchStatusFor(state, "hivecommons/hive", 15311, "newhead"); got.Excluded() { + t.Fatalf("hold on an older head must not exclude the new one: %+v", got) + } + if got := FixDispatchStatusFor(state, "hivecommons/hive", 15312, "fff"); !got.Active || got.Agent != "fixer" || got.Excluded() { + t.Fatalf("pending fix = %+v, want active", got) + } + if got := FixDispatchStatusFor(DispatchState{}, "hivecommons/hive", 1, "x"); got.Excluded() || got.Active { + t.Fatalf("unknown PR = %+v, want zero", got) + } +} diff --git a/src/pkg/stageengine/derive_pr.go b/src/pkg/stageengine/derive_pr.go index 9383592d2e..860b3c44ed 100644 --- a/src/pkg/stageengine/derive_pr.go +++ b/src/pkg/stageengine/derive_pr.go @@ -248,6 +248,10 @@ func passDecision(verdict stageVerdict, role stagelabels.Role, obs stageObservat verdict = replacement obs.Reason = replacement.Reason } + if replacement, ok := PreserveStageOnStaleDecisionAnswer(prev, true, obs, verdict, pass.FixerExcluded); ok { + verdict = replacement + obs.Reason = replacement.Reason + } } var human *stagelabels.Route if verdict.Human && !verdict.Terminal && verdict.Reason != prstage.ReasonSupersessionPending { @@ -293,6 +297,11 @@ func passDecision(verdict stageVerdict, role stagelabels.Role, obs stageObservat verdict.Rank = stageRankOwned obs.Reason = stagelabels.LaneReasonRebaseFailedTwice } + if !verdict.Terminal && pass.FixerExhausted > 0 && verdict.Stage == FixingStage(obs) { + verdict.Stage = stagelabels.NeedsDecision + verdict.Rank = stageRankOwned + obs.Reason = stagelabels.FixerExhaustedReason(pass.FixerExhausted) + } d := Decision{ Stage: verdict.Stage, Role: role, diff --git a/src/pkg/stageengine/derive_pr_classify.go b/src/pkg/stageengine/derive_pr_classify.go index 64ea4cc90f..530fe97a5c 100644 --- a/src/pkg/stageengine/derive_pr_classify.go +++ b/src/pkg/stageengine/derive_pr_classify.go @@ -603,6 +603,26 @@ func PreserveNormalizedFixingStage(prev stagelabels.Record, hadPrev bool, obs st return stageVerdict{Stage: stagelabels.Fixing, Rank: stageRankOwned, Reason: stagelabels.LaneReasonChangesRequested}, true } +// PreserveStageOnStaleDecisionAnswer keeps a PR parked at 085 when the only +// thing pulling it back to its fixing stage is a trusted human's answered +// decision on a head that has not moved since, and no fixer will act on +// that answer (fixerExcluded: exhausted, no enabled fixer, …). Without it +// the flip to 070 reset the record's dwell and stripped stage/stalled +// ("removed:[stage/stalled] reason:contributor-lane:decision-answered" on +// hive's own #15308, hivecommons/hive#15744) while nobody was fixing. +func PreserveStageOnStaleDecisionAnswer(prev stagelabels.Record, hadPrev bool, obs stageObservation, verdict stageVerdict, fixerExcluded bool) (stageVerdict, bool) { + if !hadPrev || !fixerExcluded || prev.Stage != stagelabels.NeedsDecision { + return stageVerdict{}, false + } + if verdict.Terminal || verdict.Stage != FixingStage(obs) || verdict.Reason != StageReasonDecisionAnswered { + return stageVerdict{}, false + } + if prev.ActivityHead == "" || obs.HeadSHA == "" || prev.ActivityHead != obs.HeadSHA { + return stageVerdict{}, false + } + return stageVerdict{Stage: stagelabels.NeedsDecision, Rank: stageRankOwned, Reason: prev.Reason}, true +} + // AuthorSideDone reports whether the PR has nothing left for its author to // do: the reviewer verdict for this head is approve and the lane's reason // is not a CI state. diff --git a/src/pkg/stageengine/derive_pr_fixer_exhausted_test.go b/src/pkg/stageengine/derive_pr_fixer_exhausted_test.go new file mode 100644 index 0000000000..7f6f4ea383 --- /dev/null +++ b/src/pkg/stageengine/derive_pr_fixer_exhausted_test.go @@ -0,0 +1,68 @@ +package stageengine + +import ( + "testing" + + "github.com/hivecommons/hive/pkg/authorkind" + "github.com/hivecommons/hive/pkg/stagelabels" +) + +// hivecommons/hive#15744: the hive's own PR whose fix dispatcher gave up +// derives 085 with the attempt count in the reason; a stale answered +// decision on an unmoved head must not pull it back to 070. +func hiveFixingFacts(reason string) Facts { + return Facts{ + Ref: Ref{Owner: "o", Repo: "r", Number: 15308, Kind: stagelabels.PRItem}, + Open: true, + Labels: []string{"stage/070-fixing", "author/hive"}, + PR: &PRFacts{ + HeadSHA: goldenHead, + Lanes: []LaneFact{{Lane: stageLaneSelf, Reason: reason}}, + }, + } +} + +func TestDerivePRFixerExhaustedParksAtNeedsDecision(t *testing.T) { + f := hiveFixingFacts(contributorLaneReasonChangesRequested) + if d := DerivePR(f); d.Stage != stagelabels.Fixing { + t.Fatalf("baseline stage = %s (%q), want fixing", d.Stage, d.Reason) + } + f.PR.Pass = &PRPassFacts{AuthorIsApp: true, FixerExhausted: 6, FixerExcluded: true} + d := DerivePR(f) + if d.Stage != stagelabels.NeedsDecision || d.Reason != stagelabels.FixerExhaustedReason(6) { + t.Fatalf("exhausted: stage = %s reason = %q, want 085 fixer-exhausted:6", d.Stage, d.Reason) + } + // Off the fixing stage the fact is inert. + g := hiveFixingFacts("ci-pending") + g.PR.Pass = &PRPassFacts{AuthorIsApp: true, FixerExhausted: 6, FixerExcluded: true} + if d := DerivePR(g); d.Stage == stagelabels.NeedsDecision { + t.Fatalf("exhausted fact moved a non-fixing PR to 085: %+v", d) + } +} + +func TestPreserveStageOnStaleDecisionAnswer(t *testing.T) { + obs := stageObservation{Lane: stageLaneSelf, HeadSHA: goldenHead, AuthorKind: authorkind.Hive} + fixing := stageVerdict{Stage: FixingStage(obs), Rank: stageRankOwned, Reason: StageReasonDecisionAnswered} + prev := stagelabels.Record{Stage: stagelabels.NeedsDecision, Reason: stagelabels.FixerExhaustedReason(6), ActivityHead: goldenHead} + + got, ok := PreserveStageOnStaleDecisionAnswer(prev, true, obs, fixing, true) + if !ok || got.Stage != stagelabels.NeedsDecision || got.Reason != prev.Reason { + t.Fatalf("excluded fixer, same head: got %+v %v, want prior 085 kept", got, ok) + } + if _, ok := PreserveStageOnStaleDecisionAnswer(prev, true, obs, fixing, false); ok { + t.Fatal("a fixer that can act must take the answered decision to 070") + } + moved := prev + moved.ActivityHead = "0ld0ld0ld0ld0ld0ld0ld0ld0ld0ld0ld0ld0ld0" + if _, ok := PreserveStageOnStaleDecisionAnswer(moved, true, obs, fixing, true); ok { + t.Fatal("a new head is a fresh decision; the guard must not hold it") + } + other := fixing + other.Reason = autoMergeReasonHumanChangesRequested + if _, ok := PreserveStageOnStaleDecisionAnswer(prev, true, obs, other, true); ok { + t.Fatal("only the answered-decision reason is guarded") + } + if _, ok := PreserveStageOnStaleDecisionAnswer(stagelabels.Record{Stage: stagelabels.Fixing, ActivityHead: goldenHead}, true, obs, fixing, true); ok { + t.Fatal("a PR not previously at 085 has nothing to preserve") + } +} diff --git a/src/pkg/stageengine/facts.go b/src/pkg/stageengine/facts.go index cec26eed80..61fc11eb70 100644 --- a/src/pkg/stageengine/facts.go +++ b/src/pkg/stageengine/facts.go @@ -280,6 +280,17 @@ type PRPassFacts struct { // PR reads 085 with stagelabels.LaneReasonRebaseFailedTwice so a // person decides (hivecommons/hive#15307). RebaseFailedTwice bool + // FixerExhausted is the attempt count when the review fix dispatcher + // refuses another fix for this hive-authored PR because the + // re-engagement cap is spent: the PR reads 085 with + // stagelabels.FixerExhaustedReason(n) so a person decides instead of + // the row claiming "Fixing" with nobody on it (hivecommons/hive#15744). + FixerExhausted int + // FixerExcluded is true when no fix job is active and the dispatcher + // bars one for any reason (exhausted, no enabled fixer, …): a stale + // answered decision on the same head must not pull the PR back from + // 085 to 070 and strip stage/stalled (hivecommons/hive#15744). + FixerExcluded bool // Stalled is the pass's dwell verdict (stagelabels.StalledWith over the // store record after this pass's Observe); KindLabels are the // configured author/ names. Both are outputs of the pass the diff --git a/src/pkg/stagelabels/fixer_exhausted_test.go b/src/pkg/stagelabels/fixer_exhausted_test.go new file mode 100644 index 0000000000..9e2aa20381 --- /dev/null +++ b/src/pkg/stagelabels/fixer_exhausted_test.go @@ -0,0 +1,58 @@ +package stagelabels + +import ( + "strings" + "testing" +) + +// hivecommons/hive#15744: the 085 reason and route for a hive PR whose fix +// dispatcher refused another fix after the re-engagement cap. + +func TestFixerExhaustedReasonRoundTrip(t *testing.T) { + r := FixerExhaustedReason(6) + if r != "fixer-exhausted:6" { + t.Fatalf("reason = %q", r) + } + n, ok := ParseFixerExhausted(" " + r + " ") + if !ok || n != 6 { + t.Fatalf("parse = %d %v", n, ok) + } + for _, bad := range []string{"", "fixer-exhausted:", "fixer-exhausted:x", "fixer-exhausted:-1", "human-changes-requested", "fixer-exhausted"} { + if n, ok := ParseFixerExhausted(bad); ok || n != 0 { + t.Fatalf("ParseFixerExhausted(%q) = %d %v, want 0 false", bad, n, ok) + } + } +} + +func TestFixerExhaustedRouteAndComment(t *testing.T) { + head := "5b62304abcdef0123456789" + r := FixerExhaustedRoute(head, 6, " tests fail on arm64 ") + if r.Stage != NeedsDecision || !r.SelfAuthored || r.Head != head || r.ActionsHeading != "Your options:" { + t.Fatalf("route = %+v", r) + } + if len(r.Reasons) != 2 || !strings.Contains(r.Reasons[0], "gave up after 6 attempts on `"+ShortSHA(head)+"`") || r.Reasons[1] != "last reviewer finding: tests fail on arm64" { + t.Fatalf("reasons = %q", r.Reasons) + } + if len(r.Actions) != 3 || !strings.HasPrefix(r.Actions[0], "**Re-author from scratch**") || !strings.HasPrefix(r.Actions[1], "**Hand it to a person**") || !strings.HasPrefix(r.Actions[2], "**Close as not planned**") { + t.Fatalf("actions = %q", r.Actions) + } + + c := DecisionComment(r) + for _, want := range []string{DecisionCommentMarker, "Needs a decision, not a merge:\n- the hive's fixer gave up", "- last reviewer finding: tests fail on arm64", "**Your options:**", "`adoption/supersede`", "/hive decision"} { + if !strings.Contains(c, want) { + t.Fatalf("comment missing %q:\n%s", want, c) + } + } + if strings.Contains(c, "How to release it") || DecisionActions(r) != "" { + t.Fatalf("exhausted route borrowed the hold/requires_human wording:\n%s", c) + } + + // No summary: a single reason, rendered inline. + one := FixerExhaustedRoute(head, 3, "") + if len(one.Reasons) != 1 { + t.Fatalf("reasons = %q", one.Reasons) + } + if c := DecisionComment(one); !strings.Contains(c, "Needs a decision, not a merge: the hive's fixer gave up after 3 attempts") { + t.Fatalf("single-reason comment:\n%s", c) + } +} diff --git a/src/pkg/stagelabels/human.go b/src/pkg/stagelabels/human.go index cf1f95ac4e..027399872b 100644 --- a/src/pkg/stagelabels/human.go +++ b/src/pkg/stagelabels/human.go @@ -268,6 +268,9 @@ type Route struct { Actions []string // ManualMerge means this lane has no unattended way past its rule. ManualMerge bool + // ActionsHeading replaces the "How to release it" heading over Actions + // when the list is a choice between outcomes, not release steps. + ActionsHeading string // Reasons explains the stage: what the author must fix (072/070), what // a person must decide (085), what is still running (060). Empty for a // trust-blocked 090; an approved manual-merge 090 names the rule and @@ -332,8 +335,34 @@ const ( // (hivecommons/hive#15104 guard); the same suffix under // LaneReasonAdoptionStalledPrefix parks a still-open original. LaneReasonRebaseFailedTwice = "rebase-failed-twice" + // LaneReasonFixerExhaustedPrefix opens the 085 reason for a hive-authored + // PR whose fix dispatcher refused to kick another fix because the + // re-engagement cap is spent ("fixer-exhausted:6"); the suffix is the + // attempt count (hivecommons/hive#15744). Before this the PR sat at 070 + // reading "Fixing" with nobody on it (#15308, 13h). + LaneReasonFixerExhaustedPrefix = "fixer-exhausted:" ) +// FixerExhaustedReason renders the 085 reason for a PR whose fixer gave up +// after attempts tries. +func FixerExhaustedReason(attempts int) string { + return LaneReasonFixerExhaustedPrefix + strconv.Itoa(attempts) +} + +// ParseFixerExhausted reads the attempt count out of a fixer-exhausted +// reason; ok is false for any other reason. +func ParseFixerExhausted(reason string) (attempts int, ok bool) { + reason = strings.TrimSpace(reason) + if !strings.HasPrefix(reason, LaneReasonFixerExhaustedPrefix) { + return 0, false + } + n, err := strconv.Atoi(strings.TrimPrefix(reason, LaneReasonFixerExhaustedPrefix)) + if err != nil || n < 0 { + return 0, false + } + return n, true +} + // RouteHuman decides which stage a lane's "a human must act" resolves to. func RouteHuman(e Evidence) Route { fixing := FixingStageFor(e.SelfAuthored, e.AuthorKind) @@ -691,7 +720,11 @@ func DecisionComment(r Route) string { b.WriteString(actions) } if len(r.Actions) > 0 { - b.WriteString("\n\n**How to release it:**\n\n") + heading := r.ActionsHeading + if heading == "" { + heading = "How to release it:" + } + b.WriteString("\n\n**" + heading + "**\n\n") for _, action := range r.Actions { b.WriteString("- " + action + "\n") } @@ -700,6 +733,31 @@ func DecisionComment(r Route) string { return b.String() } +// FixerExhaustedRoute is the 085 route for a hive-authored PR whose fix +// dispatcher refused another fix because the re-engagement cap is spent +// (hivecommons/hive#15744). summary is the reviewer's one-line finding the +// fixer never cleared ("" when unknown). The comment lists the three +// answers the tracker offers: re-author from scratch (the lead action), +// hand it to a person, or close it as not planned. +func FixerExhaustedRoute(head string, attempts int, summary string) Route { + reasons := []string{"the hive's fixer gave up after " + strconv.Itoa(attempts) + " attempts on `" + ShortSHA(head) + "`; no fix job is running"} + if summary = strings.TrimSpace(summary); summary != "" { + reasons = append(reasons, "last reviewer finding: "+summary) + } + return Route{ + Stage: NeedsDecision, + Reasons: reasons, + Head: head, + SelfAuthored: true, + ActionsHeading: "Your options:", + Actions: []string{ + "**Re-author from scratch** — *Re-author* in the tracker (or add the `adoption/supersede` label): the hive opens a fresh PR for the same issue from the current base and closes this one as superseded.", + "**Hand it to a person** — reply `/hive decision ` or push to the branch yourself; the hive steps back while a human owns it.", + "**Close as not planned** — close the PR (or *Close as not planned* under Decide… in the tracker) when the change is no longer wanted.", + }, + } +} + // DecisionActions names the two things a maintainer can do with a // hive-authored PR the swarm parked on `requires_human`: approve the head // to merge it as-is, or request changes — by review or `/hive decision` —