diff --git a/changelog.d/fixed-held-workflow-runs-refresh-and-stage.md b/changelog.d/fixed-held-workflow-runs-refresh-and-stage.md new file mode 100644 index 0000000000..53cee6d15a --- /dev/null +++ b/changelog.d/fixed-held-workflow-runs-refresh-and-stage.md @@ -0,0 +1 @@ +- A PR whose workflow runs GitHub holds for approval, and that the hive will not approve (an untrusted or fork head, or auto-approval off), now parks at 085 with reason `workflow-approval-required:held-runs:` instead of showing an unrelated lane reason such as "author association" or "waiting for a verdict". The approval probe also runs on green heads that are missing a required check (a fork head whose `pull_request_target` check passed), so the missing-check base refresh no longer merges the base into such a PR again and again, and it now updates a PR once per set of missing checks instead of once per check. diff --git a/src/cmd/hive/merge_eligibility.go b/src/cmd/hive/merge_eligibility.go index afb176a35f..fe6e9d2e98 100644 --- a/src/cmd/hive/merge_eligibility.go +++ b/src/cmd/hive/merge_eligibility.go @@ -168,6 +168,7 @@ func stageLabelPolicyFunc(cfg *config.Config) func() automerge.StageLabelPolicy BaseBranchRed: stageBaseBranchRed, BaseRedRemembered: stageBaseRedRemembered, ForkGuard: stageForkGuard, + HeldWorkflowRuns: stageHeldWorkflowRuns, DCORequired: stage.DCO.RequiredFor, SquashOnly: cfg.AutoMerge.SquashOnlyFor, QuietNotifications: cfg.GitHub.Notifications.Quiet(), diff --git a/src/cmd/hive/missing_check_base_refresh.go b/src/cmd/hive/missing_check_base_refresh.go index 01e8638306..1f3cf208dd 100644 --- a/src/cmd/hive/missing_check_base_refresh.go +++ b/src/cmd/hive/missing_check_base_refresh.go @@ -26,8 +26,9 @@ type missingCheckRefresher interface { // already updated a branch for. The second one is the loop guard: a check // that a fresh head still does not run (a path filter, a GITHUB_TOKEN-made // PR, see #11877) is not chased with another update every time the base -// moves. After a restart both are empty, which costs at most one more -// update per PR and check. +// moves. One update marks every check missing at the time, so a PR missing +// many checks is updated once, not once per check. After a restart both are +// empty, which costs at most one more update per PR. type missingCheckRefreshState struct { mu sync.Mutex lastRun time.Time @@ -53,11 +54,12 @@ var missingCheckRefresh = &missingCheckRefreshState{} // the base tip. // // It leaves alone a PR that is a draft, held, exempt or parked on a human -// decision, red, conflicting, waiting on workflow approval, a fork with -// "allow edits by maintainers" off, a dependency bot's branch, returned to -// its contributor (072) or adopted by the hive (074). It shares the stale-red -// refresh's interval and caps, and updates a PR at most once per (head, base -// tip) and once per missing check. +// decision, red, conflicting, waiting on workflow approval (GitHub holds its +// runs, and an update would only make a new head whose runs are held again), +// a fork with "allow edits by maintainers" off, a dependency bot's branch, +// returned to its contributor (072) or adopted by the hive (074). It shares +// the stale-red refresh's interval and caps, and updates a PR at most once +// per (head, base tip) and once per set of missing checks. // // It returns the escalation keys of the PRs it updated. func refreshMissingCheckBehindPRs( @@ -182,7 +184,18 @@ func refreshMissingCheckBehindPRs( } continue } - st.refreshed[memoKey+"|"+check] = true + // One update answers every missing check that already runs today (a + // run on the base tip or another open PR): a fresh head that still + // lacks them will not grow them on the next base move either, so + // none of them earns another update (projectbluefin/server#686 has + // 12 missing checks). A missing check with no such evidence yet did + // not justify this update; it keeps its own once the workflow that + // reports it lands on the base (#18945 review). + for _, name := range pending { + if base.state.Reported[name] || seenOnPRs[bk][name] { + st.refreshed[memoKey+"|"+name] = true + } + } perRepo[repo]++ total++ if logger != nil { diff --git a/src/cmd/hive/missing_check_base_refresh_test.go b/src/cmd/hive/missing_check_base_refresh_test.go index 2b0a9d2ccd..66616223bd 100644 --- a/src/cmd/hive/missing_check_base_refresh_test.go +++ b/src/cmd/hive/missing_check_base_refresh_test.go @@ -248,3 +248,80 @@ func TestMissingCheckRefreshSkip(t *testing.T) { }) } } + +// projectbluefin/server#686: a fork PR whose pull_request_target check (Prow) +// passed while GitHub holds every pull_request run for approval. The probe +// now runs on such a green head, and a held head is never base-refreshed: +// the update would only make a new head whose runs are held again. +func TestRefreshMissingCheckBehindPRs_HeldForkHeadLeftAlone(t *testing.T) { + resetMissingCheckRefresh(t) + pr := missingCheckPR(686, "h1") + pr.CIActivity.Checks = []github.PRActivityCheck{{Name: "prow / prow", State: "completed", Conclusion: "success", Head: "h1"}} + pr.CIActivity.WorkflowApproval = &github.PRWorkflowApproval{Pending: 4, RunIDs: []int64{1, 2, 3, 4}} + f := &fakeMissingCheckRefresher{bases: baseRunning("base1", "dashboard-build"), out: github.StaleRedRefreshOutcome{Updated: true, Settled: true}} + refreshMissingCheckBehindPRs(context.Background(), escalationTestConfig(), f, actionableWith(pr), nil, discardLogger(), time.Now()) + if len(f.calls) != 0 || len(f.baseReads) != 0 { + t.Fatalf("calls %d, base reads %v; want a head with held runs left alone with no API call", len(f.calls), f.baseReads) + } +} + +// A PR missing many required checks is updated once, not once per check: +// the one update answers every check missing at the time, so a later base +// tip does not buy another update for the next check on the list. +func TestRefreshMissingCheckBehindPRs_OncePerSetOfMissingChecks(t *testing.T) { + resetMissingCheckRefresh(t) + cfg := escalationTestConfig() + missing := []string{"build", "docs", "lint", "unit"} + pr := func(head string) github.PullRequest { + p := missingCheckPR(686, head) + p.Protection.MissingRequiredChecks = append([]string(nil), missing...) + return p + } + f := &fakeMissingCheckRefresher{bases: baseRunning("base1", missing...), out: github.StaleRedRefreshOutcome{Updated: true, Settled: true}} + now := time.Date(2026, 11, 18, 2, 9, 0, 0, time.UTC) + for i, tip := range []string{"base1", "base2", "base3", "base4"} { + f.bases = baseRunning(tip, missing...) + refreshMissingCheckBehindPRs(context.Background(), cfg, f, actionableWith(pr("head"+tip)), nil, discardLogger(), now.Add(time.Duration(i)*staleRedRefreshInterval)) + } + if len(f.calls) != 1 { + t.Fatalf("refresh calls = %d over four base tips with %d missing checks, want 1", len(f.calls), len(missing)) + } + // A check that becomes required later still gets its one update. + f.bases = baseRunning("base5", append(missing, "e2e")...) + next := pr("headbase5") + next.Protection.MissingRequiredChecks = append(next.Protection.MissingRequiredChecks, "e2e") + refreshMissingCheckBehindPRs(context.Background(), cfg, f, actionableWith(next), nil, discardLogger(), now.Add(5*staleRedRefreshInterval)) + if len(f.calls) != 2 || f.checks[1] != "e2e" { + t.Fatalf("refresh calls = %d (checks %v), want a second update for the newly required e2e", len(f.calls), f.checks) + } +} + +// A missing check with no evidence that it runs anywhere did not justify the +// update, so it is not memoised with the ones that did: once its workflow +// lands on the base, the PR earns one update for it (#18945 review). +func TestRefreshMissingCheckBehindPRs_UnevidencedCheckKeepsItsUpdate(t *testing.T) { + resetMissingCheckRefresh(t) + cfg := escalationTestConfig() + pr := func(head string) github.PullRequest { + p := missingCheckPR(686, head) + p.Protection.MissingRequiredChecks = []string{"build", "e2e"} + return p + } + f := &fakeMissingCheckRefresher{bases: baseRunning("base1", "build"), out: github.StaleRedRefreshOutcome{Updated: true, Settled: true}} + now := time.Date(2026, 11, 18, 9, 0, 0, 0, time.UTC) + refreshMissingCheckBehindPRs(context.Background(), cfg, f, actionableWith(pr("h1")), nil, discardLogger(), now) + if len(f.calls) != 1 || f.checks[0] != "build" { + t.Fatalf("first pass: calls=%d checks=%v, want one update for build", len(f.calls), f.checks) + } + // e2e's workflow now runs on the base; build still has its update spent. + f.bases = baseRunning("base2", "build", "e2e") + refreshMissingCheckBehindPRs(context.Background(), cfg, f, actionableWith(pr("h2")), nil, discardLogger(), now.Add(staleRedRefreshInterval)) + if len(f.calls) != 2 || f.checks[1] != "e2e" { + t.Fatalf("second pass: calls=%d checks=%v, want one more update for e2e", len(f.calls), f.checks) + } + f.bases = baseRunning("base3", "build", "e2e") + refreshMissingCheckBehindPRs(context.Background(), cfg, f, actionableWith(pr("h3")), nil, discardLogger(), now.Add(2*staleRedRefreshInterval)) + if len(f.calls) != 2 { + t.Fatalf("third pass: calls=%d, want no more updates", len(f.calls)) + } +} diff --git a/src/cmd/hive/stage_shared_ci.go b/src/cmd/hive/stage_shared_ci.go index 3d6f957b3b..ed8e6901ee 100644 --- a/src/cmd/hive/stage_shared_ci.go +++ b/src/cmd/hive/stage_shared_ci.go @@ -20,6 +20,9 @@ type stageSharedCIEntry struct { // is from a fork; set only when every failing check does // (PullRequest.ForkGuardChecks). forkGuard []string + // heldRuns names the workflow runs GitHub holds for approval on the head + // that the hive did not approve (PullRequest.HeldWorkflowRunIDs). + heldRuns []int64 } // stageSharedCIIndex holds, per "repo#number", the open [shared-ci] incident @@ -46,7 +49,8 @@ func publishStageSharedCIIncidents(actionable *github.ActionableResult) { e.baseRed = append([]string(nil), pr.BaseRedChecks...) } e.forkGuard = pr.ForkGuardChecks() - if e.incident > 0 || len(e.baseRed) > 0 || len(e.forkGuard) > 0 { + e.heldRuns = pr.HeldWorkflowRunIDs() + if e.incident > 0 || len(e.baseRed) > 0 || len(e.forkGuard) > 0 || len(e.heldRuns) > 0 { idx[stageSharedCIKey(pr.Repo, pr.Number)] = e } } @@ -116,6 +120,18 @@ func stageForkGuard(repo string, number int, head string) []string { return append([]string(nil), e.forkGuard...) } +// stageHeldWorkflowRuns answers StageLabelPolicy.HeldWorkflowRuns with the +// same repo/head matching as stageSharedCIIncident: the workflow runs GitHub +// holds for approval on the head that the hive did not approve. A new head +// is not covered until the next enumeration probes it. +func stageHeldWorkflowRuns(repo string, number int, head string) []int64 { + e, ok := stageSharedCILookup(repo, number, head) + if !ok || len(e.heldRuns) == 0 { + return nil + } + return append([]int64(nil), e.heldRuns...) +} + func stageSharedCILookup(repo string, number int, head string) (stageSharedCIEntry, bool) { idx := stageSharedCIIndex.Load() if idx == nil { diff --git a/src/cmd/hive/stage_shared_ci_test.go b/src/cmd/hive/stage_shared_ci_test.go index bafbf83757..b7250f9bb5 100644 --- a/src/cmd/hive/stage_shared_ci_test.go +++ b/src/cmd/hive/stage_shared_ci_test.go @@ -86,3 +86,25 @@ func TestStageBaseRedRemembered(t *testing.T) { t.Fatalf("after an empty publish: %v", got) } } + +func TestStageHeldWorkflowRunsIndex(t *testing.T) { + t.Cleanup(func() { stageSharedCIIndex.Store(nil) }) + held := &github.PRWorkflowApproval{Pending: 2, RunIDs: []int64{44261912124, 44257752419}, Denied: true} + undecided := &github.PRWorkflowApproval{Pending: 1, RunIDs: []int64{11}} + act := &github.ActionableResult{} + act.PRs.Items = []github.PullRequest{ + {Repo: "acme/server", Number: 686, HeadSHA: "aaa", CIActivity: &github.PRCIActivity{Head: "aaa", WorkflowApproval: held}}, + {Repo: "acme/server", Number: 687, HeadSHA: "bbb", CIActivity: &github.PRCIActivity{Head: "bbb", WorkflowApproval: undecided}}, + } + publishStageSharedCIIncidents(act) + + if got := stageHeldWorkflowRuns("acme/server", 686, "aaa"); len(got) != 2 || got[0] != 44261912124 { + t.Fatalf("held runs = %v", got) + } + if got := stageHeldWorkflowRuns("acme/server", 686, "new-head"); got != nil { + t.Fatalf("a new head must not inherit the old head's held runs: %v", got) + } + if got := stageHeldWorkflowRuns("acme/server", 687, "bbb"); got != nil { + t.Fatalf("runs the approval sweep has not decided on read as held: %v", got) + } +} diff --git a/src/docs/github-write-surface.md b/src/docs/github-write-surface.md index 944c675909..f556527608 100644 --- a/src/docs/github-write-surface.md +++ b/src/docs/github-write-surface.md @@ -163,7 +163,7 @@ writes no entry. | Shared-CI rerun (`rerun-failed-jobs`, or `rerun` for a `cancelled` run; `hive-shared-ci-rerun-` + `hive-shared-ci-lifted-` comment) | `shared_ci_rerun.go` | yes (`shared-ci-rerun`) | yes | | Transient-CI rerun (`rerun-failed-jobs` on a hive or trusted-bot PR whose failing runs carry `hive-ci-transient`, once per head, no comment) | `transient_ci_rerun.go` | yes (`transient-ci-rerun`) | yes | | Stale-red base refresh (`update-branch` on a red PR behind its base whose failing checks are green on the base tip, once per head and base tip, no comment) | `stale_red_base_refresh.go` | yes (`stale-red-base-refresh`) | yes | -| Missing-check base refresh (`update-branch` on a PR behind its base whose head never ran a required check that runs on the base tip or another open PR, once per head and base tip and once per check, no comment) | `missing_check_base_refresh.go` | yes (`missing-check-base-refresh`) | yes | +| Missing-check base refresh (`update-branch` on a PR behind its base whose head never ran a required check that runs on the base tip or another open PR, once per head and base tip and once per set of missing checks, never while GitHub holds the head's runs for approval, no comment) | `missing_check_base_refresh.go` | yes (`missing-check-base-refresh`) | yes | | Dependency-bot rebase request (ticks only Renovate's rebase/retry checkbox line in the PR description, once per head, on a bot PR the bot stopped rebasing; write-back kind `bot_rebase_request`; Dependabot is never asked, because it refuses commands from apps) | `dependency_bot_rebase.go` | yes (`dependency_bot_rebase_requested`) | yes | | Open-PR dispatch guard (`has-open-pr` label on a refused issue) and duplicate-fix sweep (one `hive:duplicate-fix` comment; `hive/superseded` close of the later hive PR, or `needs-human` + `Parked` fact on escalation) | `open_pr_links.go`, `pr_duplicate_fix_sweep.go` | yes (`duplicate-fix-*`) | yes | | Duplicate-PR sweep suggestions | `duplicate_sweep.go` | yes (`duplicate-sweep-suggested`) | yes (target = surviving PR) | diff --git a/src/docs/labels-and-control-signals/stage-labels.md b/src/docs/labels-and-control-signals/stage-labels.md index 6eb16b09da..bcefc7994b 100644 --- a/src/docs/labels-and-control-signals/stage-labels.md +++ b/src/docs/labels-and-control-signals/stage-labels.md @@ -414,6 +414,7 @@ used to admit the PR: | CI red (required or optional check) while the PR is deferred to an open `[shared-ci]` incident (``, the governor's `deferred_incident`) for the same head | `050-review-pending` / `060-reviewed`, reason `shared-ci-incident:`: the base or a shared check is what is red, so the PR is never sent to 070/071/072, never nudged and never stalled. A 072 nudge already posted for the head (the nudge goes out on entering 072, and the incident marker can land minutes later) is edited in place to say it was withdrawn, and the head's nudge count is reset. Once the incident closes the PR stays deferred until the shared-CI rerun sweep has re-run its failed jobs (at most 30 minutes); a head still red after that is handed back as usual, with a fresh nudge. Conflicts and requested changes stay author-facing ([#12606](https://github.com/hivecommons/hive/issues/12606)) | | CI red (required or optional) while **every** failing check on the head is also red on the base branch's tip — the governor's `base_red_checks` / `base_red`, set during CI enrichment when a same-named latest check run on the base ref (meta checks excluded) concluded failure / timed_out / cancelled / action_required / startup_failure / stale **and** the PR's changed files include none of `.github/workflows/*`, `Dockerfile` / `Dockerfile.*` / `*.Dockerfile`, `go.mod` / `go.sum` (a PR touching those plausibly owns the check and stays author-facing; a diff beyond the file-list page budget also stays author-facing). The governor remembers each base's red checks across passes: a check stays red on the base until a read sees the tip's run of it complete green, so a tip whose run is still queued or in progress, or a base listing that fails (logged at Warn), keeps the base's last red set; a check not seen red for 2 h is dropped. A new head the CI enrichment has not read yet is classified from that memory at the stage pass: every check red on the head (its check digest) is remembered red on the base and the head touches no CI-owning path (an unreadable file list stays author-facing) | `050-review-pending` / `060-reviewed`, reason `base-branch-red:,` ("waiting for the base branch's CI to recover (…) — not this PR's fault"): never 070/071/072, never nudged (a 072 nudge already posted for the head is edited in place to say it was withdrawn), never stalled, still not merged (the lane's skip reason is the same token). The fix loop counts no attempt, kicks no fixer, and posts one `` hold comment per head; an escalation the hive itself raised (ledger `label_applied`) over such a head has its `needs-human` removed with the hold wording. The next evaluation after the base goes green re-evaluates the head: a still-red PR is handed back as usual. Conflicts and requested changes stay author-facing (seen on [#13270](https://github.com/hivecommons/hive/pull/13270) and [#13258](https://github.com/hivecommons/hive/pull/13258), 2026-10-20) | | CI red (required or optional) on a **fork** PR while **every** failing check's annotation says the check refuses forks (case-insensitive `fork pull request(s) cannot`, `maintainer must push this branch`, or `cannot publish` together with `fork` — a fork-publication / fork-secrets guard) | `085-needs-decision`, reason `fork-guard:,`: never 070/071/072, never nudged, never adopted (074) — neither the author nor the hive can turn the check green by pushing to the fork. The entry comment names the check and says a maintainer must push the branch to the repository (or run or approve it for head `` as the repository requires). A new head on the same fork PR (a merge of the base, or the author's push) keeps the guard while the lane's red check is one of the recorded `fork-guard:` checks, so the PR does not drop to 072 and nobody is nudged before the annotation for that head is read. A fork PR head that nothing has classified yet (its first red head, or a PR that came to 072 for another reason first) is classified from its own check annotations with the same rule, once per head, before it can be nudged: a head red only on fork-guarded checks goes straight to 085 and the guard is recorded for later heads (projectbluefin/utah-packages#972, where the author was nudged and sent relay rework for a fork-gated Canary five minutes before the annotation was read). The hive also does not update the branch of a fork head that waits on a fork-guarded check: the run a maintainer dispatches proves one exact head, and a merge of the base would discard it. Once the check is green the guard drops and updates resume before merge. While the PR stays parked across new heads, the entry comment is rewritten in place for each one, so it always names the current head, and it says that a run which passed on an earlier head does not cover this one even when the patch is the same, because a rebase or a merge of the base changes the code the check runs (projectbluefin/utah-packages#917). A same-repo PR with the same message, or a fork PR with any other red check, is handed back as usual ([#14273](https://github.com/hivecommons/hive/issues/14273), projectbluefin/utah-packages#862 and #869) | +| GitHub holds workflow runs on the head behind its approve-workflows gate (`action_required`), and the hive's approval sweep decided it will not approve them: its trust policy does not cover the head (an untrusted author or a fork head), `governor.ci.auto_approve_workflows` is off, or GitHub refused the approval (no `actions: write`). Whatever the lane said next (`contributor-lane:author-association`, `contributor-lane:no-linked-issue`, `contributor-lane:no-verdict`, a lane error) | `085-needs-decision`, reason `workflow-approval-required:held-runs:,`: no required check can report until a maintainer clicks **Approve and run** (or approves the head on the dashboard, which the sweep honours), so this wins over the lane's own reason. A trusted head the sweep has not decided on yet is not parked. Closed, merged, draft, held and sentinel-flagged PRs keep their own stage, and a lane that already reports `workflow-approval-required:` (after posting `/ok-to-test`) or `unauthorized:` keeps its own, more specific reason. The reason clears on the next enumeration after the runs start or the head changes. The missing-check base refresh never updates such a PR: a new head's runs would be held again (projectbluefin/server#686). | | any **non-required** check on the current head is red, whatever the lane's reason (the lane gates on required checks only, so it still soaks and merges; a reader of the rollup sees ✗). Parking modifiers (`hold`, `needs-human`, admission reasons) still win | `070-fixing` / `071-needs-author` / `072-returned-to-contributor` (by author kind), reason `optional-check-failing:` (the nudge lists every red check); never 080 or 100. A red *required* check is the lane's own `required-check-failing:*` → 070/071/072 ([#12538](https://github.com/hivecommons/hive/issues/12538), seen on [#12457](https://github.com/hivecommons/hive/pull/12457)) | | `label:hold`, `label:blocked`, `held`, `sentinel`, `contributor-lane:sentinel-blocked` | **gate**, not a stage ([#17327](https://github.com/hivecommons/hive/issues/17327)): routed by the head's evidence (below) exactly as a trust cause — CI / 070 while checks run or fail, 071 on a reviewer question, `090-ready-for-human-merge` when the head is otherwise clean — with the gate reason kept on the route so the popover and the entry comment say "Held: …". The lanes refuse the merge on the label, never on the stage. `label:blocked` is a PR wearing `blocked`: its gate reason reads `blocked on owner/repo#N` when the blocked sweep has read the pull request it waits on. The lane stops at the gate before it reads mergeability, so the route takes the head's `mergeable_state` from the PR detail the enumeration fetched for the held PR this cycle (no extra request), for exactly the current head: a `dirty` head goes back to its author like any conflict (070 / 071 / 072, reason `conflicting`, the 072 nudge and fix queue included) and the gate stays a pill and a hard merge stop. `unknown`, or a state read for an older head, is not a conflict; a pass with no state at all keeps a conflict the stage already returned on the same head | | `label:needs-human`, `exempt-label`, `excluded-label:*`, `human-merge-path*`, `intent-*`, `contributor-lane:workflow-path` / `tests-removed` / `no-reviewer` / `review-off` (a verdict is required but the review swarm is off, so none will be requested) / `reporter-trust-hold`, `workflow-permission` (GitHub refused update-branch or the merge-forward with a 403 because the base changed `.github/workflows/` and the token the hive used lacks the workflows permission; no token the hive holds can push that update, so it is not sent to 070), `fork-workflow-update` (the same update-branch refusal on a fork head: the App is not installed on the fork, so no grant lets it push there), `linked-issue:hold:#N: