diff --git a/changelog.d/fixed-base-red-after-restart.md b/changelog.d/fixed-base-red-after-restart.md new file mode 100644 index 0000000000..3745ab0b57 --- /dev/null +++ b/changelog.d/fixed-base-red-after-restart.md @@ -0,0 +1 @@ +- After a restart, a PR red only on its base branch's own check is no longer sent back to its author, or adopted and superseded, before the first CI enumeration finishes. The base-red memory and index live in the process and stay empty until the first CI enumeration finishes (3 to 27 minutes on knuckle on 2026-11-18), so every base read as not red: projectbluefin/common#1689, at 072 past its grace, was adopted at once for main's red E2E check, losing a maintainer's approval, and its copy stayed red for the same reason ([projectbluefin/common#1689](https://github.com/projectbluefin/common/pull/1689)). Until the memory is published, the stage pass reads the base tip's checks itself, at most once per base every 10 minutes, and automatic adoption waits; an operator's Adopt now or adopt-supersede request does not. diff --git a/src/cmd/hive/merge_eligibility.go b/src/cmd/hive/merge_eligibility.go index 8f785f6d90..e31e7103c8 100644 --- a/src/cmd/hive/merge_eligibility.go +++ b/src/cmd/hive/merge_eligibility.go @@ -167,6 +167,7 @@ func stageLabelPolicyFunc(cfg *config.Config) func() automerge.StageLabelPolicy SharedCIIncident: stageSharedCIIncident, BaseBranchRed: stageBaseBranchRed, BaseRedRemembered: stageBaseRedRemembered, + BaseRedRememberedReady: stageBaseRedRememberedReady, ForkGuard: stageForkGuard, HeldWorkflowRuns: stageHeldWorkflowRuns, DCORequired: stage.DCO.RequiredFor, diff --git a/src/cmd/hive/stage_shared_ci.go b/src/cmd/hive/stage_shared_ci.go index ed8e6901ee..0acc3fea1c 100644 --- a/src/cmd/hive/stage_shared_ci.go +++ b/src/cmd/hive/stage_shared_ci.go @@ -86,6 +86,14 @@ func stageBaseRedRemembered(repo, baseRef string) []string { return append([]string(nil), (*m)[github.BaseRedRememberedKey(repo, baseRef)]...) } +// stageBaseRedRememberedReady answers +// StageLabelPolicy.BaseRedRememberedReady: whether a CI-enriched +// enumeration has published the memory since the hive started. Until then +// the stage reconciler reads base tips itself (projectbluefin/common#1689). +func stageBaseRedRememberedReady() bool { + return stageBaseRedMemory.Load() != nil +} + // stageSharedCIIncident answers StageLabelPolicy.SharedCIIncident. repo is // "owner/name"; the snapshot may key a PR by either form. A head pushed // since the snapshot is not covered: the deferral was decided for the old diff --git a/src/cmd/hive/stage_shared_ci_test.go b/src/cmd/hive/stage_shared_ci_test.go index b7250f9bb5..c733767dda 100644 --- a/src/cmd/hive/stage_shared_ci_test.go +++ b/src/cmd/hive/stage_shared_ci_test.go @@ -74,6 +74,9 @@ func TestStageBaseRedRemembered(t *testing.T) { if got := stageBaseRedRemembered("acme/widget", "main"); got != nil { t.Fatalf("nothing published: %v", got) } + if stageBaseRedRememberedReady() { + t.Fatal("ready before any publish") + } publishStageBaseRedMemory(map[string][]string{github.BaseRedRememberedKey("acme/widget", "main"): {"Tests Passed", "Unit Tests"}}) if got := stageBaseRedRemembered("Acme/Widget", "main"); len(got) != 2 || got[0] != "Tests Passed" || got[1] != "Unit Tests" { t.Fatalf("remembered = %v", got) @@ -85,6 +88,9 @@ func TestStageBaseRedRemembered(t *testing.T) { if got := stageBaseRedRemembered("acme/widget", "main"); len(got) != 0 { t.Fatalf("after an empty publish: %v", got) } + if !stageBaseRedRememberedReady() { + t.Fatal("an empty publish is still a publish") + } } func TestStageHeldWorkflowRunsIndex(t *testing.T) { diff --git a/src/docs/labels-and-control-signals/stage-labels.md b/src/docs/labels-and-control-signals/stage-labels.md index 1cae17b604..14857bda05 100644 --- a/src/docs/labels-and-control-signals/stage-labels.md +++ b/src/docs/labels-and-control-signals/stage-labels.md @@ -412,7 +412,7 @@ used to admit the PR: | `contributor-lane:soaking(...)` — and no red check on the head | `095-pre-merge-soak` | | `head-changed-since-eval`, `merge-failed`, `merge-queue-required` (the base branch's ruleset only accepts merges through GitHub's merge queue and the transport is a static token that cannot enqueue, so the direct REST merge is refused; with App auth on a repo opted in through `auto_merge.native_merge_queue` these bases are enqueued instead, see [native-merge-queue.md](../native-merge-queue.md); logged as a skip rather than a merge error, [#14018](https://github.com/hivecommons/hive/issues/14018); the self-authored sweep then skips the direct merge on that base for 30 minutes and logs the skip once per PR head, [#14068](https://github.com/hivecommons/hive/issues/14068); the trusted-author, queued and contributor-lane sweeps skip it the same way, sharing one memory of such bases; while a base is remembered the dashboard shows its otherwise-green PRs as a person's to merge, not Merge-eligible; with `auto_merge.require_current_base` on, a head behind such a base is not updated for 24 hours after the last refusal and holds here instead, so the base moving does not reset its review), `required-checks-expected`, `behind`, `updated-branch`, `behind-update-budget-exhausted` (a behind self-authored PR the sweep did not update only because this tick's branch-update budget was spent; retried next tick, waits on the hive, [#12607](https://github.com/hivecommons/hive/issues/12607)), `release-hold*`, `native-merge-queued: queued (position N, STATE)` (the PR was enqueued into GitHub's native merge queue) and the failures `native-merge-queue-read`, `native-merge-queue-enqueue`, `native-merge-queue-method`, `native-merge-queue-rejected` (the merge queue removed the hive's own entry for this head without merging it; see [native-merge-queue.md](../native-merge-queue.md#current-base-gate)) — and no red check on the head | `100-merge-queue` | | CI red (required or optional check) while the PR is deferred to an open `[shared-ci]` incident (``, the governor's `deferred_incident`) for the same head | `050-review-pending` / `060-reviewed`, reason `shared-ci-incident:`: the base or a shared check is what is red, so the PR is never sent to 070/071/072, never nudged and never stalled. A 072 nudge already posted for the head (the nudge goes out on entering 072, and the incident marker can land minutes later) is edited in place to say it was withdrawn, and the head's nudge count is reset. Once the incident closes the PR stays deferred until the shared-CI rerun sweep has re-run its failed jobs (at most 30 minutes); a head still red after that is handed back as usual, with a fresh nudge. Conflicts and requested changes stay author-facing ([#12606](https://github.com/hivecommons/hive/issues/12606)) | -| CI red (required or optional) while **every** failing check on the head is also red on the base branch's tip — the governor's `base_red_checks` / `base_red`, set during CI enrichment when a same-named latest check run on the base ref (meta checks excluded) concluded failure / timed_out / cancelled / action_required / startup_failure / stale **and** the PR's changed files include none of `.github/workflows/*`, `Dockerfile` / `Dockerfile.*` / `*.Dockerfile`, `go.mod` / `go.sum` (a PR touching those plausibly owns the check and stays author-facing; a diff beyond the file-list page budget also stays author-facing). The governor remembers each base's red checks across passes: a check stays red on the base until a read sees the tip's run of it complete green, so a tip whose run is still queued or in progress, or a base listing that fails (logged at Warn), keeps the base's last red set; a check not seen red for 2 h is dropped. A new head the CI enrichment has not read yet is classified from that memory at the stage pass: every check red on the head (its check digest) is remembered red on the base and the head touches no CI-owning path (an unreadable file list stays author-facing) | `050-review-pending` / `060-reviewed`, reason `base-branch-red:,` ("waiting for the base branch's CI to recover (…) — not this PR's fault"): never 070/071/072, never nudged (a 072 nudge already posted for the head is edited in place to say it was withdrawn), never stalled, still not merged (the lane's skip reason is the same token). The fix loop counts no attempt, kicks no fixer, and posts one `` hold comment per head; an escalation the hive itself raised (ledger `label_applied`) over such a head has its `needs-human` removed with the hold wording. The next evaluation after the base goes green re-evaluates the head: a still-red PR is handed back as usual. Conflicts and requested changes stay author-facing (seen on [#13270](https://github.com/hivecommons/hive/pull/13270) and [#13258](https://github.com/hivecommons/hive/pull/13258), 2026-10-20) | +| CI red (required or optional) while **every** failing check on the head is also red on the base branch's tip — the governor's `base_red_checks` / `base_red`, set during CI enrichment when a same-named latest check run on the base ref (meta checks excluded) concluded failure / timed_out / cancelled / action_required / startup_failure / stale **and** the PR's changed files include none of `.github/workflows/*`, `Dockerfile` / `Dockerfile.*` / `*.Dockerfile`, `go.mod` / `go.sum` (a PR touching those plausibly owns the check and stays author-facing; a diff beyond the file-list page budget also stays author-facing). The governor remembers each base's red checks across passes: a check stays red on the base until a read sees the tip's run of it complete green, so a tip whose run is still queued or in progress, or a base listing that fails (logged at Warn), keeps the base's last red set; a check not seen red for 2 h is dropped. A new head the CI enrichment has not read yet is classified from that memory at the stage pass: every check red on the head (its check digest) is remembered red on the base and the head touches no CI-owning path (an unreadable file list stays author-facing). The memory lives in the process and is first published when the first CI enrichment after a start finishes; until then the stage pass reads the base tip itself, at most once per base every 10 minutes (a failed read is retried after a minute), and a head whose every red check is red there is base red the same way. That read is the head digest's own (commit statuses as well as check runs, latest run per name and app, ignorable checks skipped), not the enrichment's check-runs-only read; a check still running on the tip, or with no run there, is not red, and an unreadable tip stays author-facing. Because that read can miss the base's red, an automatic adoption also waits until the memory is published (logged once per head as `adoption deferred until the base-red memory is published`); an operator's Adopt now or `adoption/supersede` request does not wait. Before this, every base read as not red for that window, and a PR already past its 072 grace was adopted for the base's red ([projectbluefin/common#1689](https://github.com/projectbluefin/common/pull/1689), 2026-11-18) | `050-review-pending` / `060-reviewed`, reason `base-branch-red:,` ("waiting for the base branch's CI to recover (…) — not this PR's fault"): never 070/071/072, never nudged (a 072 nudge already posted for the head is edited in place to say it was withdrawn), never stalled, still not merged (the lane's skip reason is the same token). The fix loop counts no attempt, kicks no fixer, and posts one `` hold comment per head; an escalation the hive itself raised (ledger `label_applied`) over such a head has its `needs-human` removed with the hold wording. The next evaluation after the base goes green re-evaluates the head: a still-red PR is handed back as usual. Conflicts and requested changes stay author-facing (seen on [#13270](https://github.com/hivecommons/hive/pull/13270) and [#13258](https://github.com/hivecommons/hive/pull/13258), 2026-10-20) | | CI red (required or optional) on a **fork** PR while **every** failing check's annotation says the check refuses forks (case-insensitive `fork pull request(s) cannot`, `maintainer must push this branch`, or `cannot publish` together with `fork` — a fork-publication / fork-secrets guard) | `085-needs-decision`, reason `fork-guard:,`: never 070/071/072, never nudged, never adopted (074) — neither the author nor the hive can turn the check green by pushing to the fork. The entry comment names the check and says a maintainer must push the branch to the repository (or run or approve it for head `` as the repository requires). A new head on the same fork PR (a merge of the base, or the author's push) keeps the guard while the lane's red check is one of the recorded `fork-guard:` checks, so the PR does not drop to 072 and nobody is nudged before the annotation for that head is read. A fork PR head that nothing has classified yet (its first red head, or a PR that came to 072 for another reason first) is classified from its own check annotations with the same rule, once per head, before it can be nudged: a head red only on fork-guarded checks goes straight to 085 and the guard is recorded for later heads (projectbluefin/utah-packages#972, where the author was nudged and sent relay rework for a fork-gated Canary five minutes before the annotation was read). The hive also does not update the branch of a fork head that waits on a fork-guarded check: the run a maintainer dispatches proves one exact head, and a merge of the base would discard it. Once the check is green the guard drops and updates resume before merge. While the PR stays parked across new heads, the entry comment is rewritten in place for each one, so it always names the current head, and it says that a run which passed on an earlier head does not cover this one even when the patch is the same, because a rebase or a merge of the base changes the code the check runs (projectbluefin/utah-packages#917). A same-repo PR with the same message, or a fork PR with any other red check, is handed back as usual ([#14273](https://github.com/hivecommons/hive/issues/14273), projectbluefin/utah-packages#862 and #869) | | GitHub holds workflow runs on the head behind its approve-workflows gate (`action_required`), and the hive's approval sweep decided it will not approve them: its trust policy does not cover the head (an untrusted author or a fork head), `governor.ci.auto_approve_workflows` is off, or GitHub refused the approval (no `actions: write`). Whatever the lane said next (`contributor-lane:author-association`, `contributor-lane:no-linked-issue`, `contributor-lane:no-verdict`, a lane error) | `085-needs-decision`, reason `workflow-approval-required:held-runs:,`: no required check can report until a maintainer clicks **Approve and run** (or approves the head on the dashboard, which the sweep honours), so this wins over the lane's own reason. A trusted head the sweep has not decided on yet is not parked. Closed, merged, draft, held and sentinel-flagged PRs keep their own stage, and a lane that already reports `workflow-approval-required:` (after posting `/ok-to-test`) or `unauthorized:` keeps its own, more specific reason. The reason clears on the next enumeration after the runs start or the head changes. The missing-check base refresh never updates such a PR: a new head's runs would be held again (projectbluefin/server#686). | | any **non-required** check on the current head is red, whatever the lane's reason (the lane gates on required checks only, so it still soaks and merges; a reader of the rollup sees ✗). Parking modifiers (`hold`, `needs-human`, admission reasons) still win | `070-fixing` / `071-needs-author` / `072-returned-to-contributor` (by author kind), reason `optional-check-failing:` (the nudge lists every red check); never 080 or 100. A red *required* check is the lane's own `required-check-failing:*` → 070/071/072 ([#12538](https://github.com/hivecommons/hive/issues/12538), seen on [#12457](https://github.com/hivecommons/hive/pull/12457)) | diff --git a/src/pkg/github/automerge/automerge_base_red_carry.go b/src/pkg/github/automerge/automerge_base_red_carry.go index 0daa4c0fb5..368672e914 100644 --- a/src/pkg/github/automerge/automerge_base_red_carry.go +++ b/src/pkg/github/automerge/automerge_base_red_carry.go @@ -28,6 +28,22 @@ import ( // CI-owning path (hgithub.TouchesCIOwningPath). Any other red check, a // CI-owning path, or a file list that could not be read keeps today's // author-facing route. +// +// After a restart (projectbluefin/common#1689). The memory and the index +// live in the process and are published only when the first CI-enriched +// enumeration finishes, which took up to 27 minutes on knuckle on +// 2026-11-18. In that window every base read as not red: common's PRs red +// only on main's own E2E check went to 070/072, and #1689 (and #1690's +// first adoption), past the 072 grace, was adopted at 1:42 PM EST and +// superseded, losing a maintainer's approval, into a copy red for the same +// reason. Until the memory is published +// (StageLabelPolicy.BaseRedRememberedReady), a red head whose red checks the +// memory does not cover reads its base tip directly, once per base per +// humanChecksTTL (baseTipRedChecks), and is base red when every check red +// on it is red there too. That read can still miss the base's red (a run in +// progress on the tip, a tip without the check, a failed read), so +// automatic adoption also waits for the publish (adoptContributorPR); an +// operator's adoption does not. // baseRedFilesPages bounds the head's file listing (100 files per page), // like the enumeration's own (hgithub.baseRedMaxFilePages): a longer list is @@ -51,8 +67,71 @@ func stageBaseRedRemembered(policy StageLabelPolicy, owner, repo, baseRef string return policy.BaseRedRemembered(owner+"/"+repo, baseRef) } +// stageBaseRedRememberedReady reports whether the policy's base-red memory +// has been published since the hive started; nil reads as ready. +func stageBaseRedRememberedReady(policy StageLabelPolicy) bool { + return policy.BaseRedRememberedReady == nil || policy.BaseRedRememberedReady() +} + +// baseTipRedErrorTTL bounds how long a failed base tip read is reused: a +// minute, not humanChecksTTL, so one API error does not leave the base +// author-facing for the rest of a short startup window. +const baseTipRedErrorTTL = time.Minute + +// baseTipRedEntry is one base branch tip's red checks as baseTipRedChecks +// read them, and when; ok is false when the read failed. +type baseTipRedEntry struct { + at time.Time + red []string + ok bool +} + +// baseTipRedChecks lists the checks red on obs's base branch tip (its +// combined status and latest check run per name and app, as the head's own +// digest reads them: commit statuses as well as check runs), cached per +// base for humanChecksTTL and a failed read for baseTipRedErrorTTL, so a +// pass costs at most one read per base. A check still queued or in progress +// on the tip, or with no run there, is not red. ok is false when the read +// failed. +func (c *Engine) baseTipRedChecks(ctx context.Context, tracker *StageTracker, obs stageObservation, now time.Time) ([]string, bool) { + key := hgithub.BaseRedRememberedKey(obs.Owner+"/"+obs.Repo, obs.BaseRef) + tracker.mu.Lock() + entry, cached := tracker.baseTipRed[key] + tracker.mu.Unlock() + ttl := humanChecksTTL + if !entry.ok { + ttl = baseTipRedErrorTTL + } + if cached && now.Sub(entry.at) < ttl { + return entry.red, entry.ok + } + summary, err := c.summarizeChecks(ctx, obs.Owner, obs.Repo, obs.BaseRef, func() map[string]bool { return nil }) + if err != nil { + c.debug("stage labels: base tip checks unavailable; a red head stays author-facing", "repo", obs.Display, "base", obs.BaseRef, "error", err) + } + entry = baseTipRedEntry{at: now, red: slices.Clone(summary.Red), ok: err == nil} + tracker.mu.Lock() + if tracker.baseTipRed == nil { + tracker.baseTipRed = map[string]baseTipRedEntry{} + } + tracker.baseTipRed[key] = entry + tracker.mu.Unlock() + return entry.red, entry.ok +} + +// allIn reports whether every name in names is in set. +func allIn(names, set []string) bool { + for _, name := range names { + if !slices.Contains(set, name) { + return false + } + } + return true +} + // foldBaseRed marks a red PR head base red when the index has no answer for -// it and every check red on it is remembered red on its base. +// it and every check red on it is remembered red on its base, or, before the +// memory is published, red on the base tip. func (c *Engine) foldBaseRed(ctx context.Context, tracker *StageTracker, list []stageObservation, now time.Time) { var o *stageObservation for i := range list { @@ -77,8 +156,10 @@ func (c *Engine) foldBaseRed(ctx context.Context, tracker *StageTracker, list [] if !red { return } - remembered := stageBaseRedRemembered(c.stagePolicy(), o.Owner, o.Repo, o.BaseRef) - if len(remembered) == 0 { + policy := c.stagePolicy() + remembered := stageBaseRedRemembered(policy, o.Owner, o.Repo, o.BaseRef) + ready := stageBaseRedRememberedReady(policy) + if len(remembered) == 0 && ready { return } key := stageKey(o.Owner, o.Repo, o.Number) @@ -86,10 +167,18 @@ func (c *Engine) foldBaseRed(ctx context.Context, tracker *StageTracker, list [] if !known || len(summary.Red) == 0 { return } - for _, name := range summary.Red { - if !slices.Contains(remembered, name) { + source := "remembered" + if !ready && !allIn(summary.Red, remembered) { + // Nothing published since the hive started: ask the base tip. + tip, ok := c.baseTipRedChecks(ctx, tracker, *o, now) + if !ok { return } + remembered = append(slices.Clone(remembered), tip...) + source = "base-tip" + } + if !allIn(summary.Red, remembered) { + return } touches, known := c.headTouchesCIOwningPath(ctx, tracker, key, *o) if !known || touches { @@ -103,7 +192,7 @@ func (c *Engine) foldBaseRed(ctx context.Context, tracker *StageTracker, list [] list[i].BaseRedChecks = append([]string(nil), checks...) } } - c.debug("stage labels: base-branch red read for a head the enumeration has not classified", "repo", o.Display, "pr", o.Number, "head", head, "base", o.BaseRef, "checks", checks) + c.debug("stage labels: base-branch red read for a head the enumeration has not classified", "repo", o.Display, "pr", o.Number, "head", head, "base", o.BaseRef, "checks", checks, "source", source) } // headTouchesCIOwningPath reports whether obs's head changes a CI-owning diff --git a/src/pkg/github/automerge/automerge_base_red_carry_test.go b/src/pkg/github/automerge/automerge_base_red_carry_test.go index 5cf5bb7e2b..ecab84fb9b 100644 --- a/src/pkg/github/automerge/automerge_base_red_carry_test.go +++ b/src/pkg/github/automerge/automerge_base_red_carry_test.go @@ -3,6 +3,7 @@ package automerge import ( "context" "encoding/json" + "errors" "net/http" "strconv" "strings" @@ -183,3 +184,177 @@ func TestFoldBaseRedRedOptionalWhileSoaking(t *testing.T) { t.Fatalf("index answer replaced: %v", got) } } + +// projectbluefin/common#1689: knuckle restarted at 1:26 PM EST on 11-18 and +// its first CI-enriched enumeration published the base-red memory and index +// up to 27 minutes later. At 1:42 PM the PR, red only on main's own E2E +// check and past its 072 grace, was adopted and superseded. Before the +// memory is published, the stage pass now reads the base tip: red there, +// the head goes to 050 base-branch-red and is not adopted; green there, it +// stays at 072 and its automatic adoption waits for the publish. Once the +// memory is published, no tip is read. +func TestBaseRedReadsTheBaseTipUntilTheMemoryIsPublished(t *testing.T) { + const e2e = "E2E — composed common suite / e2e / GNOME — common" + for _, tc := range []struct { + name string + baseRed bool + }{ + {"red on the base tip", true}, + {"green on the base tip", false}, + } { + t.Run(tc.name, func(t *testing.T) { + ready := true + policy := adoptPolicy() + policy.BaseBranchRed = func(string, int, string) []string { return nil } + policy.BaseRedRemembered = func(string, string) []string { return nil } + policy.BaseRedRememberedReady = func() bool { return ready } + h, done := newAdoptHarness(t, ContributorNudgeOutcome{Mode: "spawn", Online: true}, policy) + defer done() + baseReads := 0 + h.e.checkSummary = func(_ context.Context, _, _, sha string) (hgithub.CommitChecksSummary, error) { + if sha == "v5" { + baseReads++ + if tc.baseRed { + return hgithub.CommitChecksSummary{Total: 9, Green: 8, Red: []string{e2e}, DCO: 1}, nil + } + return hgithub.CommitChecksSummary{Total: 9, Green: 9, DCO: 1}, nil + } + return hgithub.CommitChecksSummary{Total: 9, Green: 8, Red: []string{e2e}, DCO: 1}, nil + } + reason := "check-failing:" + e2e + + // Memory published, nothing remembered on v5: 072 with a nudge, + // and no base tip read. + h.observe(adoptPR(1689, "aaa"), reason) + if len(nudgeComments(h.api)) != 1 || baseReads != 0 { + t.Fatalf("published memory: nudges=%d base reads=%d, want 1 and 0", len(nudgeComments(h.api)), baseReads) + } + + // The hive restarts after the grace has run out. + ready = false + h.clock.advance(2 * time.Hour) + h.api.reset() + labels := []string{"stage/072-returned-to-contributor", "role/contributor", "author/clanker"} + h.observe(adoptPR(1689, "aaa", labels...), reason) + h.observe(adoptPR(1689, "aaa", labels...), reason) + if baseReads != 1 { + t.Fatalf("base tip read %d times in two passes, want once", baseReads) + } + rec, _ := h.e.stageStoreOrShared().Get(stageKey("acme", "widget", 1689)) + if !tc.baseRed { + if len(h.adopt.pulls) != 0 || rec.Stage != stagelabels.ReturnedToContributor { + t.Fatalf("green base tip before the publish: pulls=%d stage %s, want 072 and no adoption yet", len(h.adopt.pulls), rec.Stage) + } + // The first enumeration publishes: the adoption goes ahead. + ready = true + h.observe(adoptPR(1689, "aaa", labels...), reason) + if len(h.adopt.pulls) != 1 { + t.Fatalf("green base tip after the publish: pulls=%d, want the adoption as before", len(h.adopt.pulls)) + } + return + } + if len(h.adopt.pulls) != 0 || len(adoptionComments(h.api)) != 0 { + t.Fatalf("adopted for the base's red: pulls=%v comments=%v", h.adopt.pulls, adoptionComments(h.api)) + } + if rec.Stage != stagelabels.ReviewPending || rec.Reason != stageReasonBaseBranchRed+e2e { + t.Fatalf("record %s %q, want 050 %q", rec.Stage, rec.Reason, stageReasonBaseBranchRed+e2e) + } + }) + } +} + +// Before the base-red memory is published an automatic adoption waits, but +// an operator's Adopt now does not. +func TestForcedAdoptionNotHeldBeforeTheBaseRedMemoryIsPublished(t *testing.T) { + policy := adoptPolicy() + policy.BaseRedRememberedReady = func() bool { return false } + h, done := newAdoptHarness(t, ContributorNudgeOutcome{Mode: "spawn", Online: true}, policy) + defer done() + h.e.checkSummary = func(_ context.Context, _, _, sha string) (hgithub.CommitChecksSummary, error) { + if sha == "v5" { + return hgithub.CommitChecksSummary{Total: 3, Green: 3, DCO: 1}, nil + } + return hgithub.CommitChecksSummary{Total: 3, Green: 2, Red: []string{"unit"}, DCO: 1}, nil + } + h.observe(adoptPR(7, "aaa"), "check-failing:unit") + labels := []string{"stage/072-returned-to-contributor", "role/contributor", "author/clanker"} + + // Grace elapsed: still not adopted while nothing is published. + h.clock.advance(2 * time.Hour) + h.observe(adoptPR(7, "aaa", labels...), "check-failing:unit") + if len(h.adopt.pulls) != 0 { + t.Fatalf("automatic adoption before the publish: pulls=%v", h.adopt.pulls) + } + if rec, _ := h.e.stageStoreOrShared().Get(stageKey("acme", "widget", 7)); rec.Adopted("aaa") { + t.Fatalf("a deferred adoption must record nothing: %+v", rec) + } + + h.e.stageStoreOrShared().NoteAdoptionForced(stageKey("acme", "widget", 7), "clubanderson", h.clock.now()) + h.clock.advance(time.Second) + h.observe(adoptPR(7, "aaa", labels...), "check-failing:unit") + if len(h.adopt.pulls) != 1 { + t.Fatalf("Adopt now held for the unpublished memory: pulls=%v", h.adopt.pulls) + } +} + +// common#1689's own route: the E2E check was optional, so the stage read it +// from the detail (optional-check-failing). Before the memory is published +// the base tip answers; a tip that cannot be read leaves the head +// author-facing. +func TestFoldBaseRedOptionalFromTheBaseTipBeforeTheMemoryIsPublished(t *testing.T) { + api, closeAPI := seededStageAPI(t) + defer closeAPI() + api.extra = func(w http.ResponseWriter, r *http.Request) bool { + if r.URL.Path != "/repos/acme/widget/pulls/1689/files" { + return false + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`[{"filename":"system_files/shared/usr/libexec/projectbluefin-countme"}]`)) + return true + } + clock := &contributorLaneClock{at: time.Date(2026, 11, 18, 18, 35, 0, 0, time.UTC)} + const e2e = "E2E — composed common suite / e2e / GNOME — common" + policy := StageLabelPolicy{Enabled: true, BaseRedRememberedReady: func() bool { return false }} + e := newNudgeEngine(t, apiURLOf(t, api), clock, &fakeNudgeSink{}, policy) + tipFails := false + e.checkSummary = func(_ context.Context, _, _, sha string) (hgithub.CommitChecksSummary, error) { + if sha == "main" && tipFails { + return hgithub.CommitChecksSummary{}, errors.New("HTTP 502") + } + return hgithub.CommitChecksSummary{Total: 9, Green: 8, Red: []string{e2e}, DCO: 1}, nil + } + detail := stageDetail{Verdict: ContributorLaneVerdict{Configured: true, Approved: true, Verdict: "approve"}, VerdictKnown: true} + detail.RedChecks, detail.RedChecksHead = []string{e2e}, "9460399a87" + obs := func() []stageObservation { + return []stageObservation{{Lane: stageLaneContributor, Owner: "acme", Repo: "widget", Number: 1689, HeadSHA: "9460399a87", BaseRef: "main", Reason: contributorLaneReasonSoaking + "(1h)", Detail: detail}} + } + list := obs() + e.foldBaseRed(context.Background(), NewStageTracker(), list, clock.now()) + if v := classifyStage(list[0]); v.Stage != stagelabels.Reviewed || v.Reason != stageReasonBaseBranchRed+e2e { + t.Fatalf("red on the base tip: classifyStage = %+v, want 060 %q", v, stageReasonBaseBranchRed+e2e) + } + + tipFails = true + tracker := NewStageTracker() + list = obs() + e.foldBaseRed(context.Background(), tracker, list, clock.now()) + if len(list[0].BaseRedChecks) != 0 { + t.Fatalf("unreadable base tip: BaseRedChecks = %v, want none", list[0].BaseRedChecks) + } + if v := classifyStage(list[0]); v.Reason != stageReasonOptionalCheckFailing+e2e { + t.Fatalf("unreadable base tip: classifyStage = %+v, want %q", v, stageReasonOptionalCheckFailing+e2e) + } + + // A failed read is reused for a minute, not the full humanChecksTTL. + tipFails = false + list = obs() + e.foldBaseRed(context.Background(), tracker, list, clock.now().Add(30*time.Second)) + if len(list[0].BaseRedChecks) != 0 { + t.Fatalf("failed read re-tried within a minute: BaseRedChecks = %v", list[0].BaseRedChecks) + } + list = obs() + e.foldBaseRed(context.Background(), tracker, list, clock.now().Add(baseTipRedErrorTTL)) + if got := list[0].BaseRedChecks; len(got) != 1 || got[0] != e2e { + t.Fatalf("failed read not re-tried after a minute: BaseRedChecks = %v", got) + } +} diff --git a/src/pkg/github/automerge/automerge_contributor_adopt.go b/src/pkg/github/automerge/automerge_contributor_adopt.go index dea0451dd1..a291659ae6 100644 --- a/src/pkg/github/automerge/automerge_contributor_adopt.go +++ b/src/pkg/github/automerge/automerge_contributor_adopt.go @@ -951,6 +951,22 @@ func (c *Engine) adoptContributorPR(ctx context.Context, store *stagelabels.Stor if !due { return 0, nil } + if !requested && !forced && !stageBaseRedRememberedReady(policy) { + // Until the first CI-enriched enumeration after a start publishes + // the base-red memory and index, a base's own red (and an open + // [shared-ci] incident) cannot be told from the PR's: on + // projectbluefin/common#1689 an automatic adoption nine minutes + // after a restart superseded a PR red only on main's E2E check. The + // stage pass's base tip read can miss a run still in progress, a + // tip without the check, or a failed read, and adoption cannot be + // undone, so an automatic one waits for the publish. Nothing is + // recorded; an operator's Adopt now or adopt-supersede request is + // not held. + if c.markAdoptionAwaitingBaseRedLogged(obs, reason, now) { + c.info("stage labels: adoption deferred until the base-red memory is published", "repo", obs.Display, "pr", obs.Number, "author", obs.Author, "head", obs.HeadSHA, "reason", reason) + } + return 0, nil + } if !requested && !forced && adoptionForkGuardOnly(obs) { // The guard asks a maintainer to publish the branch; an in-place // push cannot turn it green and an automatic copy would publish it @@ -2032,6 +2048,14 @@ func (c *Engine) markPersonAdoptionRefusalLogged(obs stageObservation, reason st return c.memory().markHoldLogged(key, now) } +// markAdoptionAwaitingBaseRedLogged records that an automatic adoption +// deferred for the unpublished base-red memory was logged for this PR head +// and due reason, and reports whether this is the first report. +func (c *Engine) markAdoptionAwaitingBaseRedLogged(obs stageObservation, reason string, now time.Time) bool { + key := c.evaluatedHeadKey(obs.Owner, obs.Repo, obs.Number) + "@" + strings.ToLower(obs.HeadSHA) + "#adopt-awaiting-base-red@" + reason + return c.memory().markHoldLogged(key, now) +} + // maintainerPolicy resolves the policy's maintainer rule ("" = adopt). func (p StageLabelPolicy) maintainerPolicy() string { switch strings.ToLower(strings.TrimSpace(p.AdoptionMaintainerPolicy)) { diff --git a/src/pkg/github/automerge/automerge_stage_labels.go b/src/pkg/github/automerge/automerge_stage_labels.go index 545f91cc7a..5c63f2ec2e 100644 --- a/src/pkg/github/automerge/automerge_stage_labels.go +++ b/src/pkg/github/automerge/automerge_stage_labels.go @@ -180,6 +180,13 @@ type StageLabelPolicy struct { // answer for yet is base red when every check red on it is one of these // and it touches no CI-owning path (foldBaseRed). Nil means none. BaseRedRemembered func(repo, baseRef string) []string + // BaseRedRememberedReady reports whether the CI enumeration has + // published BaseRedRemembered (and the BaseBranchRed index) since the + // hive started. Until it has, both are empty for every base, so + // foldBaseRed reads the base tip itself (baseTipRedChecks) and an + // automatic adoption waits (adoptContributorPR). Nil means ready: no + // tip reads, no wait. + BaseRedRememberedReady func() bool // ForkGuard returns the failing checks on a fork PR head (repo // "owner/name", number, head SHA) that refuse it only because it is from // a fork, when EVERY failing check does — @@ -313,6 +320,10 @@ type StageTracker struct { // path (foldBaseRed); created on first use. Per head, like // humanMergePathFence. baseRedPaths map[string]baseRedPathsEntry + // baseTipRed remembers, per base branch, the checks red on its tip + // while the base-red memory is not published yet (baseTipRedChecks); + // created on first use. + baseTipRed map[string]baseTipRedEntry // approvals remembers, per key, the trusted approval the human route // read for a head no lane noted one on (headTrustedApproval); created // on first use. Like humanMergePathFence it survives drain.