From 2c3fb52fd10c41d1d0c9270420f13633f40fe0f4 Mon Sep 17 00:00:00 2001 From: Danathar <6772335+Danathar@users.noreply.github.com> Date: Tue, 17 Nov 2026 17:24:05 -0500 Subject: [PATCH 1/2] =?UTF-8?q?=F0=9F=90=9B=20base=20red:=20remember=20the?= =?UTF-8?q?=20base's=20red=20checks=20and=20read=20new=20heads=20from=20th?= =?UTF-8?q?em?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit enrichBaseBranchRed re-read the base tip every pass with no memory: a tip whose run of the check was still in progress, or a failed listing, made the PR's red its own, so it went to 072, the author was nudged and a relay fix queued, and the next pass withdrew it. A new head read as plain red until the next enumeration. Seen on projectbluefin/common#1684 (ten stage changes on one head) and chairlift#811 (both heads). The client now remembers each base's red checks and drops one only when the tip's run of it completes green (2 h TTL); a failed listing reuses the set and logs at Warn. The stage pass reads a head the index has not classified from that memory when every check red on it is the base's and it touches no CI-owning path. Signed-off-by: Danathar <6772335+Danathar@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BQYnPGFJx81c8JXR3o1sg1 --- .../fixed-base-red-memory-and-new-head.md | 1 + src/cmd/hive/main_eval_cycle.go | 1 + src/cmd/hive/merge_eligibility.go | 1 + src/cmd/hive/repos_rescan.go | 1 + src/cmd/hive/stage_shared_ci.go | 27 +++ src/cmd/hive/stage_shared_ci_test.go | 18 ++ .../stage-labels.md | 2 +- .../automerge/automerge_base_red_carry.go | 166 ++++++++++++++++ .../automerge_base_red_carry_test.go | 185 ++++++++++++++++++ .../automerge/automerge_stage_labels.go | 11 ++ src/pkg/github/base_branch_red.go | 63 ++++-- src/pkg/github/base_branch_red_memory.go | 149 ++++++++++++++ src/pkg/github/base_branch_red_memory_test.go | 136 +++++++++++++ src/pkg/github/client.go | 3 + src/pkg/github/red_main.go | 2 +- 15 files changed, 746 insertions(+), 20 deletions(-) create mode 100644 changelog.d/fixed-base-red-memory-and-new-head.md create mode 100644 src/pkg/github/automerge/automerge_base_red_carry.go create mode 100644 src/pkg/github/automerge/automerge_base_red_carry_test.go create mode 100644 src/pkg/github/base_branch_red_memory.go create mode 100644 src/pkg/github/base_branch_red_memory_test.go diff --git a/changelog.d/fixed-base-red-memory-and-new-head.md b/changelog.d/fixed-base-red-memory-and-new-head.md new file mode 100644 index 0000000000..4a993c9d32 --- /dev/null +++ b/changelog.d/fixed-base-red-memory-and-new-head.md @@ -0,0 +1 @@ +- A PR whose only red checks are also red on its base branch no longer flips between 050/060 and 070/072 while the base stays red. The base's red checks are remembered across passes and drop only when the base tip's run of the check completes green, so a tip whose run is still in progress, or a failed base lookup (now logged at Warn), no longer sends the PR to its author, queues a relay fix and then withdraws the nudge. A new head the CI enrichment has not read yet is classified from that memory when every check red on it is the base's and it touches no CI-owning path. diff --git a/src/cmd/hive/main_eval_cycle.go b/src/cmd/hive/main_eval_cycle.go index 10628a4864..02519fa0d5 100644 --- a/src/cmd/hive/main_eval_cycle.go +++ b/src/cmd/hive/main_eval_cycle.go @@ -742,6 +742,7 @@ func runEvalCycle( } } publishStageSharedCIIncidents(actionable) + publishStageBaseRedMemory(ghClient.BaseRedSnapshot()) // Record enumerated issues into the lifecycle timeline so the dashboard's // lifecycle view has real data. Cheap and fully guarded: a nil dashboard or diff --git a/src/cmd/hive/merge_eligibility.go b/src/cmd/hive/merge_eligibility.go index 5000969a11..f813dad43f 100644 --- a/src/cmd/hive/merge_eligibility.go +++ b/src/cmd/hive/merge_eligibility.go @@ -165,6 +165,7 @@ func stageLabelPolicyFunc(cfg *config.Config) func() automerge.StageLabelPolicy HiveAuthor: hiveIdentity(cfg).Matches, SharedCIIncident: stageSharedCIIncident, BaseBranchRed: stageBaseBranchRed, + BaseRedRemembered: stageBaseRedRemembered, ForkGuard: stageForkGuard, DCORequired: stage.DCO.RequiredFor, SquashOnly: cfg.AutoMerge.SquashOnlyFor, diff --git a/src/cmd/hive/repos_rescan.go b/src/cmd/hive/repos_rescan.go index 2dd804d900..9820e4c9be 100644 --- a/src/cmd/hive/repos_rescan.go +++ b/src/cmd/hive/repos_rescan.go @@ -83,6 +83,7 @@ func rescanRepos( publishEnumeratedSnapshot(lastActionable, actionable, logger) publishStageSharedCIIncidents(actionable) + publishStageBaseRedMemory(ghClient.BaseRedSnapshot()) if data, marshalErr := json.Marshal(actionable); marshalErr == nil { atomicWrite(lastActionablePath, data) } diff --git a/src/cmd/hive/stage_shared_ci.go b/src/cmd/hive/stage_shared_ci.go index 31f25d89a4..3d6f957b3b 100644 --- a/src/cmd/hive/stage_shared_ci.go +++ b/src/cmd/hive/stage_shared_ci.go @@ -55,6 +55,33 @@ func publishStageSharedCIIncidents(actionable *github.ActionableResult) { stageSharedCIIndex.Store(&idx) } +// stageBaseRedMemory holds the github client's remembered base-branch red +// checks (github.Client.BaseRedSnapshot), keyed by +// github.BaseRedRememberedKey, as of the last CI-enriched enumeration. The +// stage reconciler reads it through StageLabelPolicy.BaseRedRemembered so a +// PR head the index has not read yet, red only on its base's checks, is not +// returned to its author before the next enumeration +// (projectbluefin/chairlift#811). +var stageBaseRedMemory atomic.Pointer[map[string][]string] + +// publishStageBaseRedMemory replaces the base-red memory snapshot. +func publishStageBaseRedMemory(snap map[string][]string) { + if snap == nil { + snap = map[string][]string{} + } + stageBaseRedMemory.Store(&snap) +} + +// stageBaseRedRemembered answers StageLabelPolicy.BaseRedRemembered: the +// checks remembered red on repo's ("owner/name") base branch baseRef. +func stageBaseRedRemembered(repo, baseRef string) []string { + m := stageBaseRedMemory.Load() + if m == nil { + return nil + } + return append([]string(nil), (*m)[github.BaseRedRememberedKey(repo, baseRef)]...) +} + // stageSharedCIIncident answers StageLabelPolicy.SharedCIIncident. repo is // "owner/name"; the snapshot may key a PR by either form. A head pushed // since the snapshot is not covered: the deferral was decided for the old diff --git a/src/cmd/hive/stage_shared_ci_test.go b/src/cmd/hive/stage_shared_ci_test.go index 834d6c3830..bafbf83757 100644 --- a/src/cmd/hive/stage_shared_ci_test.go +++ b/src/cmd/hive/stage_shared_ci_test.go @@ -68,3 +68,21 @@ func TestStageBaseBranchRedIndex(t *testing.T) { t.Fatalf("base-red without an incident reported incident %d", got) } } + +func TestStageBaseRedRemembered(t *testing.T) { + t.Cleanup(func() { stageBaseRedMemory.Store(nil) }) + if got := stageBaseRedRemembered("acme/widget", "main"); got != nil { + t.Fatalf("nothing published: %v", got) + } + publishStageBaseRedMemory(map[string][]string{github.BaseRedRememberedKey("acme/widget", "main"): {"Tests Passed", "Unit Tests"}}) + if got := stageBaseRedRemembered("Acme/Widget", "main"); len(got) != 2 || got[0] != "Tests Passed" || got[1] != "Unit Tests" { + t.Fatalf("remembered = %v", got) + } + if got := stageBaseRedRemembered("acme/widget", "release"); len(got) != 0 { + t.Fatalf("another base: %v", got) + } + publishStageBaseRedMemory(nil) + if got := stageBaseRedRemembered("acme/widget", "main"); len(got) != 0 { + t.Fatalf("after an empty publish: %v", got) + } +} diff --git a/src/docs/labels-and-control-signals/stage-labels.md b/src/docs/labels-and-control-signals/stage-labels.md index 47746e17b5..4e03ebdff7 100644 --- a/src/docs/labels-and-control-signals/stage-labels.md +++ b/src/docs/labels-and-control-signals/stage-labels.md @@ -412,7 +412,7 @@ used to admit the PR: | `contributor-lane:soaking(...)` — and no red check on the head | `095-pre-merge-soak` | | `head-changed-since-eval`, `merge-failed`, `merge-queue-required` (the base branch's ruleset only accepts merges through GitHub's merge queue and the transport is a static token that cannot enqueue, so the direct REST merge is refused; with App auth on a repo opted in through `auto_merge.native_merge_queue` these bases are enqueued instead, see [native-merge-queue.md](../native-merge-queue.md); logged as a skip rather than a merge error, [#14018](https://github.com/hivecommons/hive/issues/14018); the self-authored sweep then skips the direct merge on that base for 30 minutes and logs the skip once per PR head, [#14068](https://github.com/hivecommons/hive/issues/14068); the trusted-author, queued and contributor-lane sweeps skip it the same way, sharing one memory of such bases; while a base is remembered the dashboard shows its otherwise-green PRs as a person's to merge, not Merge-eligible; with `auto_merge.require_current_base` on, a head behind such a base is not updated for 24 hours after the last refusal and holds here instead, so the base moving does not reset its review), `required-checks-expected`, `behind`, `updated-branch`, `behind-update-budget-exhausted` (a behind self-authored PR the sweep did not update only because this tick's branch-update budget was spent; retried next tick, waits on the hive, [#12607](https://github.com/hivecommons/hive/issues/12607)), `release-hold*`, `native-merge-queued: queued (position N, STATE)` (the PR was enqueued into GitHub's native merge queue) and the failures `native-merge-queue-read`, `native-merge-queue-enqueue`, `native-merge-queue-method` — and no red check on the head | `100-merge-queue` | | CI red (required or optional check) while the PR is deferred to an open `[shared-ci]` incident (``, the governor's `deferred_incident`) for the same head | `050-review-pending` / `060-reviewed`, reason `shared-ci-incident:`: the base or a shared check is what is red, so the PR is never sent to 070/071/072, never nudged and never stalled. A 072 nudge already posted for the head (the nudge goes out on entering 072, and the incident marker can land minutes later) is edited in place to say it was withdrawn, and the head's nudge count is reset. Once the incident closes the PR stays deferred until the shared-CI rerun sweep has re-run its failed jobs (at most 30 minutes); a head still red after that is handed back as usual, with a fresh nudge. Conflicts and requested changes stay author-facing ([#12606](https://github.com/hivecommons/hive/issues/12606)) | -| CI red (required or optional) while **every** failing check on the head is also red on the base branch's tip — the governor's `base_red_checks` / `base_red`, set during CI enrichment when a same-named latest check run on the base ref (meta checks excluded) concluded failure / timed_out / cancelled / action_required / startup_failure / stale **and** the PR's changed files include none of `.github/workflows/*`, `Dockerfile` / `Dockerfile.*` / `*.Dockerfile`, `go.mod` / `go.sum` (a PR touching those plausibly owns the check and stays author-facing; a lookup error or a diff beyond the file-list page budget also stays author-facing) | `050-review-pending` / `060-reviewed`, reason `base-branch-red:,` ("waiting for the base branch's CI to recover (…) — not this PR's fault"): never 070/071/072, never nudged (a 072 nudge already posted for the head is edited in place to say it was withdrawn), never stalled, still not merged (the lane's skip reason is the same token). The fix loop counts no attempt, kicks no fixer, and posts one `` hold comment per head; an escalation the hive itself raised (ledger `label_applied`) over such a head has its `needs-human` removed with the hold wording. The next evaluation after the base goes green re-evaluates the head: a still-red PR is handed back as usual. Conflicts and requested changes stay author-facing (seen on [#13270](https://github.com/hivecommons/hive/pull/13270) and [#13258](https://github.com/hivecommons/hive/pull/13258), 2026-10-20) | +| CI red (required or optional) while **every** failing check on the head is also red on the base branch's tip — the governor's `base_red_checks` / `base_red`, set during CI enrichment when a same-named latest check run on the base ref (meta checks excluded) concluded failure / timed_out / cancelled / action_required / startup_failure / stale **and** the PR's changed files include none of `.github/workflows/*`, `Dockerfile` / `Dockerfile.*` / `*.Dockerfile`, `go.mod` / `go.sum` (a PR touching those plausibly owns the check and stays author-facing; a diff beyond the file-list page budget also stays author-facing). The governor remembers each base's red checks across passes: a check stays red on the base until a read sees the tip's run of it complete green, so a tip whose run is still queued or in progress, or a base listing that fails (logged at Warn), keeps the base's last red set; a check not seen red for 2 h is dropped. A new head the CI enrichment has not read yet is classified from that memory at the stage pass: every check red on the head (its check digest) is remembered red on the base and the head touches no CI-owning path (an unreadable file list stays author-facing) | `050-review-pending` / `060-reviewed`, reason `base-branch-red:,` ("waiting for the base branch's CI to recover (…) — not this PR's fault"): never 070/071/072, never nudged (a 072 nudge already posted for the head is edited in place to say it was withdrawn), never stalled, still not merged (the lane's skip reason is the same token). The fix loop counts no attempt, kicks no fixer, and posts one `` hold comment per head; an escalation the hive itself raised (ledger `label_applied`) over such a head has its `needs-human` removed with the hold wording. The next evaluation after the base goes green re-evaluates the head: a still-red PR is handed back as usual. Conflicts and requested changes stay author-facing (seen on [#13270](https://github.com/hivecommons/hive/pull/13270) and [#13258](https://github.com/hivecommons/hive/pull/13258), 2026-10-20) | | CI red (required or optional) on a **fork** PR while **every** failing check's annotation says the check refuses forks (case-insensitive `fork pull request(s) cannot`, `maintainer must push this branch`, or `cannot publish` together with `fork` — a fork-publication / fork-secrets guard) | `085-needs-decision`, reason `fork-guard:,`: never 070/071/072, never nudged, never adopted (074) — neither the author nor the hive can turn the check green by pushing to the fork. The entry comment names the check and says a maintainer must push the branch to the repository (or run or approve it for head `` as the repository requires). A new head on the same fork PR (a merge of the base, or the author's push) keeps the guard while the lane's red check is one of the recorded `fork-guard:` checks, so the PR does not drop to 072 and nobody is nudged before the annotation for that head is read. A fork PR head that nothing has classified yet (its first red head, or a PR that came to 072 for another reason first) is classified from its own check annotations with the same rule, once per head, before it can be nudged: a head red only on fork-guarded checks goes straight to 085 and the guard is recorded for later heads (projectbluefin/utah-packages#972, where the author was nudged and sent relay rework for a fork-gated Canary five minutes before the annotation was read). The hive also does not update the branch of a fork head that waits on a fork-guarded check: the run a maintainer dispatches proves one exact head, and a merge of the base would discard it. Once the check is green the guard drops and updates resume before merge. While the PR stays parked across new heads, the entry comment is rewritten in place for each one, so it always names the current head, and it says that a run which passed on an earlier head does not cover this one even when the patch is the same, because a rebase or a merge of the base changes the code the check runs (projectbluefin/utah-packages#917). A same-repo PR with the same message, or a fork PR with any other red check, is handed back as usual ([#14273](https://github.com/hivecommons/hive/issues/14273), projectbluefin/utah-packages#862 and #869) | | any **non-required** check on the current head is red, whatever the lane's reason (the lane gates on required checks only, so it still soaks and merges; a reader of the rollup sees ✗). Parking modifiers (`hold`, `needs-human`, admission reasons) still win | `070-fixing` / `071-needs-author` / `072-returned-to-contributor` (by author kind), reason `optional-check-failing:` (the nudge lists every red check); never 080 or 100. A red *required* check is the lane's own `required-check-failing:*` → 070/071/072 ([#12538](https://github.com/hivecommons/hive/issues/12538), seen on [#12457](https://github.com/hivecommons/hive/pull/12457)) | | `label:hold`, `label:blocked`, `held`, `sentinel`, `contributor-lane:sentinel-blocked` | **gate**, not a stage ([#17327](https://github.com/hivecommons/hive/issues/17327)): routed by the head's evidence (below) exactly as a trust cause — CI / 070 while checks run or fail, 071 on a reviewer question, `090-ready-for-human-merge` when the head is otherwise clean — with the gate reason kept on the route so the popover and the entry comment say "Held: …". The lanes refuse the merge on the label, never on the stage. `label:blocked` is a PR wearing `blocked`: its gate reason reads `blocked on owner/repo#N` when the blocked sweep has read the pull request it waits on. The lane stops at the gate before it reads mergeability, so the route takes the head's `mergeable_state` from the PR detail the enumeration fetched for the held PR this cycle (no extra request), for exactly the current head: a `dirty` head goes back to its author like any conflict (070 / 071 / 072, reason `conflicting`, the 072 nudge and fix queue included) and the gate stays a pill and a hard merge stop. `unknown`, or a state read for an older head, is not a conflict; a pass with no state at all keeps a conflict the stage already returned on the same head | diff --git a/src/pkg/github/automerge/automerge_base_red_carry.go b/src/pkg/github/automerge/automerge_base_red_carry.go new file mode 100644 index 0000000000..0daa4c0fb5 --- /dev/null +++ b/src/pkg/github/automerge/automerge_base_red_carry.go @@ -0,0 +1,166 @@ +package automerge + +import ( + "context" + "errors" + "slices" + "strings" + "time" + + gh "github.com/google/go-github/v72/github" + hgithub "github.com/hivecommons/hive/pkg/github" +) + +// Base red on a head the enumeration has not classified +// (projectbluefin/chairlift#811, common#1684). +// +// StageLabelPolicy.BaseBranchRed answers only for the head the last +// CI-enriched enumeration saw, so a new head (an author's push, a merge of +// the base) read as plain red CI until the next enumeration: the PR went to +// 072, its author was nudged and a fix queued in their relay, and minutes +// later the PR went back to 050 and the nudge was withdrawn. On chairlift#811 +// that happened on both heads while main stayed red on `Unit Tests`. +// +// The enumeration's client remembers each base's red checks +// (hgithub.Client.BaseRedSnapshot). A head with no answer from the index is +// read the way the index would read it: every check red on the head (its own +// check digest) is remembered red on the base, and the head touches no +// CI-owning path (hgithub.TouchesCIOwningPath). Any other red check, a +// CI-owning path, or a file list that could not be read keeps today's +// author-facing route. + +// baseRedFilesPages bounds the head's file listing (100 files per page), +// like the enumeration's own (hgithub.baseRedMaxFilePages): a longer list is +// treated as possibly owning CI. +const baseRedFilesPages = 3 + +var errBaseRedFilesTruncated = errors.New("file list longer than the base-red bound") + +// baseRedPathsEntry is one head's CI-owning-path answer. +type baseRedPathsEntry struct { + head string + touches bool +} + +// stageBaseRedRemembered asks the policy which checks are remembered red on +// the PR's base branch. +func stageBaseRedRemembered(policy StageLabelPolicy, owner, repo, baseRef string) []string { + if policy.BaseRedRemembered == nil || strings.TrimSpace(baseRef) == "" { + return nil + } + return policy.BaseRedRemembered(owner+"/"+repo, baseRef) +} + +// foldBaseRed marks a red PR head base red when the index has no answer for +// it and every check red on it is remembered red on its base. +func (c *Engine) foldBaseRed(ctx context.Context, tracker *StageTracker, list []stageObservation, now time.Time) { + var o *stageObservation + for i := range list { + if len(list[i].BaseRedChecks) > 0 || len(list[i].ForkGuardChecks) > 0 || list[i].SharedCIIncident > 0 { + return + } + if o == nil && strings.TrimSpace(list[i].HeadSHA) != "" { + o = &list[i] + } + } + if o == nil || o.Merged || strings.TrimSpace(o.BaseRef) == "" { + return + } + head := o.HeadSHA + red := false + for _, l := range list { + if l.HeadSHA == head && (classifyCIReason(l.Reason) == ciReasonRed || l.Detail.RedCheck(head) != "") { + red = true + break + } + } + if !red { + return + } + remembered := stageBaseRedRemembered(c.stagePolicy(), o.Owner, o.Repo, o.BaseRef) + if len(remembered) == 0 { + return + } + key := stageKey(o.Owner, o.Repo, o.Number) + summary, known := c.headChecks(ctx, tracker, key, *o, now) + if !known || len(summary.Red) == 0 { + return + } + for _, name := range summary.Red { + if !slices.Contains(remembered, name) { + return + } + } + touches, known := c.headTouchesCIOwningPath(ctx, tracker, key, *o) + if !known || touches { + return + } + checks := slices.Clone(summary.Red) + slices.Sort(checks) + checks = slices.Compact(checks) + for i := range list { + if list[i].HeadSHA == head { + list[i].BaseRedChecks = append([]string(nil), checks...) + } + } + c.debug("stage labels: base-branch red read for a head the enumeration has not classified", "repo", o.Display, "pr", o.Number, "head", head, "base", o.BaseRef, "checks", checks) +} + +// headTouchesCIOwningPath reports whether obs's head changes a CI-owning +// path, from the duplicate sweep's cached file list or one listing, cached +// per head. known is false when the list could not be read in full; nothing +// is cached then, so the next pass tries again. +func (c *Engine) headTouchesCIOwningPath(ctx context.Context, tracker *StageTracker, key string, obs stageObservation) (touches, known bool) { + tracker.mu.Lock() + entry, ok := tracker.baseRedPaths[key] + tracker.mu.Unlock() + if ok && entry.head == obs.HeadSHA { + return entry.touches, true + } + files := hgithub.CachedPRChangedPaths(obs.Owner+"/"+obs.Repo, obs.Number, obs.HeadSHA) + if len(files) == 0 { + var err error + files, err = c.baseRedHeadFiles(ctx, obs.Owner, obs.Repo, obs.Number) + if err != nil { + c.debug("stage labels: PR file list unavailable; a red head stays author-facing", "repo", obs.Display, "pr", obs.Number, "head", obs.HeadSHA, "error", err) + return false, false + } + } + touches = hgithub.TouchesCIOwningPath(files) + tracker.mu.Lock() + if tracker.baseRedPaths == nil { + tracker.baseRedPaths = map[string]baseRedPathsEntry{} + } + tracker.baseRedPaths[key] = baseRedPathsEntry{head: obs.HeadSHA, touches: touches} + tracker.mu.Unlock() + return touches, true +} + +// baseRedHeadFiles lists the PR's changed paths (renames: both names), or +// errBaseRedFilesTruncated past baseRedFilesPages. +func (c *Engine) baseRedHeadFiles(ctx context.Context, owner, repo string, number int) ([]string, error) { + if c == nil || c.gh == nil { + return nil, hgithub.ErrNoGitHubClient + } + var files []string + opts := &gh.ListOptions{PerPage: 100} + for page := 0; page < baseRedFilesPages; page++ { + list, resp, err := c.gh.PullRequests.ListFiles(ctx, owner, repo, number, opts) + if err != nil { + return nil, err + } + for _, f := range list { + if name := f.GetFilename(); name != "" { + files = append(files, name) + } + if prev := f.GetPreviousFilename(); prev != "" { + files = append(files, prev) + } + } + if resp == nil || resp.NextPage == 0 { + return files, nil + } + opts.Page = resp.NextPage + } + return nil, errBaseRedFilesTruncated +} diff --git a/src/pkg/github/automerge/automerge_base_red_carry_test.go b/src/pkg/github/automerge/automerge_base_red_carry_test.go new file mode 100644 index 0000000000..5cf5bb7e2b --- /dev/null +++ b/src/pkg/github/automerge/automerge_base_red_carry_test.go @@ -0,0 +1,185 @@ +package automerge + +import ( + "context" + "encoding/json" + "net/http" + "strconv" + "strings" + "testing" + "time" + + gh "github.com/google/go-github/v72/github" + hgithub "github.com/hivecommons/hive/pkg/github" + "github.com/hivecommons/hive/pkg/stagelabels" +) + +// projectbluefin/chairlift#811: main was red on `Unit Tests` all afternoon. +// Each new head the enumeration had not read yet (130c2547 at 4:29 PM, +// ead00404 at 4:46 PM) went to 072 with "Failing checks: Unit Tests" and a +// relay fix queued, then back to 050 minutes later. A head whose every red +// check is remembered red on its base now stays at 050 with no nudge; any +// other red check, or a CI-owning path, still goes to the author. +func TestBaseRedCarriesToHeadsTheIndexHasNotRead(t *testing.T) { + api, closeAPI := seededStageAPI(t) + defer closeAPI() + files := map[int][]string{ + 811: {"internal/widget/widget.go"}, + 812: {".github/workflows/test.yml"}, + } + api.extra = func(w http.ResponseWriter, r *http.Request) bool { + path := r.URL.Path + if r.Method != http.MethodGet || !strings.HasPrefix(path, "/repos/acme/widget/pulls/") || !strings.HasSuffix(path, "/files") { + return false + } + n, _ := strconv.Atoi(strings.TrimSuffix(strings.TrimPrefix(path, "/repos/acme/widget/pulls/"), "/files")) + list, ok := files[n] + if !ok { + return false + } + out := []map[string]any{} + for _, f := range list { + out = append(out, map[string]any{"filename": f, "status": "modified"}) + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(out) + return true + } + clock := &contributorLaneClock{at: time.Date(2026, 11, 17, 21, 29, 0, 0, time.UTC)} + sink := &fakeNudgeSink{outcome: ContributorNudgeOutcome{Mode: "queued", Online: true}} + policy := StageLabelPolicy{ + Enabled: true, + ContributorNudge: true, + // The index has read none of these heads. + BaseBranchRed: func(string, int, string) []string { return nil }, + BaseRedRemembered: func(repo, baseRef string) []string { + if repo == "acme/widget" && baseRef == "main" { + return []string{"Race Detection", "Tests Passed", "Unit Tests"} + } + return nil + }, + } + e := newNudgeEngine(t, apiURLOf(t, api), clock, sink, policy) + e.checkSummary = func(_ context.Context, _, _, sha string) (hgithub.CommitChecksSummary, error) { + if sha == "extra" { + return hgithub.CommitChecksSummary{Total: 4, Green: 2, Red: []string{"Unit Tests", "lint"}, DCO: 1}, nil + } + return hgithub.CommitChecksSummary{Total: 4, Green: 3, Red: []string{"Unit Tests"}, DCO: 1}, nil + } + ctx := context.Background() + pass := func(number int, head, base, reason string) stagelabels.Record { + t.Helper() + pr := nudgePR(number, head) + pr.Base = &gh.PullRequestBranch{Ref: gh.Ptr(base)} + e.recordStage(stageLaneContributor, "acme/widget", "acme", "widget", pr, reason, false, nil) + e.noteStageRole("acme", "widget", number, stagelabels.RoleContributor) + if _, err := e.ApplyStageLabels(ctx); err != nil { + t.Fatal(err) + } + rec, _ := e.stageStoreOrShared().Get(stageKey("acme", "widget", number)) + return rec + } + + for _, head := range []string{"130c2547", "ead00404"} { + rec := pass(811, head, "main", "required-check-failing:Unit Tests") + if rec.Stage != stagelabels.ReviewPending || rec.Reason != stageReasonBaseBranchRed+"Unit Tests" { + t.Fatalf("head %s red only on the base's check: record %s %q, want 050 %q", head, rec.Stage, rec.Reason, stageReasonBaseBranchRed+"Unit Tests") + } + if len(sink.nudges) != 0 || len(nudgeComments(api)) != 0 { + t.Fatalf("head %s: author nudged for the base's red: sink=%+v comments=%v", head, sink.nudges, nudgeComments(api)) + } + clock.advance(5 * time.Minute) + } + + // Another red check on the head: the author's, as before. + if rec := pass(811, "extra", "main", "required-check-failing:lint"); rec.Stage != stagelabels.ReturnedToContributor || len(sink.nudges) != 1 { + t.Fatalf("a red check the base does not share: stage %s nudges %d, want 072 and one nudge", rec.Stage, len(sink.nudges)) + } + + // A head that edits a workflow may own the very check that is red. + if rec := pass(812, "wf", "main", "required-check-failing:Unit Tests"); strings.HasPrefix(rec.Reason, stageReasonBaseBranchRed) || rec.Stage == stagelabels.ReviewPending { + t.Fatalf("CI-owning path: record %s %q, want author-facing", rec.Stage, rec.Reason) + } + + // Nothing remembered on this base: today's route. + if rec := pass(813, "rel", "release", "required-check-failing:Unit Tests"); strings.HasPrefix(rec.Reason, stageReasonBaseBranchRed) { + t.Fatalf("no remembered base red: record %s %q, want author-facing", rec.Stage, rec.Reason) + } +} + +func TestHeadTouchesCIOwningPathCachedPerHead(t *testing.T) { + api, closeAPI := seededStageAPI(t) + defer closeAPI() + reads := 0 + api.extra = func(w http.ResponseWriter, r *http.Request) bool { + if r.URL.Path != "/repos/acme/widget/pulls/9/files" { + return false + } + reads++ + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`[{"filename":"README.md"}]`)) + return true + } + clock := &contributorLaneClock{at: time.Date(2026, 11, 17, 21, 0, 0, 0, time.UTC)} + e := newNudgeEngine(t, apiURLOf(t, api), clock, &fakeNudgeSink{}, StageLabelPolicy{Enabled: true}) + tracker := NewStageTracker() + obs := stageObservation{Owner: "acme", Repo: "widget", Number: 9, HeadSHA: "aaa"} + for i := 0; i < 2; i++ { + if touches, known := e.headTouchesCIOwningPath(context.Background(), tracker, stageKey("acme", "widget", 9), obs); touches || !known { + t.Fatalf("README-only head: touches=%v known=%v", touches, known) + } + } + if reads != 1 { + t.Fatalf("file list read %d times for one head, want once", reads) + } + obs.HeadSHA = "bbb" + e.headTouchesCIOwningPath(context.Background(), tracker, stageKey("acme", "widget", 9), obs) + if reads != 2 { + t.Fatalf("a new head must be read again: %d reads", reads) + } +} + +// projectbluefin/common#1684: the red check was optional, so the lane soaked +// and the stage read it from the detail (optional-check-failing:E2E…). A head +// the index had not read yet is folded to base red from the same detail. +func TestFoldBaseRedRedOptionalWhileSoaking(t *testing.T) { + api, closeAPI := seededStageAPI(t) + defer closeAPI() + api.extra = func(w http.ResponseWriter, r *http.Request) bool { + if r.URL.Path != "/repos/acme/widget/pulls/1684/files" { + return false + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`[{"filename":"system_files/shared/usr/bin/ujust"}]`)) + return true + } + clock := &contributorLaneClock{at: time.Date(2026, 11, 17, 18, 34, 0, 0, time.UTC)} + const e2e = "E2E — composed common suite / e2e / GNOME — common" + policy := StageLabelPolicy{Enabled: true, BaseRedRemembered: func(repo, baseRef string) []string { + if repo == "acme/widget" && baseRef == "main" { + return []string{e2e} + } + return nil + }} + e := newNudgeEngine(t, apiURLOf(t, api), clock, &fakeNudgeSink{}, policy) + e.checkSummary = func(context.Context, string, string, string) (hgithub.CommitChecksSummary, error) { + return hgithub.CommitChecksSummary{Total: 9, Green: 8, Red: []string{e2e}, DCO: 1}, nil + } + detail := stageDetail{Verdict: ContributorLaneVerdict{Configured: true, Approved: true, Verdict: "approve"}, VerdictKnown: true} + detail.RedChecks, detail.RedChecksHead = []string{e2e}, "4d94a546a0" + list := []stageObservation{{Lane: stageLaneContributor, Owner: "acme", Repo: "widget", Number: 1684, HeadSHA: "4d94a546a0", BaseRef: "main", Reason: contributorLaneReasonSoaking + "(1h)", Detail: detail}} + e.foldBaseRed(context.Background(), NewStageTracker(), list, clock.now()) + if got := list[0].BaseRedChecks; len(got) != 1 || got[0] != e2e { + t.Fatalf("BaseRedChecks = %v, want [%s]", got, e2e) + } + if v := classifyStage(list[0]); v.Stage != stagelabels.Reviewed || v.Reason != stageReasonBaseBranchRed+e2e { + t.Fatalf("classifyStage = %+v, want 060 %q", v, stageReasonBaseBranchRed+e2e) + } + + // The index already answered for the head: nothing to fold. + answered := []stageObservation{{Lane: stageLaneContributor, Owner: "acme", Repo: "widget", Number: 1684, HeadSHA: "4d94a546a0", BaseRef: "main", Reason: "required-check-failing:build", BaseRedChecks: []string{"build"}}} + e.foldBaseRed(context.Background(), NewStageTracker(), answered, clock.now()) + if got := answered[0].BaseRedChecks; len(got) != 1 || got[0] != "build" { + t.Fatalf("index answer replaced: %v", got) + } +} diff --git a/src/pkg/github/automerge/automerge_stage_labels.go b/src/pkg/github/automerge/automerge_stage_labels.go index 3c5f327d2b..738cbb8cf7 100644 --- a/src/pkg/github/automerge/automerge_stage_labels.go +++ b/src/pkg/github/automerge/automerge_stage_labels.go @@ -169,6 +169,12 @@ type StageLabelPolicy struct { // reason `base-branch-red:` and never nudged: the red is the // base branch's, not the author's. nil = unknown (no deferral). BaseBranchRed func(repo string, number int, head string) []string + // BaseRedRemembered returns the checks the CI enumeration last saw red + // on repo's ("owner/name") base branch baseRef and has not seen green + // since (hgithub.Client.BaseRedSnapshot). A head BaseBranchRed has no + // answer for yet is base red when every check red on it is one of these + // and it touches no CI-owning path (foldBaseRed). Nil means none. + BaseRedRemembered func(repo, baseRef string) []string // ForkGuard returns the failing checks on a fork PR head (repo // "owner/name", number, head SHA) that refuse it only because it is from // a fork, when EVERY failing check does — @@ -291,6 +297,10 @@ type StageTracker struct { // (automerge_human_merge_path_fence.go); created on first use. Unlike // obs it survives drain: it is per head, not per pass. humanMergePathFence map[string]humanMergePathFenceEntry + // baseRedPaths remembers, per key, whether the head touches a CI-owning + // path (foldBaseRed); created on first use. Per head, like + // humanMergePathFence. + baseRedPaths map[string]baseRedPathsEntry } // stageOwner is the lane that set a PR's current stage, for which head, and @@ -1070,6 +1080,7 @@ func (c *Engine) ApplyStageLabels(ctx context.Context) (StageLabelResult, error) c.foldHumanReturnLabel(ctx, store, list) c.foldRecordedDecision(ctx, list) c.foldForkGuard(ctx, tracker, store, list, now) + c.foldBaseRed(ctx, tracker, list, now) verdict, role, obs := resolveStage(list) key := stageKey(obs.Owner, obs.Repo, obs.Number) if verdict.Rank == stageRankNone { diff --git a/src/pkg/github/base_branch_red.go b/src/pkg/github/base_branch_red.go index 9f382ee89e..a85bd3835d 100644 --- a/src/pkg/github/base_branch_red.go +++ b/src/pkg/github/base_branch_red.go @@ -31,9 +31,12 @@ import ( // "waiting on the author", and a hive PR burned three fix attempts against // the red baseline and escalated to needs-human. // -// Error side. Every lookup failure leaves BaseRedChecks empty: the PR is -// then routed exactly as before this rule existed (author-facing), which is -// the conservative direction — a wrongly "base red" PR would sit unrepaired. +// Error side. A failed base listing reuses the base's remembered red set +// (base_branch_red_memory.go), and with nothing remembered leaves +// BaseRedChecks empty: the PR is then routed exactly as before this rule +// existed (author-facing), which is the conservative direction — a wrongly +// "base red" PR would sit unrepaired. A failed or truncated PR file listing +// leaves it empty too. // baseRedMaxFilePages bounds the PR file listing (100 files per page). A PR // larger than this is treated as possibly owning CI (not base red). @@ -101,9 +104,11 @@ func (p PullRequest) AllFailuresBaseRed() bool { return true } -// redCheckNamesForRef returns the names of the latest check runs on ref -// that concluded red (checkConclusionIsRed, meta checks excluded). -func (c *Client) redCheckNamesForRef(ctx context.Context, owner, repo, ref string) (map[string]bool, error) { +// baseCheckStatesForRef returns the state of the latest check run of each +// name on ref (meta checks excluded): red when it concluded red +// (checkConclusionIsRed), green when it completed otherwise, running while +// it is queued or in progress. +func (c *Client) baseCheckStatesForRef(ctx context.Context, owner, repo, ref string) (map[string]baseCheckState, error) { ctx = WithRESTCaller(ctx, "hive:base_branch_red") var all []*gh.CheckRun opts := &gh.ListCheckRunsOptions{ListOptions: gh.ListOptions{PerPage: 100}} @@ -118,17 +123,36 @@ func (c *Client) redCheckNamesForRef(ctx context.Context, owner, repo, ref strin } opts.Page = resp.NextPage } - red := map[string]bool{} + states := map[string]baseCheckState{} for _, cr := range latestCheckRunsByNameAndApp(all) { name := cr.GetName() - if name == "" || isMetaCheck(name) || cr.GetStatus() != "completed" { + if name == "" || isMetaCheck(name) { continue } - if checkConclusionIsRed(cr.GetConclusion()) { - red[name] = true + st := baseCheckGreen + switch { + case cr.GetStatus() != "completed": + st = baseCheckRunning + case checkConclusionIsRed(cr.GetConclusion()): + st = baseCheckRed } + // Two apps may run a check of the same name: red outranks running, + // and running outranks green, so green means every run is green. + if prev, seen := states[name]; !seen || baseCheckRank(st) > baseCheckRank(prev) { + states[name] = st + } + } + return states, nil +} + +func baseCheckRank(st baseCheckState) int { + switch st { + case baseCheckRed: + return 2 + case baseCheckRunning: + return 1 } - return red, nil + return 0 } // prChangedPaths lists the PR's changed file paths, up to @@ -157,9 +181,10 @@ func (c *Client) prChangedPaths(ctx context.Context, owner, repo string, number } // enrichBaseBranchRed stamps BaseRedChecks on every red PR in prs whose -// failing checks are also red on its base branch tip. One check-run -// listing per distinct repo+base per pass; one file listing per red PR -// whose failing checks overlap the base's. +// failing checks are also red on its base branch: on the tip, or +// remembered red and not yet seen green there (baseRedMemory). One +// check-run listing per distinct repo+base per pass; one file listing per +// red PR whose failing checks overlap the base's. func (c *Client) enrichBaseBranchRed(ctx context.Context, prs []PullRequest) { type baseKey struct{ owner, repo, ref string } baseRed := map[baseKey]map[string]bool{} @@ -173,11 +198,13 @@ func (c *Client) enrichBaseBranchRed(ctx context.Context, prs []PullRequest) { key := baseKey{owner, repoName, pr.BaseRef} red, known := baseRed[key] if !known { - var err error - red, err = c.redCheckNamesForRef(ctx, owner, repoName, pr.BaseRef) + memKey := baseRedMemoryKey(owner, repoName, pr.BaseRef) + states, err := c.baseCheckStatesForRef(ctx, owner, repoName, pr.BaseRef) if err != nil { - c.logger.Debug("base branch check runs unavailable; red PRs stay author-facing", "repo", pr.Repo, "base", pr.BaseRef, "error", err) - red = nil + red = c.baseRed.remembered(memKey) + c.logger.Warn("base branch check runs unavailable; using the base's remembered red checks", "repo", pr.Repo, "base", pr.BaseRef, "remembered", len(red), "error", err) + } else { + red = c.baseRed.observe(memKey, states) } baseRed[key] = red } diff --git a/src/pkg/github/base_branch_red_memory.go b/src/pkg/github/base_branch_red_memory.go new file mode 100644 index 0000000000..6c8d27f152 --- /dev/null +++ b/src/pkg/github/base_branch_red_memory.go @@ -0,0 +1,149 @@ +package github + +import ( + "sort" + "strings" + "sync" + "time" +) + +// Base red across passes (projectbluefin/common#1684, chairlift#811). +// +// enrichBaseBranchRed re-reads the base tip on every pass, and a pass that +// missed the base's red sent the PR back to its author: the tip had just +// moved and its run of the check was still queued or in progress, or the +// listing failed. On common#1684 one head moved between 060 and 072 ten +// times in an afternoon while the same E2E check stayed red on main, and +// every 072 queued a fix for the author that the next pass withdrew. +// +// The client therefore remembers, per base, the checks it last saw red. A +// check leaves the set only when a read sees the tip's run of it complete +// with a non-red conclusion; a run still queued or in progress, a tip with +// no run of it yet, or a failed listing keeps it. A check not seen red for +// baseRedMemoryTTL is dropped, so a base nobody reads again cannot pin a +// stale red. +const baseRedMemoryTTL = 2 * time.Hour + +// baseCheckState is one check's latest run on a base tip. +type baseCheckState int + +const ( + // baseCheckRunning: queued or in progress, so it says nothing yet. + baseCheckRunning baseCheckState = iota + baseCheckRed + baseCheckGreen +) + +// baseRedMemory is the per-base red set. The zero value is ready to use. +type baseRedMemory struct { + mu sync.Mutex + // bases maps baseRedMemoryKey → check name → when it was last seen red. + bases map[string]map[string]time.Time + now func() time.Time +} + +func baseRedMemoryKey(owner, repo, ref string) string { + return strings.ToLower(strings.TrimSpace(owner) + "/" + strings.TrimSpace(repo) + "@" + strings.TrimSpace(ref)) +} + +func (m *baseRedMemory) clock() time.Time { + if m.now != nil { + return m.now() + } + return time.Now() +} + +// observe folds one successful read of a base tip into the memory and +// returns the base's red set: the checks red on the tip now, plus the +// remembered ones the tip has not yet shown green. +func (m *baseRedMemory) observe(key string, states map[string]baseCheckState) map[string]bool { + m.mu.Lock() + defer m.mu.Unlock() + now := m.clock() + if m.bases == nil { + m.bases = map[string]map[string]time.Time{} + } + checks := m.bases[key] + if checks == nil { + checks = map[string]time.Time{} + m.bases[key] = checks + } + for name, st := range states { + switch st { + case baseCheckRed: + checks[name] = now + case baseCheckGreen: + delete(checks, name) + } + } + return m.liveLocked(key, now) +} + +// remembered is the base's red set for a pass whose read failed, or nil. +func (m *baseRedMemory) remembered(key string) map[string]bool { + m.mu.Lock() + defer m.mu.Unlock() + return m.liveLocked(key, m.clock()) +} + +// liveLocked drops the base's checks past the TTL and returns the rest. +func (m *baseRedMemory) liveLocked(key string, now time.Time) map[string]bool { + checks := m.bases[key] + var red map[string]bool + for name, at := range checks { + if now.Sub(at) >= baseRedMemoryTTL { + delete(checks, name) + continue + } + if red == nil { + red = map[string]bool{} + } + red[name] = true + } + if len(checks) == 0 { + delete(m.bases, key) + } + return red +} + +// snapshot is every remembered base's red checks, sorted, keyed by +// baseRedMemoryKey. +func (m *baseRedMemory) snapshot() map[string][]string { + m.mu.Lock() + defer m.mu.Unlock() + now := m.clock() + out := map[string][]string{} + for key := range m.bases { + var names []string + for name := range m.liveLocked(key, now) { + names = append(names, name) + } + if len(names) > 0 { + sort.Strings(names) + out[key] = names + } + } + return out +} + +// BaseRedSnapshot returns the checks this client last saw red on each base +// branch tip and has not seen turn green since, keyed by "owner/repo@base" +// (lower case). The stage reconciler uses it to keep a PR head the +// CI-enriched snapshot has not read yet out of the repair stages when its +// only red checks are the base's (BaseRedRememberedKey). +func (c *Client) BaseRedSnapshot() map[string][]string { + if c == nil { + return nil + } + return c.baseRed.snapshot() +} + +// BaseRedRememberedKey is the BaseRedSnapshot key for repo ("owner/name") +// and base ref. +func BaseRedRememberedKey(repo, baseRef string) string { + owner, name, ok := strings.Cut(strings.TrimSpace(repo), "/") + if !ok { + return "" + } + return baseRedMemoryKey(owner, name, baseRef) +} diff --git a/src/pkg/github/base_branch_red_memory_test.go b/src/pkg/github/base_branch_red_memory_test.go new file mode 100644 index 0000000000..bc5c4dfecf --- /dev/null +++ b/src/pkg/github/base_branch_red_memory_test.go @@ -0,0 +1,136 @@ +package github + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "reflect" + "testing" + "time" +) + +func TestBaseRedMemory(t *testing.T) { + now := time.Date(2026, 11, 17, 19, 0, 0, 0, time.UTC) + m := &baseRedMemory{now: func() time.Time { return now }} + key := baseRedMemoryKey("Acme", "Widget", "main") + names := func(red map[string]bool) []string { + var out []string + for _, n := range []string{"E2E", "lint", "unit"} { + if red[n] { + out = append(out, n) + } + } + return out + } + + if got := m.remembered(key); got != nil { + t.Fatalf("nothing read yet: %v, want nil (today's behaviour)", got) + } + if got := names(m.observe(key, map[string]baseCheckState{"E2E": baseCheckRed, "unit": baseCheckGreen})); !reflect.DeepEqual(got, []string{"E2E"}) { + t.Fatalf("first read = %v", got) + } + // The base tip moved: its E2E run is in progress, and lint has no run yet. + now = now.Add(5 * time.Minute) + if got := names(m.observe(key, map[string]baseCheckState{"E2E": baseCheckRunning, "unit": baseCheckGreen})); !reflect.DeepEqual(got, []string{"E2E"}) { + t.Fatalf("tip run in progress: %v, want E2E still red", got) + } + if got := names(m.observe(key, map[string]baseCheckState{})); !reflect.DeepEqual(got, []string{"E2E"}) { + t.Fatalf("tip with no run of E2E yet: %v, want E2E still red", got) + } + // A failed listing reuses the set. + if got := names(m.remembered(key)); !reflect.DeepEqual(got, []string{"E2E"}) { + t.Fatalf("lookup error: %v, want the remembered E2E", got) + } + if got := m.snapshot(); !reflect.DeepEqual(got, map[string][]string{"acme/widget@main": {"E2E"}}) { + t.Fatalf("snapshot = %v", got) + } + if got := BaseRedRememberedKey("acme/widget", "main"); got != key { + t.Fatalf("BaseRedRememberedKey = %q, want %q", got, key) + } + // Only a completed non-red run of the check drops it. + if got := m.observe(key, map[string]baseCheckState{"E2E": baseCheckGreen}); got != nil { + t.Fatalf("tip completed green: %v, want nothing red", got) + } + if got := m.remembered(key); got != nil { + t.Fatalf("after green: %v, want nil", got) + } + + // The TTL bounds a check nobody sees red again. + m.observe(key, map[string]baseCheckState{"lint": baseCheckRed}) + now = now.Add(baseRedMemoryTTL - time.Minute) + if got := names(m.remembered(key)); !reflect.DeepEqual(got, []string{"lint"}) { + t.Fatalf("inside the TTL: %v", got) + } + now = now.Add(time.Minute) + if got := m.remembered(key); got != nil { + t.Fatalf("TTL expired: %v, want nil", got) + } + if got := m.snapshot(); len(got) != 0 { + t.Fatalf("snapshot after expiry = %v, want empty", got) + } +} + +// projectbluefin/common#1684: main's E2E check stayed red all afternoon, +// but a pass that read the tip while its run was in progress, or could not +// list the runs, called the PR's E2E red its own. The base's red now holds +// until the tip's run of the check completes green. +func TestEnrichBaseBranchRedRemembersAcrossPasses(t *testing.T) { + baseRuns := []string{ + `{"total_count":2,"check_runs":[{"id":1,"name":"E2E","status":"completed","conclusion":"failure"},{"id":2,"name":"unit","status":"completed","conclusion":"success"}]}`, + `{"total_count":2,"check_runs":[{"id":3,"name":"E2E","status":"in_progress"},{"id":4,"name":"unit","status":"completed","conclusion":"success"}]}`, + "", // the listing fails + `{"total_count":2,"check_runs":[{"id":5,"name":"E2E","status":"completed","conclusion":"success"},{"id":6,"name":"unit","status":"completed","conclusion":"success"}]}`, + } + pass := 0 + mux := http.NewServeMux() + mux.HandleFunc("/repos/acme/widget/commits/main/check-runs", func(w http.ResponseWriter, r *http.Request) { + body := baseRuns[pass] + if body == "" { + http.NotFound(w, r) + return + } + _, _ = w.Write([]byte(body)) + }) + mux.HandleFunc("/repos/acme/widget/pulls/1684/files", func(w http.ResponseWriter, r *http.Request) { + _ = json.NewEncoder(w).Encode([]map[string]any{{"filename": "system_files/shared/usr/bin/ujust"}}) + }) + srv := httptest.NewServer(mux) + defer srv.Close() + c := newTestClient(t, srv, "acme", []string{"widget"}) + + for i, want := range [][]string{{"E2E"}, {"E2E"}, {"E2E"}, nil} { + pass = i + prs := []PullRequest{{Repo: "acme/widget", Number: 1684, BaseRef: "main", CIStatus: "failure", FailingChecks: []string{"E2E"}}} + c.enrichBaseBranchRed(context.Background(), prs) + if !reflect.DeepEqual(prs[0].BaseRedChecks, want) { + t.Fatalf("pass %d: BaseRedChecks = %v, want %v", i, prs[0].BaseRedChecks, want) + } + } + if got := c.BaseRedSnapshot(); len(got) != 0 { + t.Fatalf("snapshot after the base went green = %v, want empty", got) + } +} + +func TestBaseCheckStatesForRefPrecedence(t *testing.T) { + mux := http.NewServeMux() + mux.HandleFunc("/repos/acme/widget/commits/main/check-runs", func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(`{"total_count":5,"check_runs":[ + {"id":1,"name":"E2E","status":"completed","conclusion":"success","app":{"id":1}}, + {"id":2,"name":"E2E","status":"in_progress","app":{"id":2}}, + {"id":3,"name":"unit","status":"completed","conclusion":"failure","app":{"id":1}}, + {"id":4,"name":"unit","status":"in_progress","app":{"id":2}}, + {"id":5,"name":"lint","status":"completed","conclusion":"success"}]}`)) + }) + srv := httptest.NewServer(mux) + defer srv.Close() + c := newTestClient(t, srv, "acme", []string{"widget"}) + got, err := c.baseCheckStatesForRef(context.Background(), "acme", "widget", "main") + if err != nil { + t.Fatal(err) + } + want := map[string]baseCheckState{"E2E": baseCheckRunning, "unit": baseCheckRed, "lint": baseCheckGreen} + if !reflect.DeepEqual(got, want) { + t.Fatalf("states = %v, want %v (red > running > green across apps)", got, want) + } +} diff --git a/src/pkg/github/client.go b/src/pkg/github/client.go index 2439875b38..d89c35e965 100644 --- a/src/pkg/github/client.go +++ b/src/pkg/github/client.go @@ -54,6 +54,9 @@ type Client struct { // (kubestellar/hive#5733). Per-client because the artifact it corrects is a // property of THIS client's token minting. Zero value is ready to use. rateLimits rateLimitTracker + // baseRed remembers each base branch's red checks across passes + // (base_branch_red_memory.go). Zero value is ready to use. + baseRed baseRedMemory // lastGoodEnum keeps each repo's newest successful enumeration so a // transiently refused fetch does not publish the repo as empty // (enumerate_lastgood.go). Zero value is ready to use. diff --git a/src/pkg/github/red_main.go b/src/pkg/github/red_main.go index 14497a5277..379400a29c 100644 --- a/src/pkg/github/red_main.go +++ b/src/pkg/github/red_main.go @@ -220,7 +220,7 @@ func (c *Client) SweepRedMain(ctx context.Context, now time.Time) (RedMainSweepR } // redCheckRunsForRef returns the latest red check run per name on ref -// (meta checks excluded) — redCheckNamesForRef with the runs kept — and the +// (meta checks excluded) — baseCheckStatesForRef's red runs, kept — and the // names whose latest runs have all completed (red or not). A name missing // from settled has a run still queued or in progress. func (c *Client) redCheckRunsForRef(ctx context.Context, owner, repo, ref string) (map[string]*gh.CheckRun, map[string]bool, error) { From e80a99a48034dd40f7145f5eec89e764e17bd02a Mon Sep 17 00:00:00 2001 From: Danathar <6772335+Danathar@users.noreply.github.com> Date: Tue, 17 Nov 2026 19:13:01 -0500 Subject: [PATCH 2/2] docs: re-cite RankActionableIssues for this branch's client.go lines This branch adds three lines to client.go above RankActionableIssues, so after the v6 merge it starts at 5600, not 5597; scale-envelope.md now cites 5600-5610 and the docs citation check finds AgeMinutes again. Signed-off-by: Danathar <6772335+Danathar@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BQYnPGFJx81c8JXR3o1sg1 --- src/docs/scale-envelope.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/docs/scale-envelope.md b/src/docs/scale-envelope.md index 1f9da47b4e..c0ba535a7c 100644 --- a/src/docs/scale-envelope.md +++ b/src/docs/scale-envelope.md @@ -34,7 +34,7 @@ ones to: 2. **Truncation must preserve the prioritization signal, not cut an arbitrary slice.** This holds today and is easy to assume backwards: - Issues are sorted by actionable rank tier, then **oldest first** within each tier before any cap is applied - (`RankActionableIssues`, `src/pkg/github/client.go:5593-5603`, descending `AgeMinutes` within each tier). + (`RankActionableIssues`, `src/pkg/github/client.go:5600-5610`, descending `AgeMinutes` within each tier). - PRs are sorted by **review class then oldest-first within class** (`SortPullRequestsForReview`, `src/pkg/github/review_priority.go:334`; fixes → refactors/docs → tests).