From 35140140552044b187114dec0bfaaa4b00a0b5b1 Mon Sep 17 00:00:00 2001 From: Danathar Date: Sat, 7 Nov 2026 14:27:04 -0500 Subject: [PATCH 1/2] fix(adoption): route an adopted red PR to the fixer once per red episode An adopted contributor PR red at an unchanged head was logged "adopted PR routed to fixer" on every eligibility pass (61 times each for projectbluefin/hplip-printer-app#199 and ghostscript-printer-app#190 in eight hours) and stayed in the scanner's FIX-BEFORE-NEW list even after the hive parked it at stage/085 with nothing to push, so the scanner re-triaged an upstream fsdk-containers bug it had already diagnosed. - Route and announce fix-ci once per (head, failing-check set); the row stays listed and queued for the 074 dwell between announcements. - An adopted PR whose current head is parked at 085 (AdoptionEscalated) leaves ci-failing.json until the head moves or the escalation is released, with one log line naming the reason. - The sentinel alert alone still does not block repair (#13280). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BQYnPGFJx81c8JXR3o1sg1 Signed-off-by: Danathar --- .../fixed-adopted-fix-ci-route-once.md | 1 + src/cmd/hive/adopted_fix_ci_routes.go | 88 +++++++++++++ src/cmd/hive/adopted_fix_ci_routes_test.go | 122 ++++++++++++++++++ src/cmd/hive/merge_eligibility.go | 25 +++- .../stage-labels.md | 8 +- 5 files changed, 242 insertions(+), 2 deletions(-) create mode 100644 src/changelog.d/fixed-adopted-fix-ci-route-once.md create mode 100644 src/cmd/hive/adopted_fix_ci_routes.go create mode 100644 src/cmd/hive/adopted_fix_ci_routes_test.go diff --git a/src/changelog.d/fixed-adopted-fix-ci-route-once.md b/src/changelog.d/fixed-adopted-fix-ci-route-once.md new file mode 100644 index 0000000000..4f829a09d1 --- /dev/null +++ b/src/changelog.d/fixed-adopted-fix-ci-route-once.md @@ -0,0 +1 @@ +- Adoption: an adopted red PR is routed to the fixer once per red episode (its head plus its failing checks) instead of being re-announced on every eligibility pass, and a PR the hive already parked at `stage/085` (`adoption-stalled:`) on its current head leaves `ci-failing.json`, so the scanner is no longer kicked to re-triage a red it cannot fix until the head moves or the escalation is released diff --git a/src/cmd/hive/adopted_fix_ci_routes.go b/src/cmd/hive/adopted_fix_ci_routes.go new file mode 100644 index 0000000000..306f7580bb --- /dev/null +++ b/src/cmd/hive/adopted_fix_ci_routes.go @@ -0,0 +1,88 @@ +package main + +import ( + "slices" + "strings" + "sync" + + "github.com/hivecommons/hive/pkg/stagelabels" +) + +// adoptedFixCIRouteLedger remembers, per adopted PR, the red episode the +// fixer was last routed for (or the stage-085 park it was last held at), so +// writeMergeEligible announces each one once instead of on every eligibility +// pass. An unchanged head with the same failing checks is the same episode: +// on the projectbluefin spoke two adopted printer-app PRs whose FSDK builds +// were red on a fsdk-containers bug were "routed to fixer" 61 times each at +// one head. A new head, a different set of failing checks, or a park that is +// released starts a new episode and is announced again. +// +// It is in memory: after a restart each open episode is announced once more. +type adoptedFixCIRouteLedger struct { + mu sync.Mutex + last map[string]string +} + +// adoptedFixCIRoutes is the process-wide ledger; tests swap it. +var adoptedFixCIRoutes = &adoptedFixCIRouteLedger{} + +// first records episode for repo#number and reports whether it differs from +// the one recorded before (true the first time a PR is seen). +func (l *adoptedFixCIRouteLedger) first(repo string, number int, episode string) bool { + key := stagelabels.Key(repo, number) + l.mu.Lock() + defer l.mu.Unlock() + if l.last == nil { + l.last = map[string]string{} + } + if l.last[key] == episode { + return false + } + l.last[key] = episode + return true +} + +// retain forgets every PR whose key is not in open, so a closed or merged +// PR's episode does not linger. +func (l *adoptedFixCIRouteLedger) retain(open map[string]bool) { + l.mu.Lock() + defer l.mu.Unlock() + for key := range l.last { + if !open[key] { + delete(l.last, key) + } + } +} + +// fixCIEpisode names a red episode: the head plus its failing-check set, +// order-insensitive. +func fixCIEpisode(head string, failing []string) string { + checks := make([]string, 0, len(failing)) + for _, c := range failing { + if c = strings.TrimSpace(c); c != "" { + checks = append(checks, c) + } + } + slices.Sort(checks) + checks = slices.Compact(checks) + return "fix-ci\x00" + head + "\x00" + strings.Join(checks, "\x00") +} + +// parkedEpisode names a stage-085 park of the adoption at head for reason. +func parkedEpisode(head, reason string) string { + return "parked\x00" + head + "\x00" + reason +} + +// adoptedFixCIParked reports the reason the hive parked an adopted PR's +// current head at stage 085 (stagelabels.Record.AdoptionEscalated): the +// 074 dwell ran out with nothing the hive could push, and a person was told +// so. Until the head moves or the escalation is released (the blocker +// cleared, a recovery attempt, an operator's intent) the PR is not fixer +// work, so it is not routed and not listed in ci-failing.json. +func adoptedFixCIParked(repo string, number int, head string) (string, bool) { + rec, ok := stagelabels.Shared().Get(stagelabels.Key(repo, number)) + if !ok || !rec.AdoptionEscalated(head) { + return "", false + } + return rec.AdoptEscalatedReason, true +} diff --git a/src/cmd/hive/adopted_fix_ci_routes_test.go b/src/cmd/hive/adopted_fix_ci_routes_test.go new file mode 100644 index 0000000000..6ee9050260 --- /dev/null +++ b/src/cmd/hive/adopted_fix_ci_routes_test.go @@ -0,0 +1,122 @@ +package main + +import ( + "bytes" + "log/slog" + "path/filepath" + "strings" + "testing" + + "github.com/hivecommons/hive/pkg/github" + "github.com/hivecommons/hive/pkg/stagelabels" +) + +// An adopted red PR whose head does not move is the same red episode on every +// eligibility pass. projectbluefin/hplip-printer-app#199 and +// ghostscript-printer-app#190 (FSDK builds red on an fsdk-containers bug, +// heads unchanged for a day) were "routed to fixer" 61 times each, and stayed +// in the scanner's FIX-BEFORE-NEW list after the hive had parked them at 085 +// saying it had nothing to push. + +// useMemoryFixCIRoutes gives the test its own route ledger. +func useMemoryFixCIRoutes(t *testing.T) { + t.Helper() + prev := adoptedFixCIRoutes + adoptedFixCIRoutes = &adoptedFixCIRouteLedger{} + t.Cleanup(func() { adoptedFixCIRoutes = prev }) +} + +// sweepFixCILogged runs one writeMergeEligible pass and returns the +// ci-failing rows and the log text. +func sweepFixCILogged(t *testing.T, prs []github.PullRequest) ([]failingEntry, string) { + t.Helper() + dir := t.TempDir() + origMerge, origFail := mergeEligiblePath, ciFailingPath + mergeEligiblePath = filepath.Join(dir, "merge-eligible.json") + ciFailingPath = filepath.Join(dir, "ci-failing.json") + defer func() { mergeEligiblePath, ciFailingPath = origMerge, origFail }() + var logs bytes.Buffer + actionable := &github.ActionableResult{PRs: github.PRResult{Items: prs}} + writeMergeEligible(actionable, github.HoldResult{}, "projectbluefin", nil, false, nil, false, nil, nil, nil, slog.New(slog.NewTextHandler(&logs, nil))) + var payload struct { + CIFailing []failingEntry `json:"ci_failing"` + } + readJSON(t, ciFailingPath, &payload) + return payload.CIFailing, logs.String() +} + +func adoptedRedPrinterPR(head string, checks ...string) github.PullRequest { + return github.PullRequest{Number: 199, Repo: "hplip-printer-app", Author: "Karibhou", HeadSHA: head, + HeadRef: "slim-printing", HeadRepo: "Karibhou/hplip-printer-app", FromFork: true, MaintainerCanModify: true, + Mergeable: github.MergeableNo, MergeableState: "blocked", CIStatus: "failure", + FailingChecks: checks, + Labels: []string{"sentinel-alert", "stage/074-adopted-by-hive", "author/clanker"}} +} + +const fixCIRouted = "adopted PR routed to fixer" + +func TestWriteMergeEligible_AdoptedFixCIRoutedOncePerHeadAndCheckSet(t *testing.T) { + useMemoryFixCIRoutes(t) + const repo = "projectbluefin/hplip-printer-app" + adoptInPlace(t, repo, 199, "2b9a441") + pr := adoptedRedPrinterPR("2b9a441", "FSDK image and socket-sink (x86_64)", "FSDK image and socket-sink (aarch64)") + + failing, logs := sweepFixCILogged(t, []github.PullRequest{pr}) + if len(failing) != 1 || strings.Count(logs, fixCIRouted) != 1 { + t.Fatalf("first sweep: rows=%+v routed=%d, want the red PR listed and routed once", failing, strings.Count(logs, fixCIRouted)) + } + // Same head, same failing checks (in another order): still listed for + // the fixer's kick, but not routed again. + pr.FailingChecks = []string{"FSDK image and socket-sink (aarch64)", "FSDK image and socket-sink (x86_64)"} + failing, logs = sweepFixCILogged(t, []github.PullRequest{pr}) + if len(failing) != 1 || strings.Count(logs, fixCIRouted) != 0 { + t.Fatalf("second sweep: rows=%+v routed=%d, want listed but not re-routed", failing, strings.Count(logs, fixCIRouted)) + } + // A newly failing check on the same head is a new episode. + pr.FailingChecks = append(pr.FailingChecks, "lint") + if _, logs = sweepFixCILogged(t, []github.PullRequest{pr}); strings.Count(logs, fixCIRouted) != 1 { + t.Fatalf("new failing check: routed=%d, want once", strings.Count(logs, fixCIRouted)) + } + // A new head routes once, then goes quiet again. + adoptInPlace(t, repo, 199, "3c0b552") + pr.HeadSHA = "3c0b552" + if _, logs = sweepFixCILogged(t, []github.PullRequest{pr}); strings.Count(logs, fixCIRouted) != 1 { + t.Fatalf("new head: routed=%d, want once", strings.Count(logs, fixCIRouted)) + } + if _, logs = sweepFixCILogged(t, []github.PullRequest{pr}); strings.Count(logs, fixCIRouted) != 0 { + t.Fatalf("new head re-sweep: routed=%d, want none", strings.Count(logs, fixCIRouted)) + } +} + +func TestWriteMergeEligible_AdoptedPRParkedAt085IsNotFixCIWork(t *testing.T) { + useMemoryFixCIRoutes(t) + const repo = "projectbluefin/hplip-printer-app" + const reason = "adoption-stalled:check-failing:FSDK image and socket-sink (aarch64)" + adoptInPlace(t, repo, 199, "2b9a441") + key := stagelabels.Key(repo, 199) + stagelabels.Shared().MarkAdoptionEscalated(key, "2b9a441", reason) + pr := adoptedRedPrinterPR("2b9a441", "FSDK image and socket-sink (x86_64)", "FSDK image and socket-sink (aarch64)") + + for sweep := 1; sweep <= 2; sweep++ { + failing, logs := sweepFixCILogged(t, []github.PullRequest{pr}) + if len(failing) != 0 || strings.Contains(logs, fixCIRouted) { + t.Fatalf("sweep %d: rows=%+v routed=%v, want a PR parked at 085 off the fixer list", sweep, failing, strings.Contains(logs, fixCIRouted)) + } + if got, want := strings.Count(logs, "fix-ci not routed"), map[int]int{1: 1, 2: 0}[sweep]; got != want { + t.Fatalf("sweep %d: park lines=%d, want %d", sweep, got, want) + } + } + // The escalation is released (blocker cleared, a recovery attempt, an + // operator's intent): the fixer gets the head again. + stagelabels.Shared().ClearAdoptionEscalation(key) + if failing, logs := sweepFixCILogged(t, []github.PullRequest{pr}); len(failing) != 1 || strings.Count(logs, fixCIRouted) != 1 { + t.Fatalf("released: rows=%+v routed=%d, want listed and routed once", failing, strings.Count(logs, fixCIRouted)) + } + // A new head is not parked by the old head's escalation. + stagelabels.Shared().MarkAdoptionEscalated(key, "2b9a441", reason) + adoptInPlace(t, repo, 199, "3c0b552") + pr.HeadSHA = "3c0b552" + if failing, logs := sweepFixCILogged(t, []github.PullRequest{pr}); len(failing) != 1 || strings.Count(logs, fixCIRouted) != 1 { + t.Fatalf("new head: rows=%+v routed=%d, want listed and routed once", failing, strings.Count(logs, fixCIRouted)) + } +} diff --git a/src/cmd/hive/merge_eligibility.go b/src/cmd/hive/merge_eligibility.go index b27a7e55cf..3ccd75ac69 100644 --- a/src/cmd/hive/merge_eligibility.go +++ b/src/cmd/hive/merge_eligibility.go @@ -1331,6 +1331,7 @@ func writeMergeEligible(actionable *github.ActionableResult, hold github.HoldRes requiredChecks: requiredChecks, } seen := make(map[string]bool, len(candidates)) + openPRs := make(map[string]bool, len(candidates)) for _, cand := range candidates { pr := cand.pr key := fmt.Sprintf("%s/%d", pr.Repo, pr.Number) @@ -1342,6 +1343,7 @@ func writeMergeEligible(actionable *github.ActionableResult, hold github.HoldRes // the hold snapshot; both mean the same thing here. held := cand.held || holdSet[key] fullRepo := fullRepoName(pr.Repo, org) + openPRs[stagelabels.Key(fullRepo, pr.Number)] = true prGates := gates if cfg != nil { if set, ok := cfg.AutoMerge.RequiredCheckSetForRepo(fullRepo); ok { @@ -1507,6 +1509,17 @@ func writeMergeEligible(actionable *github.ActionableResult, hold github.HoldRes failing = append(failing, row) continue case mergeBucketFailing: + if adopted { + // Parked at 085 on this head: the hive already told a person + // it has nothing to push, so the fixer is not kicked to + // re-triage the same red on every sweep. + if reason, parked := adoptedFixCIParked(fullRepo, pr.Number, pr.HeadSHA); parked { + if adoptedFixCIRoutes.first(fullRepo, pr.Number, parkedEpisode(pr.HeadSHA, reason)) { + logger.Info("adopted PR parked at needs-decision; fix-ci not routed", "repo", fullRepo, "number", pr.Number, "head", pr.HeadSHA, "reason", reason, "failing_checks", pr.FailingChecks) + } + continue + } + } agent := prFixAgent(pr, prAgents[fmt.Sprintf("%s#%d", fullRepo, pr.Number)]) reroutedFrom := "" if adopted { @@ -1519,8 +1532,13 @@ func writeMergeEligible(actionable *github.ActionableResult, hold github.HoldRes row.DeferredIncident = pr.SharedCIIncident row.DeferredExternal = pr.ExternalCIDeferred if adopted { + // The row stays listed (and queued for the 074 dwell) every + // sweep, but each red episode, its head plus failing checks, + // is routed and announced once. markAdoptedFixerQueued(fullRepo, pr.Number, pr.HeadSHA, agent, cfg) - logger.Info("adopted PR routed to fixer", "repo", fullRepo, "number", pr.Number, "work", "fix-ci", "failing_checks", pr.FailingChecks, "fixer", agent, "sentinel_alert", row.SentinelAlert) + if adoptedFixCIRoutes.first(fullRepo, pr.Number, fixCIEpisode(pr.HeadSHA, pr.FailingChecks)) { + logger.Info("adopted PR routed to fixer", "repo", fullRepo, "number", pr.Number, "work", "fix-ci", "head", pr.HeadSHA, "failing_checks", pr.FailingChecks, "fixer", agent, "sentinel_alert", row.SentinelAlert) + } } failing = append(failing, row) continue @@ -1542,6 +1560,11 @@ func writeMergeEligible(actionable *github.ActionableResult, hold github.HoldRes }) } + if len(openPRs) > 0 { + // An empty pass is more likely a failed read than every PR closing. + adoptedFixCIRoutes.retain(openPRs) + } + _ = os.MkdirAll("/var/run/hive-metrics", 0o755) payload := map[string]any{ diff --git a/src/docs/labels-and-control-signals/stage-labels.md b/src/docs/labels-and-control-signals/stage-labels.md index 71cb59f359..abdcca3788 100644 --- a/src/docs/labels-and-control-signals/stage-labels.md +++ b/src/docs/labels-and-control-signals/stage-labels.md @@ -1059,7 +1059,13 @@ When a contributor-agent PR goes red the hive does not just relabel it (a human- `ci-failing.json` as its own (`"adopted": true`, `reachable_action: push`, conflicted rows included — ordinarily only hive-authored conflict PRs are listed) so the scanner's FIX-BEFORE-NEW kick rebases - and fixes CI **on the PR's branch**. No superseding PR, no `hive/adopt-` + and fixes CI **on the PR's branch**. A red row is logged as + `adopted PR routed to fixer` once per red episode (its head plus its + set of failing checks), not on every sweep. Once the adoption parks at + 085 `adoption-stalled:` on that head, the row leaves + `ci-failing.json` (one `fix-ci not routed` line names the reason) + until the head moves or the escalation is released. + No superseding PR, no `hive/adopt-` branch — with one exception, the DCO re-home described below, which converts the in-place adoption into a supersession. The hive's own pushes keep the adoption and the stage stays From 844f657eb052e07583f25de480d53f057c1029c2 Mon Sep 17 00:00:00 2001 From: Danathar Date: Sat, 7 Nov 2026 14:49:50 -0500 Subject: [PATCH 2/2] changelog: move the fragment to the repo-root changelog.d/ changelog-fragment-guard failed because the fragment sat under src/changelog.d/, which is never compiled into CHANGELOG.md. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BQYnPGFJx81c8JXR3o1sg1 Signed-off-by: Danathar --- .../fixed-adopted-fix-ci-route-once.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename {src/changelog.d => changelog.d}/fixed-adopted-fix-ci-route-once.md (100%) diff --git a/src/changelog.d/fixed-adopted-fix-ci-route-once.md b/changelog.d/fixed-adopted-fix-ci-route-once.md similarity index 100% rename from src/changelog.d/fixed-adopted-fix-ci-route-once.md rename to changelog.d/fixed-adopted-fix-ci-route-once.md