diff --git a/changelog.d/fixed-adopted-fix-ci-route-once.md b/changelog.d/fixed-adopted-fix-ci-route-once.md new file mode 100644 index 0000000000..4f829a09d1 --- /dev/null +++ b/changelog.d/fixed-adopted-fix-ci-route-once.md @@ -0,0 +1 @@ +- Adoption: an adopted red PR is routed to the fixer once per red episode (its head plus its failing checks) instead of being re-announced on every eligibility pass, and a PR the hive already parked at `stage/085` (`adoption-stalled:`) on its current head leaves `ci-failing.json`, so the scanner is no longer kicked to re-triage a red it cannot fix until the head moves or the escalation is released diff --git a/src/cmd/hive/adopted_fix_ci_routes.go b/src/cmd/hive/adopted_fix_ci_routes.go new file mode 100644 index 0000000000..306f7580bb --- /dev/null +++ b/src/cmd/hive/adopted_fix_ci_routes.go @@ -0,0 +1,88 @@ +package main + +import ( + "slices" + "strings" + "sync" + + "github.com/hivecommons/hive/pkg/stagelabels" +) + +// adoptedFixCIRouteLedger remembers, per adopted PR, the red episode the +// fixer was last routed for (or the stage-085 park it was last held at), so +// writeMergeEligible announces each one once instead of on every eligibility +// pass. An unchanged head with the same failing checks is the same episode: +// on the projectbluefin spoke two adopted printer-app PRs whose FSDK builds +// were red on a fsdk-containers bug were "routed to fixer" 61 times each at +// one head. A new head, a different set of failing checks, or a park that is +// released starts a new episode and is announced again. +// +// It is in memory: after a restart each open episode is announced once more. +type adoptedFixCIRouteLedger struct { + mu sync.Mutex + last map[string]string +} + +// adoptedFixCIRoutes is the process-wide ledger; tests swap it. +var adoptedFixCIRoutes = &adoptedFixCIRouteLedger{} + +// first records episode for repo#number and reports whether it differs from +// the one recorded before (true the first time a PR is seen). +func (l *adoptedFixCIRouteLedger) first(repo string, number int, episode string) bool { + key := stagelabels.Key(repo, number) + l.mu.Lock() + defer l.mu.Unlock() + if l.last == nil { + l.last = map[string]string{} + } + if l.last[key] == episode { + return false + } + l.last[key] = episode + return true +} + +// retain forgets every PR whose key is not in open, so a closed or merged +// PR's episode does not linger. +func (l *adoptedFixCIRouteLedger) retain(open map[string]bool) { + l.mu.Lock() + defer l.mu.Unlock() + for key := range l.last { + if !open[key] { + delete(l.last, key) + } + } +} + +// fixCIEpisode names a red episode: the head plus its failing-check set, +// order-insensitive. +func fixCIEpisode(head string, failing []string) string { + checks := make([]string, 0, len(failing)) + for _, c := range failing { + if c = strings.TrimSpace(c); c != "" { + checks = append(checks, c) + } + } + slices.Sort(checks) + checks = slices.Compact(checks) + return "fix-ci\x00" + head + "\x00" + strings.Join(checks, "\x00") +} + +// parkedEpisode names a stage-085 park of the adoption at head for reason. +func parkedEpisode(head, reason string) string { + return "parked\x00" + head + "\x00" + reason +} + +// adoptedFixCIParked reports the reason the hive parked an adopted PR's +// current head at stage 085 (stagelabels.Record.AdoptionEscalated): the +// 074 dwell ran out with nothing the hive could push, and a person was told +// so. Until the head moves or the escalation is released (the blocker +// cleared, a recovery attempt, an operator's intent) the PR is not fixer +// work, so it is not routed and not listed in ci-failing.json. +func adoptedFixCIParked(repo string, number int, head string) (string, bool) { + rec, ok := stagelabels.Shared().Get(stagelabels.Key(repo, number)) + if !ok || !rec.AdoptionEscalated(head) { + return "", false + } + return rec.AdoptEscalatedReason, true +} diff --git a/src/cmd/hive/adopted_fix_ci_routes_test.go b/src/cmd/hive/adopted_fix_ci_routes_test.go new file mode 100644 index 0000000000..6ee9050260 --- /dev/null +++ b/src/cmd/hive/adopted_fix_ci_routes_test.go @@ -0,0 +1,122 @@ +package main + +import ( + "bytes" + "log/slog" + "path/filepath" + "strings" + "testing" + + "github.com/hivecommons/hive/pkg/github" + "github.com/hivecommons/hive/pkg/stagelabels" +) + +// An adopted red PR whose head does not move is the same red episode on every +// eligibility pass. projectbluefin/hplip-printer-app#199 and +// ghostscript-printer-app#190 (FSDK builds red on an fsdk-containers bug, +// heads unchanged for a day) were "routed to fixer" 61 times each, and stayed +// in the scanner's FIX-BEFORE-NEW list after the hive had parked them at 085 +// saying it had nothing to push. + +// useMemoryFixCIRoutes gives the test its own route ledger. +func useMemoryFixCIRoutes(t *testing.T) { + t.Helper() + prev := adoptedFixCIRoutes + adoptedFixCIRoutes = &adoptedFixCIRouteLedger{} + t.Cleanup(func() { adoptedFixCIRoutes = prev }) +} + +// sweepFixCILogged runs one writeMergeEligible pass and returns the +// ci-failing rows and the log text. +func sweepFixCILogged(t *testing.T, prs []github.PullRequest) ([]failingEntry, string) { + t.Helper() + dir := t.TempDir() + origMerge, origFail := mergeEligiblePath, ciFailingPath + mergeEligiblePath = filepath.Join(dir, "merge-eligible.json") + ciFailingPath = filepath.Join(dir, "ci-failing.json") + defer func() { mergeEligiblePath, ciFailingPath = origMerge, origFail }() + var logs bytes.Buffer + actionable := &github.ActionableResult{PRs: github.PRResult{Items: prs}} + writeMergeEligible(actionable, github.HoldResult{}, "projectbluefin", nil, false, nil, false, nil, nil, nil, slog.New(slog.NewTextHandler(&logs, nil))) + var payload struct { + CIFailing []failingEntry `json:"ci_failing"` + } + readJSON(t, ciFailingPath, &payload) + return payload.CIFailing, logs.String() +} + +func adoptedRedPrinterPR(head string, checks ...string) github.PullRequest { + return github.PullRequest{Number: 199, Repo: "hplip-printer-app", Author: "Karibhou", HeadSHA: head, + HeadRef: "slim-printing", HeadRepo: "Karibhou/hplip-printer-app", FromFork: true, MaintainerCanModify: true, + Mergeable: github.MergeableNo, MergeableState: "blocked", CIStatus: "failure", + FailingChecks: checks, + Labels: []string{"sentinel-alert", "stage/074-adopted-by-hive", "author/clanker"}} +} + +const fixCIRouted = "adopted PR routed to fixer" + +func TestWriteMergeEligible_AdoptedFixCIRoutedOncePerHeadAndCheckSet(t *testing.T) { + useMemoryFixCIRoutes(t) + const repo = "projectbluefin/hplip-printer-app" + adoptInPlace(t, repo, 199, "2b9a441") + pr := adoptedRedPrinterPR("2b9a441", "FSDK image and socket-sink (x86_64)", "FSDK image and socket-sink (aarch64)") + + failing, logs := sweepFixCILogged(t, []github.PullRequest{pr}) + if len(failing) != 1 || strings.Count(logs, fixCIRouted) != 1 { + t.Fatalf("first sweep: rows=%+v routed=%d, want the red PR listed and routed once", failing, strings.Count(logs, fixCIRouted)) + } + // Same head, same failing checks (in another order): still listed for + // the fixer's kick, but not routed again. + pr.FailingChecks = []string{"FSDK image and socket-sink (aarch64)", "FSDK image and socket-sink (x86_64)"} + failing, logs = sweepFixCILogged(t, []github.PullRequest{pr}) + if len(failing) != 1 || strings.Count(logs, fixCIRouted) != 0 { + t.Fatalf("second sweep: rows=%+v routed=%d, want listed but not re-routed", failing, strings.Count(logs, fixCIRouted)) + } + // A newly failing check on the same head is a new episode. + pr.FailingChecks = append(pr.FailingChecks, "lint") + if _, logs = sweepFixCILogged(t, []github.PullRequest{pr}); strings.Count(logs, fixCIRouted) != 1 { + t.Fatalf("new failing check: routed=%d, want once", strings.Count(logs, fixCIRouted)) + } + // A new head routes once, then goes quiet again. + adoptInPlace(t, repo, 199, "3c0b552") + pr.HeadSHA = "3c0b552" + if _, logs = sweepFixCILogged(t, []github.PullRequest{pr}); strings.Count(logs, fixCIRouted) != 1 { + t.Fatalf("new head: routed=%d, want once", strings.Count(logs, fixCIRouted)) + } + if _, logs = sweepFixCILogged(t, []github.PullRequest{pr}); strings.Count(logs, fixCIRouted) != 0 { + t.Fatalf("new head re-sweep: routed=%d, want none", strings.Count(logs, fixCIRouted)) + } +} + +func TestWriteMergeEligible_AdoptedPRParkedAt085IsNotFixCIWork(t *testing.T) { + useMemoryFixCIRoutes(t) + const repo = "projectbluefin/hplip-printer-app" + const reason = "adoption-stalled:check-failing:FSDK image and socket-sink (aarch64)" + adoptInPlace(t, repo, 199, "2b9a441") + key := stagelabels.Key(repo, 199) + stagelabels.Shared().MarkAdoptionEscalated(key, "2b9a441", reason) + pr := adoptedRedPrinterPR("2b9a441", "FSDK image and socket-sink (x86_64)", "FSDK image and socket-sink (aarch64)") + + for sweep := 1; sweep <= 2; sweep++ { + failing, logs := sweepFixCILogged(t, []github.PullRequest{pr}) + if len(failing) != 0 || strings.Contains(logs, fixCIRouted) { + t.Fatalf("sweep %d: rows=%+v routed=%v, want a PR parked at 085 off the fixer list", sweep, failing, strings.Contains(logs, fixCIRouted)) + } + if got, want := strings.Count(logs, "fix-ci not routed"), map[int]int{1: 1, 2: 0}[sweep]; got != want { + t.Fatalf("sweep %d: park lines=%d, want %d", sweep, got, want) + } + } + // The escalation is released (blocker cleared, a recovery attempt, an + // operator's intent): the fixer gets the head again. + stagelabels.Shared().ClearAdoptionEscalation(key) + if failing, logs := sweepFixCILogged(t, []github.PullRequest{pr}); len(failing) != 1 || strings.Count(logs, fixCIRouted) != 1 { + t.Fatalf("released: rows=%+v routed=%d, want listed and routed once", failing, strings.Count(logs, fixCIRouted)) + } + // A new head is not parked by the old head's escalation. + stagelabels.Shared().MarkAdoptionEscalated(key, "2b9a441", reason) + adoptInPlace(t, repo, 199, "3c0b552") + pr.HeadSHA = "3c0b552" + if failing, logs := sweepFixCILogged(t, []github.PullRequest{pr}); len(failing) != 1 || strings.Count(logs, fixCIRouted) != 1 { + t.Fatalf("new head: rows=%+v routed=%d, want listed and routed once", failing, strings.Count(logs, fixCIRouted)) + } +} diff --git a/src/cmd/hive/merge_eligibility.go b/src/cmd/hive/merge_eligibility.go index b27a7e55cf..3ccd75ac69 100644 --- a/src/cmd/hive/merge_eligibility.go +++ b/src/cmd/hive/merge_eligibility.go @@ -1331,6 +1331,7 @@ func writeMergeEligible(actionable *github.ActionableResult, hold github.HoldRes requiredChecks: requiredChecks, } seen := make(map[string]bool, len(candidates)) + openPRs := make(map[string]bool, len(candidates)) for _, cand := range candidates { pr := cand.pr key := fmt.Sprintf("%s/%d", pr.Repo, pr.Number) @@ -1342,6 +1343,7 @@ func writeMergeEligible(actionable *github.ActionableResult, hold github.HoldRes // the hold snapshot; both mean the same thing here. held := cand.held || holdSet[key] fullRepo := fullRepoName(pr.Repo, org) + openPRs[stagelabels.Key(fullRepo, pr.Number)] = true prGates := gates if cfg != nil { if set, ok := cfg.AutoMerge.RequiredCheckSetForRepo(fullRepo); ok { @@ -1507,6 +1509,17 @@ func writeMergeEligible(actionable *github.ActionableResult, hold github.HoldRes failing = append(failing, row) continue case mergeBucketFailing: + if adopted { + // Parked at 085 on this head: the hive already told a person + // it has nothing to push, so the fixer is not kicked to + // re-triage the same red on every sweep. + if reason, parked := adoptedFixCIParked(fullRepo, pr.Number, pr.HeadSHA); parked { + if adoptedFixCIRoutes.first(fullRepo, pr.Number, parkedEpisode(pr.HeadSHA, reason)) { + logger.Info("adopted PR parked at needs-decision; fix-ci not routed", "repo", fullRepo, "number", pr.Number, "head", pr.HeadSHA, "reason", reason, "failing_checks", pr.FailingChecks) + } + continue + } + } agent := prFixAgent(pr, prAgents[fmt.Sprintf("%s#%d", fullRepo, pr.Number)]) reroutedFrom := "" if adopted { @@ -1519,8 +1532,13 @@ func writeMergeEligible(actionable *github.ActionableResult, hold github.HoldRes row.DeferredIncident = pr.SharedCIIncident row.DeferredExternal = pr.ExternalCIDeferred if adopted { + // The row stays listed (and queued for the 074 dwell) every + // sweep, but each red episode, its head plus failing checks, + // is routed and announced once. markAdoptedFixerQueued(fullRepo, pr.Number, pr.HeadSHA, agent, cfg) - logger.Info("adopted PR routed to fixer", "repo", fullRepo, "number", pr.Number, "work", "fix-ci", "failing_checks", pr.FailingChecks, "fixer", agent, "sentinel_alert", row.SentinelAlert) + if adoptedFixCIRoutes.first(fullRepo, pr.Number, fixCIEpisode(pr.HeadSHA, pr.FailingChecks)) { + logger.Info("adopted PR routed to fixer", "repo", fullRepo, "number", pr.Number, "work", "fix-ci", "head", pr.HeadSHA, "failing_checks", pr.FailingChecks, "fixer", agent, "sentinel_alert", row.SentinelAlert) + } } failing = append(failing, row) continue @@ -1542,6 +1560,11 @@ func writeMergeEligible(actionable *github.ActionableResult, hold github.HoldRes }) } + if len(openPRs) > 0 { + // An empty pass is more likely a failed read than every PR closing. + adoptedFixCIRoutes.retain(openPRs) + } + _ = os.MkdirAll("/var/run/hive-metrics", 0o755) payload := map[string]any{ diff --git a/src/docs/labels-and-control-signals/stage-labels.md b/src/docs/labels-and-control-signals/stage-labels.md index 71cb59f359..abdcca3788 100644 --- a/src/docs/labels-and-control-signals/stage-labels.md +++ b/src/docs/labels-and-control-signals/stage-labels.md @@ -1059,7 +1059,13 @@ When a contributor-agent PR goes red the hive does not just relabel it (a human- `ci-failing.json` as its own (`"adopted": true`, `reachable_action: push`, conflicted rows included — ordinarily only hive-authored conflict PRs are listed) so the scanner's FIX-BEFORE-NEW kick rebases - and fixes CI **on the PR's branch**. No superseding PR, no `hive/adopt-` + and fixes CI **on the PR's branch**. A red row is logged as + `adopted PR routed to fixer` once per red episode (its head plus its + set of failing checks), not on every sweep. Once the adoption parks at + 085 `adoption-stalled:` on that head, the row leaves + `ci-failing.json` (one `fix-ci not routed` line names the reason) + until the head moves or the escalation is released. + No superseding PR, no `hive/adopt-` branch — with one exception, the DCO re-home described below, which converts the in-place adoption into a supersession. The hive's own pushes keep the adoption and the stage stays