Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/fixed-adopted-fix-ci-route-once.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- Adoption: an adopted red PR is routed to the fixer once per red episode (its head plus its failing checks) instead of being re-announced on every eligibility pass, and a PR the hive already parked at `stage/085` (`adoption-stalled:<reason>`) on its current head leaves `ci-failing.json`, so the scanner is no longer kicked to re-triage a red it cannot fix until the head moves or the escalation is released
88 changes: 88 additions & 0 deletions src/cmd/hive/adopted_fix_ci_routes.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
package main

import (
"slices"
"strings"
"sync"

"github.com/hivecommons/hive/pkg/stagelabels"
)

// adoptedFixCIRouteLedger remembers, per adopted PR, the red episode the
// fixer was last routed for (or the stage-085 park it was last held at), so
// writeMergeEligible announces each one once instead of on every eligibility
// pass. An unchanged head with the same failing checks is the same episode:
// on the projectbluefin spoke two adopted printer-app PRs whose FSDK builds
// were red on a fsdk-containers bug were "routed to fixer" 61 times each at
// one head. A new head, a different set of failing checks, or a park that is
// released starts a new episode and is announced again.
//
// It is in memory: after a restart each open episode is announced once more.
type adoptedFixCIRouteLedger struct {
mu sync.Mutex
last map[string]string
}

// adoptedFixCIRoutes is the process-wide ledger; tests swap it.
var adoptedFixCIRoutes = &adoptedFixCIRouteLedger{}

// first records episode for repo#number and reports whether it differs from
// the one recorded before (true the first time a PR is seen).
func (l *adoptedFixCIRouteLedger) first(repo string, number int, episode string) bool {
key := stagelabels.Key(repo, number)
l.mu.Lock()
defer l.mu.Unlock()
if l.last == nil {
l.last = map[string]string{}
}
if l.last[key] == episode {
return false
}
l.last[key] = episode
return true
}

// retain forgets every PR whose key is not in open, so a closed or merged
// PR's episode does not linger.
func (l *adoptedFixCIRouteLedger) retain(open map[string]bool) {
l.mu.Lock()
defer l.mu.Unlock()
for key := range l.last {
if !open[key] {
delete(l.last, key)
}
}
}

// fixCIEpisode names a red episode: the head plus its failing-check set,
// order-insensitive.
func fixCIEpisode(head string, failing []string) string {
checks := make([]string, 0, len(failing))
for _, c := range failing {
if c = strings.TrimSpace(c); c != "" {
checks = append(checks, c)
}
}
slices.Sort(checks)
checks = slices.Compact(checks)
return "fix-ci\x00" + head + "\x00" + strings.Join(checks, "\x00")
}

// parkedEpisode names a stage-085 park of the adoption at head for reason.
func parkedEpisode(head, reason string) string {
return "parked\x00" + head + "\x00" + reason
}

// adoptedFixCIParked reports the reason the hive parked an adopted PR's
// current head at stage 085 (stagelabels.Record.AdoptionEscalated): the
// 074 dwell ran out with nothing the hive could push, and a person was told
// so. Until the head moves or the escalation is released (the blocker
// cleared, a recovery attempt, an operator's intent) the PR is not fixer
// work, so it is not routed and not listed in ci-failing.json.
func adoptedFixCIParked(repo string, number int, head string) (string, bool) {
rec, ok := stagelabels.Shared().Get(stagelabels.Key(repo, number))
if !ok || !rec.AdoptionEscalated(head) {
return "", false
}
return rec.AdoptEscalatedReason, true
}
122 changes: 122 additions & 0 deletions src/cmd/hive/adopted_fix_ci_routes_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,122 @@
package main

import (
"bytes"
"log/slog"
"path/filepath"
"strings"
"testing"

"github.com/hivecommons/hive/pkg/github"
"github.com/hivecommons/hive/pkg/stagelabels"
)

// An adopted red PR whose head does not move is the same red episode on every
// eligibility pass. projectbluefin/hplip-printer-app#199 and
// ghostscript-printer-app#190 (FSDK builds red on an fsdk-containers bug,
// heads unchanged for a day) were "routed to fixer" 61 times each, and stayed
// in the scanner's FIX-BEFORE-NEW list after the hive had parked them at 085
// saying it had nothing to push.

// useMemoryFixCIRoutes gives the test its own route ledger.
func useMemoryFixCIRoutes(t *testing.T) {
t.Helper()
prev := adoptedFixCIRoutes
adoptedFixCIRoutes = &adoptedFixCIRouteLedger{}
t.Cleanup(func() { adoptedFixCIRoutes = prev })
}

// sweepFixCILogged runs one writeMergeEligible pass and returns the
// ci-failing rows and the log text.
func sweepFixCILogged(t *testing.T, prs []github.PullRequest) ([]failingEntry, string) {
t.Helper()
dir := t.TempDir()
origMerge, origFail := mergeEligiblePath, ciFailingPath
mergeEligiblePath = filepath.Join(dir, "merge-eligible.json")
ciFailingPath = filepath.Join(dir, "ci-failing.json")
defer func() { mergeEligiblePath, ciFailingPath = origMerge, origFail }()
var logs bytes.Buffer
actionable := &github.ActionableResult{PRs: github.PRResult{Items: prs}}
writeMergeEligible(actionable, github.HoldResult{}, "projectbluefin", nil, false, nil, false, nil, nil, nil, slog.New(slog.NewTextHandler(&logs, nil)))
var payload struct {
CIFailing []failingEntry `json:"ci_failing"`
}
readJSON(t, ciFailingPath, &payload)
return payload.CIFailing, logs.String()
}

func adoptedRedPrinterPR(head string, checks ...string) github.PullRequest {
return github.PullRequest{Number: 199, Repo: "hplip-printer-app", Author: "Karibhou", HeadSHA: head,
HeadRef: "slim-printing", HeadRepo: "Karibhou/hplip-printer-app", FromFork: true, MaintainerCanModify: true,
Mergeable: github.MergeableNo, MergeableState: "blocked", CIStatus: "failure",
FailingChecks: checks,
Labels: []string{"sentinel-alert", "stage/074-adopted-by-hive", "author/clanker"}}
}

const fixCIRouted = "adopted PR routed to fixer"

func TestWriteMergeEligible_AdoptedFixCIRoutedOncePerHeadAndCheckSet(t *testing.T) {
useMemoryFixCIRoutes(t)
const repo = "projectbluefin/hplip-printer-app"
adoptInPlace(t, repo, 199, "2b9a441")
pr := adoptedRedPrinterPR("2b9a441", "FSDK image and socket-sink (x86_64)", "FSDK image and socket-sink (aarch64)")

failing, logs := sweepFixCILogged(t, []github.PullRequest{pr})
if len(failing) != 1 || strings.Count(logs, fixCIRouted) != 1 {
t.Fatalf("first sweep: rows=%+v routed=%d, want the red PR listed and routed once", failing, strings.Count(logs, fixCIRouted))
}
// Same head, same failing checks (in another order): still listed for
// the fixer's kick, but not routed again.
pr.FailingChecks = []string{"FSDK image and socket-sink (aarch64)", "FSDK image and socket-sink (x86_64)"}
failing, logs = sweepFixCILogged(t, []github.PullRequest{pr})
if len(failing) != 1 || strings.Count(logs, fixCIRouted) != 0 {
t.Fatalf("second sweep: rows=%+v routed=%d, want listed but not re-routed", failing, strings.Count(logs, fixCIRouted))
}
// A newly failing check on the same head is a new episode.
pr.FailingChecks = append(pr.FailingChecks, "lint")
if _, logs = sweepFixCILogged(t, []github.PullRequest{pr}); strings.Count(logs, fixCIRouted) != 1 {
t.Fatalf("new failing check: routed=%d, want once", strings.Count(logs, fixCIRouted))
}
// A new head routes once, then goes quiet again.
adoptInPlace(t, repo, 199, "3c0b552")
pr.HeadSHA = "3c0b552"
if _, logs = sweepFixCILogged(t, []github.PullRequest{pr}); strings.Count(logs, fixCIRouted) != 1 {
t.Fatalf("new head: routed=%d, want once", strings.Count(logs, fixCIRouted))
}
if _, logs = sweepFixCILogged(t, []github.PullRequest{pr}); strings.Count(logs, fixCIRouted) != 0 {
t.Fatalf("new head re-sweep: routed=%d, want none", strings.Count(logs, fixCIRouted))
}
}

func TestWriteMergeEligible_AdoptedPRParkedAt085IsNotFixCIWork(t *testing.T) {
useMemoryFixCIRoutes(t)
const repo = "projectbluefin/hplip-printer-app"
const reason = "adoption-stalled:check-failing:FSDK image and socket-sink (aarch64)"
adoptInPlace(t, repo, 199, "2b9a441")
key := stagelabels.Key(repo, 199)
stagelabels.Shared().MarkAdoptionEscalated(key, "2b9a441", reason)
pr := adoptedRedPrinterPR("2b9a441", "FSDK image and socket-sink (x86_64)", "FSDK image and socket-sink (aarch64)")

for sweep := 1; sweep <= 2; sweep++ {
failing, logs := sweepFixCILogged(t, []github.PullRequest{pr})
if len(failing) != 0 || strings.Contains(logs, fixCIRouted) {
t.Fatalf("sweep %d: rows=%+v routed=%v, want a PR parked at 085 off the fixer list", sweep, failing, strings.Contains(logs, fixCIRouted))
}
if got, want := strings.Count(logs, "fix-ci not routed"), map[int]int{1: 1, 2: 0}[sweep]; got != want {
t.Fatalf("sweep %d: park lines=%d, want %d", sweep, got, want)
}
}
// The escalation is released (blocker cleared, a recovery attempt, an
// operator's intent): the fixer gets the head again.
stagelabels.Shared().ClearAdoptionEscalation(key)
if failing, logs := sweepFixCILogged(t, []github.PullRequest{pr}); len(failing) != 1 || strings.Count(logs, fixCIRouted) != 1 {
t.Fatalf("released: rows=%+v routed=%d, want listed and routed once", failing, strings.Count(logs, fixCIRouted))
}
// A new head is not parked by the old head's escalation.
stagelabels.Shared().MarkAdoptionEscalated(key, "2b9a441", reason)
adoptInPlace(t, repo, 199, "3c0b552")
pr.HeadSHA = "3c0b552"
if failing, logs := sweepFixCILogged(t, []github.PullRequest{pr}); len(failing) != 1 || strings.Count(logs, fixCIRouted) != 1 {
t.Fatalf("new head: rows=%+v routed=%d, want listed and routed once", failing, strings.Count(logs, fixCIRouted))
}
}
25 changes: 24 additions & 1 deletion src/cmd/hive/merge_eligibility.go
Original file line number Diff line number Diff line change
Expand Up @@ -1331,6 +1331,7 @@ func writeMergeEligible(actionable *github.ActionableResult, hold github.HoldRes
requiredChecks: requiredChecks,
}
seen := make(map[string]bool, len(candidates))
openPRs := make(map[string]bool, len(candidates))
for _, cand := range candidates {
pr := cand.pr
key := fmt.Sprintf("%s/%d", pr.Repo, pr.Number)
Expand All @@ -1342,6 +1343,7 @@ func writeMergeEligible(actionable *github.ActionableResult, hold github.HoldRes
// the hold snapshot; both mean the same thing here.
held := cand.held || holdSet[key]
fullRepo := fullRepoName(pr.Repo, org)
openPRs[stagelabels.Key(fullRepo, pr.Number)] = true
prGates := gates
if cfg != nil {
if set, ok := cfg.AutoMerge.RequiredCheckSetForRepo(fullRepo); ok {
Expand Down Expand Up @@ -1507,6 +1509,17 @@ func writeMergeEligible(actionable *github.ActionableResult, hold github.HoldRes
failing = append(failing, row)
continue
case mergeBucketFailing:
if adopted {
// Parked at 085 on this head: the hive already told a person
// it has nothing to push, so the fixer is not kicked to
// re-triage the same red on every sweep.
if reason, parked := adoptedFixCIParked(fullRepo, pr.Number, pr.HeadSHA); parked {
if adoptedFixCIRoutes.first(fullRepo, pr.Number, parkedEpisode(pr.HeadSHA, reason)) {
logger.Info("adopted PR parked at needs-decision; fix-ci not routed", "repo", fullRepo, "number", pr.Number, "head", pr.HeadSHA, "reason", reason, "failing_checks", pr.FailingChecks)
}
continue
}
}
agent := prFixAgent(pr, prAgents[fmt.Sprintf("%s#%d", fullRepo, pr.Number)])
reroutedFrom := ""
if adopted {
Expand All @@ -1519,8 +1532,13 @@ func writeMergeEligible(actionable *github.ActionableResult, hold github.HoldRes
row.DeferredIncident = pr.SharedCIIncident
row.DeferredExternal = pr.ExternalCIDeferred
if adopted {
// The row stays listed (and queued for the 074 dwell) every
// sweep, but each red episode, its head plus failing checks,
// is routed and announced once.
markAdoptedFixerQueued(fullRepo, pr.Number, pr.HeadSHA, agent, cfg)
logger.Info("adopted PR routed to fixer", "repo", fullRepo, "number", pr.Number, "work", "fix-ci", "failing_checks", pr.FailingChecks, "fixer", agent, "sentinel_alert", row.SentinelAlert)
if adoptedFixCIRoutes.first(fullRepo, pr.Number, fixCIEpisode(pr.HeadSHA, pr.FailingChecks)) {
logger.Info("adopted PR routed to fixer", "repo", fullRepo, "number", pr.Number, "work", "fix-ci", "head", pr.HeadSHA, "failing_checks", pr.FailingChecks, "fixer", agent, "sentinel_alert", row.SentinelAlert)
}
}
failing = append(failing, row)
continue
Expand All @@ -1542,6 +1560,11 @@ func writeMergeEligible(actionable *github.ActionableResult, hold github.HoldRes
})
}

if len(openPRs) > 0 {
// An empty pass is more likely a failed read than every PR closing.
adoptedFixCIRoutes.retain(openPRs)
}

_ = os.MkdirAll("/var/run/hive-metrics", 0o755)

payload := map[string]any{
Expand Down
8 changes: 7 additions & 1 deletion src/docs/labels-and-control-signals/stage-labels.md
Original file line number Diff line number Diff line change
Expand Up @@ -1059,7 +1059,13 @@ When a contributor-agent PR goes red the hive does not just relabel it (a human-
`ci-failing.json` as its own (`"adopted": true`, `reachable_action:
push`, conflicted rows included — ordinarily only hive-authored
conflict PRs are listed) so the scanner's FIX-BEFORE-NEW kick rebases
and fixes CI **on the PR's branch**. No superseding PR, no `hive/adopt-<n>`
and fixes CI **on the PR's branch**. A red row is logged as
`adopted PR routed to fixer` once per red episode (its head plus its
set of failing checks), not on every sweep. Once the adoption parks at
085 `adoption-stalled:<reason>` on that head, the row leaves
`ci-failing.json` (one `fix-ci not routed` line names the reason)
until the head moves or the escalation is released.
No superseding PR, no `hive/adopt-<n>`
branch — with one exception, the DCO re-home described below, which
converts the in-place adoption into a supersession. The hive's own
pushes keep the adoption and the stage stays
Expand Down
Loading