diff --git a/changelog.d/added-stale-red-base-refresh.md b/changelog.d/added-stale-red-base-refresh.md new file mode 100644 index 0000000000..56025b0057 --- /dev/null +++ b/changelog.d/added-stale-red-base-refresh.md @@ -0,0 +1 @@ +- A red pull request that is behind its base, and whose every failing check has since gone green on the base tip, now gets the base merged in by the hive (the same merge "Update branch" makes), so its CI runs again against the fix instead of waiting for a person. It covers PRs the automerge adoption recovery does not, including repositories with auto-merge off; it skips drafts, held PRs, forks without maintainer edits, dependency-bot branches and PRs at stage 072 or 074, makes at most 2 updates per repository and 6 per pass every 5 minutes, tries each head and base tip once, and goes through the new `base_refresh` write-back kind (hive-authored PRs, else L6 with the opt-in label; `writeback.kinds.base_refresh.unlabeled_min_level: 6` reaches every PR). Seen on projectbluefin/actions#544, which sat red on `pytest` for six hours after main repaired the baseline. diff --git a/src/cmd/hive/main_eval_cycle.go b/src/cmd/hive/main_eval_cycle.go index 7e1276c957..5866eea2c2 100644 --- a/src/cmd/hive/main_eval_cycle.go +++ b/src/cmd/hive/main_eval_cycle.go @@ -816,6 +816,19 @@ func runEvalCycle( } reran := rerunTransientTriagedPRs(ctx, cfg, ghClient, actionable, escalatedPRs, audit, logger) reapSkip = reaperSkipAfterRerun(escalatedPRs, reran) + // A red PR whose failing checks are green on its base tip needs only + // the base merged in (projectbluefin/actions#544); the reaper does + // not also kick a fixer for a head the merge replaces. + if !budgetDecision.SkipUnstick { + var refreshAudit func(string, int, string) + if dashSrv != nil { + refreshAudit = func(repo string, number int, detail string) { + dashSrv.AuditLogRecord("system", "stale-red-base-refresh", detail, "", repo, number) + } + } + refreshed := refreshStaleRedBehindPRs(ctx, cfg, ghClient, actionable, refreshAudit, logger, time.Now()) + reapSkip = reaperSkipAfterRerun(reapSkip, refreshed) + } } reapStuckRedPRs(cfg, actionable, reapSkip, agentKicker{mgr: agentMgr}, agentAvailability(cfg, agentMgr), logger) diff --git a/src/cmd/hive/stale_red_base_refresh.go b/src/cmd/hive/stale_red_base_refresh.go new file mode 100644 index 0000000000..b17c2631a4 --- /dev/null +++ b/src/cmd/hive/stale_red_base_refresh.go @@ -0,0 +1,201 @@ +package main + +import ( + "context" + "fmt" + "log/slog" + "strings" + "sync" + "time" + + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/escalation" + "github.com/hivecommons/hive/pkg/github" + "github.com/hivecommons/hive/pkg/stagelabels" +) + +const ( + // staleRedRefreshInterval spaces the passes: each one reads the tip and + // check runs of every base branch a red PR targets. + staleRedRefreshInterval = 5 * time.Minute + // staleRedRefreshMaxPerRepo and staleRedRefreshMaxPerPass cap the + // branch updates one pass makes. A fixed base can leave a dozen PRs red + // at once; the rest are picked up on later passes. + staleRedRefreshMaxPerRepo = 2 + staleRedRefreshMaxPerPass = 6 + // staleRedRefreshMemoCap bounds the settled (PR, head, base tip) memo. + staleRedRefreshMemoCap = 4096 +) + +// staleRedRefresher is the GitHub side of the pass; *github.Client +// implements it. +type staleRedRefresher interface { + StaleRedBaseStateFor(ctx context.Context, repo, baseRef string) (github.StaleRedBaseState, bool, error) + RefreshStaleRedPR(ctx context.Context, pr github.PullRequest, base github.StaleRedBaseState) (github.StaleRedRefreshOutcome, error) +} + +// staleRedRefreshState is the pass clock and the memo of (PR → head|base +// tip) pairs already settled, so a PR is updated at most once per base tip. +// It is in memory: after a restart a settled pair costs one compare call +// (and GitHub answers an already-current branch without a new commit). +type staleRedRefreshState struct { + mu sync.Mutex + lastRun time.Time + settled map[string]string +} + +var staleRedRefresh = &staleRedRefreshState{} + +// refreshStaleRedBehindPRs merges the base into open PRs that are red only +// because of something their base has since fixed: the head is behind the +// base, and every check that failed on it completed green on the current +// base tip (projectbluefin/actions#544: pytest red from a baseline main fixed +// in #734; the PR waited six hours for a person to press "Update branch"). +// The automerge lanes already do this for PRs the hive adopted; this pass +// covers every other PR in the actionable set, including repositories with +// auto-merge off. +// +// It leaves alone a PR that is a draft, held or exempt, still running CI, +// deferred to an open shared-CI incident, red on the base as well, a fork +// with "allow edits by maintainers" off, a dependency bot's branch (the bot +// rebases it), returned to its contributor (072, whose hive pushes there) +// or adopted by the hive (074, whose recovery owns it). Each update goes +// through the write-back gate (base_refresh). It runs at most every +// staleRedRefreshInterval and makes at most staleRedRefreshMaxPerRepo +// updates per repository and staleRedRefreshMaxPerPass in all. +// +// It returns the escalation keys of the PRs it updated, so the reaper in the +// same cycle does not also kick a fixer for a head that is about to be +// replaced. +func refreshStaleRedBehindPRs( + ctx context.Context, + cfg *config.Config, + refresher staleRedRefresher, + actionable *github.ActionableResult, + audit func(repo string, number int, detail string), + logger *slog.Logger, + now time.Time, +) map[string]bool { + if cfg == nil || refresher == nil || actionable == nil { + return nil + } + st := staleRedRefresh + st.mu.Lock() + defer st.mu.Unlock() + if !st.lastRun.IsZero() && now.Sub(st.lastRun) < staleRedRefreshInterval { + return nil + } + st.lastRun = now + if st.settled == nil || len(st.settled) > staleRedRefreshMemoCap { + st.settled = map[string]string{} + } + + type baseKey struct{ repo, ref string } + type baseRead struct { + state github.StaleRedBaseState + ok bool + } + bases := map[baseKey]baseRead{} + perRepo := map[string]int{} + total := 0 + var updated map[string]bool + for _, pr := range actionable.PRs.Items { + if total >= staleRedRefreshMaxPerPass { + break + } + if staleRedRefreshSkip(pr) != "" { + continue + } + repo := fullRepoName(pr.Repo, cfg.Project.Org) + if perRepo[repo] >= staleRedRefreshMaxPerRepo { + continue + } + bk := baseKey{repo, pr.BaseRef} + base, read := bases[bk] + if !read { + state, exists, err := refresher.StaleRedBaseStateFor(ctx, repo, pr.BaseRef) + if err != nil && logger != nil { + logger.Warn("stale-red base refresh: could not read the base tip; retrying next pass", "repo", repo, "base", pr.BaseRef, "error", err) + } + base = baseRead{state: state, ok: err == nil && exists} + bases[bk] = base + } + if !base.ok { + continue + } + memoKey := fmt.Sprintf("%s#%d", strings.ToLower(repo), pr.Number) + pair := pr.HeadSHA + "|" + base.state.SHA + if st.settled[memoKey] == pair { + continue + } + if verdict, _ := github.StaleRedBaseVerdictFor(pr.FailingChecks, base.state); verdict != github.StaleRedBaseClears { + // Nothing is spent on a PR the base does not clear, and the + // base tip's checks may still complete: read again next pass. + continue + } + pr.Repo = repo + outcome, err := refresher.RefreshStaleRedPR(ctx, pr, base.state) + if err != nil { + if logger != nil { + logger.Warn("stale-red base refresh: update failed; retrying next pass", "repo", repo, "pr", pr.Number, "head", pr.HeadSHA, "base_sha", base.state.SHA, "error", err) + } + continue + } + if outcome.Settled { + st.settled[memoKey] = pair + } + if !outcome.Updated { + if logger != nil && outcome.Skipped != "current" { + logger.Info("stale-red base refresh skipped", "repo", repo, "pr", pr.Number, "head", pr.HeadSHA, "base_sha", base.state.SHA, "reason", outcome.Skipped) + } + continue + } + perRepo[repo]++ + total++ + if logger != nil { + logger.Info("stale-red base refresh: merged the base into a red PR whose failing checks are green on the base tip", + "repo", repo, "pr", pr.Number, "head", pr.HeadSHA, "base", pr.BaseRef, "base_sha", base.state.SHA, "behind_by", outcome.BehindBy, "failing_checks", pr.FailingChecks) + } + if audit != nil { + audit(repo, pr.Number, fmt.Sprintf("repo=%s, pr=%d, head=%s, base_sha=%s, behind_by=%d, checks=%s, reason=failing checks green on the base tip", + repo, pr.Number, pr.HeadSHA, base.state.SHA, outcome.BehindBy, strings.Join(pr.FailingChecks, ","))) + } + if updated == nil { + updated = map[string]bool{} + } + updated[escalation.Key(repo, pr.Number)] = true + } + return updated +} + +// staleRedRefreshSkip names why pr is not a stale-red refresh candidate, or +// "" when it is: an open, non-draft PR whose settled CI is red on named +// checks that are not red on the base, with nothing that owns its branch +// sync elsewhere. +func staleRedRefreshSkip(pr github.PullRequest) string { + switch { + case pr.Draft: + return "draft" + case github.HasHoldLabel(pr.Labels) || github.HasExemptLabel(pr.Labels, nil): + return "held" + case pr.CIStatus != "failure" || len(pr.FailingChecks) == 0: + return "not-red" + case pr.CIChecksRunning: + return "ci-running" + case pr.SharedCIIncident > 0: + return "shared-ci-incident-open" + case len(pr.BaseRedChecks) > 0: + return "base-red" + case strings.TrimSpace(pr.HeadSHA) == "" || strings.TrimSpace(pr.BaseRef) == "": + return "head-or-base-unknown" + case pr.FromFork && !pr.MaintainerCanModify: + return "fork-not-editable" + case github.DependencyBotBranch(pr.HeadRef, pr.Author): + return "dependency-bot-branch" + case hasLabel(pr.Labels, stagelabels.ReturnedToContributor.Label()): + return "returned-to-contributor" + case hasLabel(pr.Labels, stagelabels.AdoptedByHive.Label()): + return "adopted-by-hive" + } + return "" +} diff --git a/src/cmd/hive/stale_red_base_refresh_test.go b/src/cmd/hive/stale_red_base_refresh_test.go new file mode 100644 index 0000000000..1149b73576 --- /dev/null +++ b/src/cmd/hive/stale_red_base_refresh_test.go @@ -0,0 +1,201 @@ +package main + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/github" + "github.com/hivecommons/hive/pkg/stagelabels" +) + +type fakeStaleRedRefresher struct { + bases map[string]github.StaleRedBaseState + baseErr error + baseReads []string + calls []github.PullRequest + out github.StaleRedRefreshOutcome + err error +} + +func (f *fakeStaleRedRefresher) StaleRedBaseStateFor(_ context.Context, repo, baseRef string) (github.StaleRedBaseState, bool, error) { + f.baseReads = append(f.baseReads, repo+"@"+baseRef) + if f.baseErr != nil { + return github.StaleRedBaseState{}, false, f.baseErr + } + st, ok := f.bases[repo+"@"+baseRef] + return st, ok, nil +} + +func (f *fakeStaleRedRefresher) RefreshStaleRedPR(_ context.Context, pr github.PullRequest, _ github.StaleRedBaseState) (github.StaleRedRefreshOutcome, error) { + f.calls = append(f.calls, pr) + return f.out, f.err +} + +// resetStaleRedRefresh gives each test a fresh pass clock and memo. +func resetStaleRedRefresh(t *testing.T) { + t.Helper() + prev := staleRedRefresh + staleRedRefresh = &staleRedRefreshState{} + t.Cleanup(func() { staleRedRefresh = prev }) +} + +// staleRedPR is projectbluefin/actions#544 in miniature: a contributor's +// fork PR (maintainers may edit), red on pytest, CI settled. +func staleRedPR(number int, head string) github.PullRequest { + pr := redPR("widgets", number, "contributor", head) + pr.FailingChecks = []string{"pytest"} + pr.BaseRef = "main" + pr.FromFork = true + pr.MaintainerCanModify = true + pr.Labels = []string{"needs-human"} + return pr +} + +func greenPytestBase(sha string) map[string]github.StaleRedBaseState { + return map[string]github.StaleRedBaseState{"acme/widgets@main": { + SHA: sha, Completed: map[string]bool{"pytest": true}, Red: map[string]bool{}, + }} +} + +// The base fixed what the PR's pytest tripped on: the hive merges the base +// in once for that (head, base tip), audits it, and tells the reaper to hold +// its fix kick. A second pass on the same head and tip makes no call; a new +// base tip is judged again. +func TestRefreshStaleRedBehindPRs_OncePerHeadAndBaseTip(t *testing.T) { + resetStaleRedRefresh(t) + cfg := escalationTestConfig() + actionable := actionableWith(staleRedPR(544, "head1")) + f := &fakeStaleRedRefresher{bases: greenPytestBase("base1"), out: github.StaleRedRefreshOutcome{Updated: true, Settled: true, BehindBy: 3}} + var audited []string + audit := func(repo string, number int, detail string) { audited = append(audited, detail) } + now := time.Date(2026, 11, 10, 5, 5, 0, 0, time.UTC) + + got := refreshStaleRedBehindPRs(context.Background(), cfg, f, actionable, audit, discardLogger(), now) + if len(f.calls) != 1 || f.calls[0].Repo != "acme/widgets" { + t.Fatalf("refresh calls = %+v, want one for acme/widgets#544", f.calls) + } + if !got["acme/widgets#544"] { + t.Fatalf("returned keys = %v, want acme/widgets#544 so the reaper skips it", got) + } + if len(audited) != 1 { + t.Fatalf("audit lines = %v, want one", audited) + } + + // Same head, same base tip, next pass: nothing. + refreshStaleRedBehindPRs(context.Background(), cfg, f, actionable, audit, discardLogger(), now.Add(staleRedRefreshInterval)) + if len(f.calls) != 1 { + t.Fatalf("refresh calls = %d after a second pass on the same head and base tip, want 1", len(f.calls)) + } + + // The base moved: the same head is judged against the new tip. + f.bases = greenPytestBase("base2") + refreshStaleRedBehindPRs(context.Background(), cfg, f, actionable, audit, discardLogger(), now.Add(2*staleRedRefreshInterval)) + if len(f.calls) != 2 { + t.Fatalf("refresh calls = %d after the base moved, want 2", len(f.calls)) + } +} + +// Passes are spaced: a cycle inside the interval reads nothing. +func TestRefreshStaleRedBehindPRs_Interval(t *testing.T) { + resetStaleRedRefresh(t) + cfg := escalationTestConfig() + f := &fakeStaleRedRefresher{bases: greenPytestBase("base1"), out: github.StaleRedRefreshOutcome{Settled: true, Skipped: "current"}} + now := time.Date(2026, 11, 10, 5, 5, 0, 0, time.UTC) + refreshStaleRedBehindPRs(context.Background(), cfg, f, actionableWith(staleRedPR(1, "h1")), nil, discardLogger(), now) + refreshStaleRedBehindPRs(context.Background(), cfg, f, actionableWith(staleRedPR(2, "h2")), nil, discardLogger(), now.Add(time.Minute)) + if len(f.baseReads) != 1 || len(f.calls) != 1 { + t.Fatalf("base reads %v, calls %d; want the second cycle inside the interval to do nothing", f.baseReads, len(f.calls)) + } +} + +// Only a base that has completed the failing check green clears a PR: a check +// still red there, or one the base never ran (a PR-only check), is not the +// base's to fix, and the PR is not touched. +func TestRefreshStaleRedBehindPRs_BaseMustClearEveryFailingCheck(t *testing.T) { + resetStaleRedRefresh(t) + cfg := escalationTestConfig() + redOnBase := staleRedPR(1, "h1") + redOnBase.FailingChecks = []string{"pytest", "build"} + prOnly := staleRedPR(2, "h2") + prOnly.FailingChecks = []string{"pytest", "Check consumer validation evidence"} + f := &fakeStaleRedRefresher{bases: map[string]github.StaleRedBaseState{"acme/widgets@main": { + SHA: "base1", Completed: map[string]bool{"pytest": true, "build": true}, Red: map[string]bool{"build": true}, + }}} + refreshStaleRedBehindPRs(context.Background(), cfg, f, actionableWith(redOnBase, prOnly), nil, discardLogger(), time.Now()) + if len(f.calls) != 0 { + t.Fatalf("refreshed %d PRs the base does not clear, want 0", len(f.calls)) + } + if len(f.baseReads) != 1 { + t.Fatalf("base reads = %v, want one per repo and base per pass", f.baseReads) + } +} + +// Updates are capped per repository per pass; the rest wait for a later pass. +func TestRefreshStaleRedBehindPRs_PerRepoCap(t *testing.T) { + resetStaleRedRefresh(t) + cfg := escalationTestConfig() + var prs []github.PullRequest + for i := 1; i <= staleRedRefreshMaxPerRepo+3; i++ { + prs = append(prs, staleRedPR(i, "h")) + } + f := &fakeStaleRedRefresher{bases: greenPytestBase("base1"), out: github.StaleRedRefreshOutcome{Updated: true, Settled: true, BehindBy: 1}} + got := refreshStaleRedBehindPRs(context.Background(), cfg, f, actionableWith(prs...), nil, discardLogger(), time.Now()) + if len(f.calls) != staleRedRefreshMaxPerRepo || len(got) != staleRedRefreshMaxPerRepo { + t.Fatalf("updates = %d (keys %v), want the per-repo cap %d", len(f.calls), got, staleRedRefreshMaxPerRepo) + } +} + +// A transient failure is not settled: the next pass tries the same pair again. +func TestRefreshStaleRedBehindPRs_ErrorRetriesNextPass(t *testing.T) { + resetStaleRedRefresh(t) + cfg := escalationTestConfig() + actionable := actionableWith(staleRedPR(544, "head1")) + f := &fakeStaleRedRefresher{bases: greenPytestBase("base1"), err: errors.New("502")} + now := time.Now() + refreshStaleRedBehindPRs(context.Background(), cfg, f, actionable, nil, discardLogger(), now) + f.err = nil + f.out = github.StaleRedRefreshOutcome{Updated: true, Settled: true} + refreshStaleRedBehindPRs(context.Background(), cfg, f, actionable, nil, discardLogger(), now.Add(staleRedRefreshInterval)) + if len(f.calls) != 2 { + t.Fatalf("refresh calls = %d, want a retry after the failed pass", len(f.calls)) + } +} + +func TestStaleRedRefreshSkip(t *testing.T) { + base := staleRedPR(1, "h") + cases := []struct { + name string + edit func(*github.PullRequest) + want string + }{ + {"candidate", func(*github.PullRequest) {}, ""}, + {"draft", func(p *github.PullRequest) { p.Draft = true }, "draft"}, + {"hold label", func(p *github.PullRequest) { p.Labels = append(p.Labels, "hold") }, "held"}, + {"do-not-merge", func(p *github.PullRequest) { p.Labels = append(p.Labels, "do-not-merge") }, "held"}, + {"green", func(p *github.PullRequest) { p.CIStatus = "success" }, "not-red"}, + {"red with no named check", func(p *github.PullRequest) { p.FailingChecks = nil }, "not-red"}, + {"CI still running", func(p *github.PullRequest) { p.CIChecksRunning = true }, "ci-running"}, + {"open shared-CI incident", func(p *github.PullRequest) { p.SharedCIIncident = 12 }, "shared-ci-incident-open"}, + {"red on the base too", func(p *github.PullRequest) { p.BaseRedChecks = []string{"pytest"} }, "base-red"}, + {"no base ref", func(p *github.PullRequest) { p.BaseRef = "" }, "head-or-base-unknown"}, + {"fork without maintainer edits", func(p *github.PullRequest) { p.MaintainerCanModify = false }, "fork-not-editable"}, + {"dependency bot branch", func(p *github.PullRequest) { p.HeadRef = "renovate/actions"; p.Author = "app/mergeraptor" }, "dependency-bot-branch"}, + {"returned to contributor", func(p *github.PullRequest) { + p.Labels = append(p.Labels, stagelabels.ReturnedToContributor.Label()) + }, "returned-to-contributor"}, + {"adopted by the hive", func(p *github.PullRequest) { p.Labels = append(p.Labels, stagelabels.AdoptedByHive.Label()) }, "adopted-by-hive"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + pr := base + pr.Labels = append([]string(nil), base.Labels...) + pr.FailingChecks = append([]string(nil), base.FailingChecks...) + tc.edit(&pr) + if got := staleRedRefreshSkip(pr); got != tc.want { + t.Fatalf("staleRedRefreshSkip = %q, want %q", got, tc.want) + } + }) + } +} diff --git a/src/docs/acmm-policy-matrix.md b/src/docs/acmm-policy-matrix.md index 00753cea08..e8b43ae4f8 100644 --- a/src/docs/acmm-policy-matrix.md +++ b/src/docs/acmm-policy-matrix.md @@ -262,7 +262,7 @@ gated. Defaults (override under `writeback.kinds.`): | `hold_gate_notice` (mergeability / drift) | — | hive PRs only | hive PRs only | hive PRs only | | `stage_label`, `needs_decision_comment` on foreign PRs | internal tracking | internal tracking | opt-in label | opt-in label | | `supersession_notice`, `duplicate_fix_notice`, `reporter_trust_hold`, `covered_by_pr_label`, `issue_close`, `parent_issue_edit`, `gate_comment` | — | — | opt-in label | opt-in label | -| `supersession_close`, `contributor_adopt`, `ci_rerun` | — | — | — | opt-in label | +| `supersession_close`, `contributor_adopt`, `ci_rerun`, `base_refresh` | — | — | — | opt-in label | | `verdict_label` | L3: opt-in label | opt-in label | opt-in label | opt-in label | | `claim_comment` | — | opt-in label | opt-in label | opt-in label | | `welcome_comment` | — | — | opt-in label | all items | diff --git a/src/docs/design/agent-turn-model.md b/src/docs/design/agent-turn-model.md index be699369d7..0e461b1a85 100644 --- a/src/docs/design/agent-turn-model.md +++ b/src/docs/design/agent-turn-model.md @@ -139,9 +139,9 @@ Kick *timing* lives in `pkg/governor`; kick *text* is built in `pkg/scheduler`. `deps.newTicker(initialEvalInterval)` at `src/cmd/hive/main.go:5772`, loop at `src/cmd/hive/main.go:6112`, evaluation via `deps.runEval` at `src/cmd/hive/main.go:5956` (`runEvalCycle`, `src/cmd/hive/main_eval_cycle.go:471`), - message assembly via `sched.BuildKickMessagesWithSkips` at `src/cmd/hive/main_eval_cycle.go:1094` + message assembly via `sched.BuildKickMessagesWithSkips` at `src/cmd/hive/main_eval_cycle.go:1107` (`src/pkg/scheduler/kickmessage.go:261`), and delivery via - `agentMgr.SendKick` wired into `dispatchAgentKicks` at `src/cmd/hive/main_eval_cycle.go:1580`. + `agentMgr.SendKick` wired into `dispatchAgentKicks` at `src/cmd/hive/main_eval_cycle.go:1593`. This matters for the RFC: the scheduler is already **stateless with respect to turns**. It does not hold a continuation, does not await turn *N* before diff --git a/src/docs/github-write-surface.md b/src/docs/github-write-surface.md index f9ff0af45f..16c587b07c 100644 --- a/src/docs/github-write-surface.md +++ b/src/docs/github-write-surface.md @@ -158,6 +158,7 @@ writes no entry. | Supersession sweep (notice, `hive/superseded` close, own-PR update-branch + body `Closes`→`Refs` rewrite, `needs-human` + `Parked` fact on escalation) | `pr_supersession_sweep.go`, `pr_supersession_autoclose.go`, `pr_supersession_resolve.go` | yes (`supersession-sweep-*`) | yes | | Shared-CI rerun (`rerun-failed-jobs`, `hive-shared-ci-rerun-` comment) | `shared_ci_rerun.go` | yes (`shared-ci-rerun`) | yes | | Transient-CI rerun (`rerun-failed-jobs` on a hive PR whose failing runs carry `hive-ci-transient`, once per head, no comment) | `transient_ci_rerun.go` | yes (`transient-ci-rerun`) | yes | +| Stale-red base refresh (`update-branch` on a red PR behind its base whose failing checks are green on the base tip, once per head and base tip, no comment) | `stale_red_base_refresh.go` | yes (`stale-red-base-refresh`) | yes | | Open-PR dispatch guard (`has-open-pr` label on a refused issue) and duplicate-fix sweep (one `hive:duplicate-fix` comment; `hive/superseded` close of the later hive PR, or `needs-human` + `Parked` fact on escalation) | `open_pr_links.go`, `pr_duplicate_fix_sweep.go` | yes (`duplicate-fix-*`) | yes | | Duplicate-PR sweep suggestions | `duplicate_sweep.go` | yes (`duplicate-sweep-suggested`) | yes (target = surviving PR) | | Hold-label migration (one-time) | `hive_hold_migration.go` | yes (`hold_migration_label_added`, one per item labeled; the migration report file is still written) | yes | diff --git a/src/docs/operator-reference.md b/src/docs/operator-reference.md index 941e315dab..4b4013d77b 100644 --- a/src/docs/operator-reference.md +++ b/src/docs/operator-reference.md @@ -43,6 +43,8 @@ The reference is organized into the sections below. Existing links to this page - [Central stage engine rollout](operator-reference/sweeps-and-operations.md#central-stage-engine-rollout) - [Shared-CI rerun sweep (`shared_ci_rerun`)](operator-reference/sweeps-and-operations.md#shared-ci-rerun-sweep-shared_ci_rerun) + +- [Stale-red base refresh (`stale_red_base_refresh`)](operator-reference/sweeps-and-operations.md#stale-red-base-refresh-stale_red_base_refresh) - [Open-PR dispatch guard and duplicate-fix sweep (`scanner`)](operator-reference/sweeps-and-operations.md#open-pr-dispatch-guard-and-duplicate-fix-sweep-scanner) diff --git a/src/docs/operator-reference/sweeps-and-operations.md b/src/docs/operator-reference/sweeps-and-operations.md index be6d1766a5..02c8661a61 100644 --- a/src/docs/operator-reference/sweeps-and-operations.md +++ b/src/docs/operator-reference/sweeps-and-operations.md @@ -308,6 +308,46 @@ someone re-runs it. The shared-CI rerun sweep |---|---|---| | `shared_ci_rerun.enabled` | `true` (unset) | Set `false` to stop re-running PRs when a `[shared-ci]` incident closes. | +## Stale-red base refresh (`stale_red_base_refresh`) + +A PR that went red on a check its base branch also broke stays red after the +base is fixed: re-running the check reuses the old merge ref, so only a new +head picks the fix up. On projectbluefin/actions#544 `pytest` failed on a +baseline pin that main fixed in #734; the PR sat red and behind for six hours +until a person pressed "Update branch", and `pytest` then passed. The +automerge lanes already refresh PRs the hive adopted (stage 074); the eval +cycle's stale-red base refresh (`src/cmd/hive/stale_red_base_refresh.go`, +`src/pkg/github/stale_red_base_refresh.go`) covers every other open PR, also +in repositories with auto-merge off: + +- A candidate is red on named checks with CI settled, not deferred to an open + `[shared-ci]` incident, not red on the base too, and behind its base. Every + check that failed on its head must have completed green on the current base + tip; a check the base never runs, or has not finished yet, does not count. + No guess is made about which files a check depends on. +- It merges the base in with `update-branch`, pinned to the evaluated head + (the same merge commit the "Update branch" button makes). It posts nothing. +- It skips drafts, held and exempt PRs, fork PRs with "allow edits by + maintainers" off, dependency-bot branches (the bot rebases them), PRs at + `stage/072-returned-to-contributor` (the author's hive pushes there) and at + `stage/074-adopted-by-hive` (the adoption recovery owns them). Repositories + in `auto_merge.require_current_base` are left to the automerge preparation + front. +- Each update goes through the write-back gate as `base_refresh`: allowed on + hive-authored PRs, and on other PRs at L6 with the opt-in label by default. + `writeback.kinds.base_refresh.unlabeled_min_level: 6` lets it reach every + PR at L6. +- A pass runs at most every 5 minutes and makes at most 2 updates per + repository and 6 in all. A (PR, head, base tip) it has settled is not tried + again until the head or the base moves. A conflict, an already-current + branch or a GitHub refusal (fork not editable, a workflow file the App may + not push) is logged and left for a person. Each update is logged as + `stale-red base refresh` and audited as `stale-red-base-refresh`, and the + stuck-PR reaper does not kick a fixer for that PR in the same cycle. +- It is an unstick step (`stale_red_base_refresh` in + `api_budget.unstick_steps`): it keeps running in `conserve` and is shed only + in `critical` and on cached-snapshot cycles. + ## Open-PR dispatch guard and duplicate-fix sweep (`scanner`) An issue whose fix is already open must not be dispatched again @@ -457,6 +497,7 @@ that level, the power the write-back exists to govern? | Code scanning sweep (`pkg/github/code_scanning.go`, `pkg/codescanning`) | hourly sweep (`code_scanning`, opt-in per repo) | **new** grouped issues from code scanning alerts, alert-table body edits, one closing comment + close when every alert is fixed or dismissed | the hive's own `hive/code-scanning` issues in repos with `code_scanning` on | `issue_file` → L3, no label (own items); see [code-scanning.md](../code-scanning.md) | | Shared-CI rerun (`shared_ci_rerun.go`) | CI failure watcher | `rerun-failed-jobs`, `hive-shared-ci-rerun-` comment | PRs with failed shared workflows | `ci_rerun` → L6 + label | | Transient-CI rerun (`transient_ci_rerun.go`) | eval cycle, before the stuck-PR reaper | `rerun-failed-jobs`, once per head, no comment | hive-authored red PRs whose every failing run carries `hive-ci-transient` | `ci_rerun` (own items: allowed unless the repo is paused, off or issues-only) | +| Stale-red base refresh (`cmd/hive/stale_red_base_refresh.go`, `pkg/github/stale_red_base_refresh.go`) | eval cycle, before the stuck-PR reaper, at most every 5 min | **branch push** (`update-branch` merge of the base), once per head and base tip, no comment | red PRs behind their base whose every failing check is green on the base tip; not drafts, held, non-editable forks, dependency-bot branches, 072 or 074 | `base_refresh` → hive-authored PRs, else L6 + label | | Level hold, self-authorization & reporter-trust releases (`pr_level_hold.go`, `automerge/self_authorization_release.go`, `reporter_trust_release.go`) | PR watcher | `hold` label add/remove | agent PRs | own items (hive-authored) — not gated | | Migrations & retirements (`hive_hold_migration.go`, `hive_id_label_retire.go`, `claimed_by_human_migration.go`, `origin_labels.go`) | boot, one-shot | label renames | items carrying hive labels | opted in by carrying hive labels | @@ -918,7 +959,7 @@ Spoke GitHub App installations share one core REST bucket across the scan loop a | Mode | Condition | Default behavior | | --- | --- | --- | | `normal` | `remaining >= github.api_reserve` | Full eval fan-out. Optional sweeps run every `governor.optional_sweep_every_n_cycles` cycles (default `1`). | -| `conserve` | `remaining < github.api_reserve` (default `800`) | Critical paths and unstick steps (escalation, SHA hold, supersession sweep, shared-CI rerun) continue; optional advisory/recommendation/duplicate/review-thread/collector enrichment sweeps are skipped. Eval interval is multiplied by `governor.conserve_interval_multiplier` (default `2`). | +| `conserve` | `remaining < github.api_reserve` (default `800`) | Critical paths and unstick steps (escalation, SHA hold, supersession sweep, shared-CI rerun, stale-red base refresh) continue; optional advisory/recommendation/duplicate/review-thread/collector enrichment sweeps are skipped. Eval interval is multiplied by `governor.conserve_interval_multiplier` (default `2`). | | `critical` | `remaining < github.api_critical` (default `250`) | Same shedding as conserve plus the unstick steps, with the eval interval stretched by `4x`. | Mode exits use hysteresis: Hive leaves conserve/critical only after quota rises above the threshold plus 100 calls or after the reset time passes. The effective interval is capped by `governor.eval_interval_max_s` (default `1800`). `/api/status` and `/api/gh-rate-limits` expose `api_budget` with the mode, remaining/limit/reset, mode `since`, `skipped_steps`, `unstick_steps` and `enrichment_steps`; the dashboard shows an API budget pill beside the GitHub rate-limit display. diff --git a/src/pkg/github/apibudget.go b/src/pkg/github/apibudget.go index 6682492e63..c68eb20e1d 100644 --- a/src/pkg/github/apibudget.go +++ b/src/pkg/github/apibudget.go @@ -152,6 +152,7 @@ var ( "supersession_sweep", "shared_ci_rerun", "escalation", + "stale_red_base_refresh", } APIBudgetEnrichmentSteps = []string{ "advisory_ensure", diff --git a/src/pkg/github/stale_red_base_refresh.go b/src/pkg/github/stale_red_base_refresh.go new file mode 100644 index 0000000000..14b0342311 --- /dev/null +++ b/src/pkg/github/stale_red_base_refresh.go @@ -0,0 +1,192 @@ +package github + +import ( + "context" + "fmt" + "sort" + "strings" + + gh "github.com/google/go-github/v72/github" + "github.com/hivecommons/hive/pkg/writeback" +) + +// Stale-red base refresh: a red PR whose base has since fixed the failure. +// +// A PR goes red on a check that is broken on its base branch too (a flaky +// test, a stale baseline). Someone fixes the base. The PR's CI never runs +// again: a re-run reuses the old merge ref, so only a new head picks up the +// fix. On projectbluefin/actions#544 `pytest` was red from a baseline pin +// that main fixed in #734 at 05:00Z on 2026-11-10; the PR sat red and +// behind until a maintainer pressed "Update branch" at 11:33Z, and pytest +// then passed. The automerge lanes have the same repair for PRs the hive +// adopted (automerge_adoption_recovery.go), but nothing ran it for any +// other PR, or for any PR in a repository with auto-merge off. +// +// The signal is deliberately simple: the PR is behind its base, and every +// check that failed on its head has completed green on the current base +// tip. No guess is made about which files a check depends on. The caller +// (cmd/hive) picks the candidates, caps the updates and remembers each +// (head, base tip) it settled, so a PR is updated at most once per base tip. + +// StaleRedBaseState is a base branch tip and the latest completed check runs +// on it. +type StaleRedBaseState struct { + // SHA is the base branch tip the check runs were read for. + SHA string + // Completed names every check with a completed latest run on SHA; + // Red names those whose latest run concluded red. + Completed map[string]bool + Red map[string]bool +} + +// StaleRedBaseVerdict is what the base tip says about a PR's failing checks. +type StaleRedBaseVerdict string + +const ( + // StaleRedBaseClears: every failing check is green on the base tip. + StaleRedBaseClears StaleRedBaseVerdict = "clears" + // StaleRedBaseRed: a failing check is red on the base tip too; a + // refresh would only pull in the same failure. + StaleRedBaseRed StaleRedBaseVerdict = "base-red" + // StaleRedBaseUnknown: a failing check has no completed run on the base + // tip (still running, or a PR-only check the base never runs). Nothing + // proves the base fixed it, so no refresh. + StaleRedBaseUnknown StaleRedBaseVerdict = "base-unknown" +) + +// StaleRedBaseVerdictFor judges a PR's failing checks against the base tip. +// It also returns the first check (sorted) that kept the verdict from +// clearing, or "" when it clears. +func StaleRedBaseVerdictFor(failing []string, base StaleRedBaseState) (StaleRedBaseVerdict, string) { + names := append([]string(nil), failing...) + sort.Strings(names) + if len(names) == 0 { + return StaleRedBaseUnknown, "" + } + for _, name := range names { + if base.Red[name] { + return StaleRedBaseRed, name + } + } + for _, name := range names { + if !base.Completed[name] { + return StaleRedBaseUnknown, name + } + } + return StaleRedBaseClears, "" +} + +// StaleRedBaseStateFor reads the tip of baseRef in repo and the latest +// completed check run per name on it (meta checks excluded, as the CI +// rollup excludes them). exists is false when the branch is gone. +func (c *Client) StaleRedBaseStateFor(ctx context.Context, repo, baseRef string) (StaleRedBaseState, bool, error) { + if c == nil || c.client == nil { + return StaleRedBaseState{}, false, ErrNoGitHubClient + } + ctx = WithRESTCaller(ctx, "hive:stale_red_base_refresh") + sha, exists, err := c.BranchHead(ctx, repo, baseRef) + if err != nil || !exists { + return StaleRedBaseState{}, exists, err + } + owner, name := c.splitRepo(repo) + var all []*gh.CheckRun + opts := &gh.ListCheckRunsOptions{ListOptions: gh.ListOptions{PerPage: 100}} + for { + runs, resp, err := c.client.Checks.ListCheckRunsForRef(ctx, owner, name, sha, opts) + if err != nil { + return StaleRedBaseState{}, true, fmt.Errorf("listing check runs on %s/%s@%s: %w", owner, name, shortSHA(sha), err) + } + if runs != nil { + all = append(all, runs.CheckRuns...) + } + if resp == nil || resp.NextPage == 0 { + break + } + opts.Page = resp.NextPage + } + st := StaleRedBaseState{SHA: sha, Completed: map[string]bool{}, Red: map[string]bool{}} + for _, cr := range latestCheckRunsByNameAndApp(all) { + n := cr.GetName() + if n == "" || isMetaCheck(n) || cr.GetStatus() != "completed" { + continue + } + st.Completed[n] = true + if checkConclusionIsRed(cr.GetConclusion()) { + st.Red[n] = true + } + } + return st, true, nil +} + +// StaleRedRefreshOutcome is what RefreshStaleRedPR did for one PR. +type StaleRedRefreshOutcome struct { + // Updated means GitHub accepted the update-branch request. + Updated bool + // Settled means this (head, base tip) needs no further attempt: the + // branch was updated, is already current, or the write-back gate or + // GitHub refused in a way the same token would meet again. Unsettled + // means a later pass may try again (the head moved under the request). + Settled bool + // BehindBy is how many base commits the head lacked. + BehindBy int + // Skipped names why nothing was updated ("" when Updated). + Skipped string +} + +// RefreshStaleRedPR merges the base tip into pr's branch when the head is +// behind base.SHA. The caller has already established that every failing +// check on the head is green on base.SHA (StaleRedBaseVerdictFor). The +// update is pinned to the evaluated head, so a push that landed after +// enumeration is never merged onto unseen. It is an unsolicited write on a +// PR the hive may not own, so the write-back gate (KindBaseRefresh) decides +// first. A repository whose base refresh belongs to the automerge +// preparation front (auto_merge.require_current_base) is left to it. +func (c *Client) RefreshStaleRedPR(ctx context.Context, pr PullRequest, base StaleRedBaseState) (StaleRedRefreshOutcome, error) { + if c == nil || c.client == nil { + return StaleRedRefreshOutcome{}, ErrNoGitHubClient + } + owner, name := c.splitRepo(pr.Repo) + repo := owner + "/" + name + if strings.TrimSpace(pr.HeadSHA) == "" || strings.TrimSpace(base.SHA) == "" { + return StaleRedRefreshOutcome{Settled: true, Skipped: "head-or-base-unknown"}, nil + } + if c.baseRefreshScheduled(repo) { + return StaleRedRefreshOutcome{Settled: true, Skipped: "automerge-owns-base-refresh"}, nil + } + preview := fmt.Sprintf("merge %s@%s into %s#%d at %s (failing checks green on the base tip)", repo, shortSHA(base.SHA), repo, pr.Number, shortSHA(pr.HeadSHA)) + if d := c.WritebackAllowed(repo, c.WritebackItemForPR(pr), writeback.KindBaseRefresh, preview); !d.Allowed { + return StaleRedRefreshOutcome{Settled: true, Skipped: d.Reason}, nil + } + cmp, _, err := c.client.Repositories.CompareCommits(WithRESTCaller(ctx, "hive:stale_red_base_refresh"), owner, name, base.SHA, pr.HeadSHA, &gh.ListOptions{PerPage: 1}) + if err != nil { + return StaleRedRefreshOutcome{}, fmt.Errorf("comparing %s@%s with %s#%d head %s: %w", repo, shortSHA(base.SHA), repo, pr.Number, shortSHA(pr.HeadSHA), err) + } + out := StaleRedRefreshOutcome{BehindBy: cmp.GetBehindBy()} + if out.BehindBy <= 0 { + out.Settled, out.Skipped = true, "current" + return out, nil + } + if err := c.UpdateBranchExpectedHead(ctx, repo, pr.Number, pr.HeadSHA, base.SHA); err != nil { // writeback:gated — WritebackAllowed(KindBaseRefresh) consulted above + if IsUpdateBranchStaleHead(err) { + out.Skipped = "head-moved" + return out, nil + } + switch ClassifyBranchUpdateError(err, pr.FromFork) { + case BranchUpdateUpToDate: + out.Settled, out.Skipped = true, "current" + case BranchUpdateConflict: + out.Settled, out.Skipped = true, "conflict" + case BranchUpdateForkNotEditable: + out.Settled, out.Skipped = true, "fork-not-editable" + case BranchUpdateForkWorkflowUpdate: + out.Settled, out.Skipped = true, "fork-workflow-update" + case BranchUpdateWorkflowPermission: + out.Settled, out.Skipped = true, "workflow-permission" + default: + return out, err + } + return out, nil + } + out.Updated, out.Settled = true, true + return out, nil +} diff --git a/src/pkg/github/stale_red_base_refresh_test.go b/src/pkg/github/stale_red_base_refresh_test.go new file mode 100644 index 0000000000..6beab27284 --- /dev/null +++ b/src/pkg/github/stale_red_base_refresh_test.go @@ -0,0 +1,260 @@ +package github + +import ( + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + + "github.com/hivecommons/hive/pkg/writeback" +) + +func TestStaleRedBaseVerdictFor(t *testing.T) { + base := StaleRedBaseState{ + SHA: "base1", + Completed: map[string]bool{"pytest": true, "lint": true, "build": true}, + Red: map[string]bool{"build": true}, + } + cases := []struct { + name string + failing []string + want StaleRedBaseVerdict + wantCheck string + }{ + {"failing check green on the base tip", []string{"pytest"}, StaleRedBaseClears, ""}, + {"every failing check green", []string{"lint", "pytest"}, StaleRedBaseClears, ""}, + {"failing check red on the base too", []string{"pytest", "build"}, StaleRedBaseRed, "build"}, + {"PR-only check the base never ran", []string{"pytest", "Check consumer validation evidence"}, StaleRedBaseUnknown, "Check consumer validation evidence"}, + {"nothing failing", nil, StaleRedBaseUnknown, ""}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got, check := StaleRedBaseVerdictFor(tc.failing, base) + if got != tc.want || check != tc.wantCheck { + t.Fatalf("StaleRedBaseVerdictFor(%v) = (%s, %q), want (%s, %q)", tc.failing, got, check, tc.want, tc.wantCheck) + } + }) + } +} + +// The base state is the branch tip and the newest completed run per check: +// a re-run that went green replaces the earlier failure, a run still in +// progress is not "completed", and meta checks are ignored. +func TestStaleRedBaseStateFor(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodGet && r.URL.Path == "/repos/acme/widgets/git/ref/heads/main": + _, _ = io.WriteString(w, `{"ref":"refs/heads/main","object":{"sha":"base1","type":"commit"}}`) + case r.Method == http.MethodGet && r.URL.Path == "/repos/acme/widgets/commits/base1/check-runs": + _, _ = io.WriteString(w, `{"total_count":5,"check_runs":[ + {"id":1,"name":"pytest","head_sha":"base1","status":"completed","conclusion":"failure","app":{"id":15368}}, + {"id":2,"name":"pytest","head_sha":"base1","status":"completed","conclusion":"success","app":{"id":15368}}, + {"id":3,"name":"lint","head_sha":"base1","status":"in_progress","app":{"id":15368}}, + {"id":4,"name":"build","head_sha":"base1","status":"completed","conclusion":"failure","app":{"id":15368}}, + {"id":5,"name":"tide","head_sha":"base1","status":"completed","conclusion":"failure","app":{"id":99}} + ]}`) + default: + t.Errorf("unexpected request: %s %s", r.Method, r.URL.Path) + http.NotFound(w, r) + } + })) + defer srv.Close() + c := newTestClient(t, srv, "acme", []string{"widgets"}) + + st, exists, err := c.StaleRedBaseStateFor(context.Background(), "acme/widgets", "main") + if err != nil || !exists { + t.Fatalf("StaleRedBaseStateFor = (%v, %v), want the tip", exists, err) + } + if st.SHA != "base1" { + t.Fatalf("SHA = %q, want base1", st.SHA) + } + if !st.Completed["pytest"] || st.Red["pytest"] { + t.Fatalf("pytest: completed=%v red=%v, want the newer green run to win", st.Completed["pytest"], st.Red["pytest"]) + } + if st.Completed["lint"] { + t.Fatal("lint is still running on the base tip; it must not read as completed") + } + if !st.Red["build"] { + t.Fatal("build is red on the base tip") + } + if st.Completed["tide"] || st.Red["tide"] { + t.Fatal("meta checks (tide) must be ignored") + } +} + +type staleRedRefreshFake struct { + mu sync.Mutex + behindBy int + updateCode int + updateMsg string + compares []string + updates []string + expectHeads []string +} + +func (f *staleRedRefreshFake) handler(t *testing.T) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + f.mu.Lock() + defer f.mu.Unlock() + switch { + case r.Method == http.MethodGet && strings.HasPrefix(r.URL.Path, "/repos/acme/widgets/compare/"): + f.compares = append(f.compares, strings.TrimPrefix(r.URL.Path, "/repos/acme/widgets/compare/")) + _ = json.NewEncoder(w).Encode(map[string]any{"behind_by": f.behindBy, "ahead_by": 2, "base_commit": map[string]any{"sha": "base1"}}) + case r.Method == http.MethodPut && r.URL.Path == "/repos/acme/widgets/pulls/7/update-branch": + var body struct { + ExpectedHeadSHA string `json:"expected_head_sha"` + } + _ = json.NewDecoder(r.Body).Decode(&body) + f.updates = append(f.updates, r.URL.Path) + f.expectHeads = append(f.expectHeads, body.ExpectedHeadSHA) + code := f.updateCode + if code == 0 { + code = http.StatusAccepted + } + w.WriteHeader(code) + msg := f.updateMsg + if msg == "" { + msg = "Updating pull request branch." + } + _ = json.NewEncoder(w).Encode(map[string]any{"message": msg}) + case r.Method == http.MethodGet && r.URL.Path == "/repos/acme/widgets/pulls/7": + _, _ = io.WriteString(w, `{"number":7,"head":{"sha":"head1"}}`) + default: + t.Errorf("unexpected request: %s %s", r.Method, r.URL.Path) + http.NotFound(w, r) + } + }) +} + +// projectbluefin/actions#544: pytest red on the head, green on the main tip +// after #734, head behind main. The hive merges main in, pinned to the head +// it judged, and posts nothing. +func TestRefreshStaleRedPR(t *testing.T) { + pr := PullRequest{Repo: "acme/widgets", Number: 7, Author: "contributor", HeadSHA: "head1", BaseRef: "main", + CIStatus: "failure", FailingChecks: []string{"pytest"}, FromFork: true, MaintainerCanModify: true} + base := StaleRedBaseState{SHA: "base1", Completed: map[string]bool{"pytest": true}, Red: map[string]bool{}} + fullGate := func() *writeback.Gate { + return writeback.New(writeback.Config{PerRepo: map[string]writeback.RepoOverride{"acme/widgets": {Mode: writeback.RepoModeFull}}}, writeback.Deps{}, "") + } + + t.Run("behind: merges the base in, pinned to the evaluated head", func(t *testing.T) { + fake := &staleRedRefreshFake{behindBy: 3} + srv := httptest.NewServer(fake.handler(t)) + defer srv.Close() + c := newTestClient(t, srv, "acme", []string{"widgets"}) + c.SetWritebackGate(fullGate()) + got, err := c.RefreshStaleRedPR(context.Background(), pr, base) + if err != nil { + t.Fatalf("RefreshStaleRedPR: %v", err) + } + if !got.Updated || !got.Settled || got.BehindBy != 3 || got.Skipped != "" { + t.Fatalf("outcome = %+v, want updated and settled, behind by 3", got) + } + if len(fake.compares) != 1 || fake.compares[0] != "base1...head1" { + t.Fatalf("compares = %v, want base1...head1", fake.compares) + } + if len(fake.updates) != 1 || fake.expectHeads[0] != "head1" { + t.Fatalf("update-branch calls = %v (expected heads %v), want one pinned to head1", fake.updates, fake.expectHeads) + } + }) + + t.Run("current: nothing to merge", func(t *testing.T) { + fake := &staleRedRefreshFake{behindBy: 0} + srv := httptest.NewServer(fake.handler(t)) + defer srv.Close() + c := newTestClient(t, srv, "acme", []string{"widgets"}) + c.SetWritebackGate(fullGate()) + got, err := c.RefreshStaleRedPR(context.Background(), pr, base) + if err != nil { + t.Fatalf("RefreshStaleRedPR: %v", err) + } + if got.Updated || !got.Settled || got.Skipped != "current" || len(fake.updates) != 0 { + t.Fatalf("outcome = %+v, updates %v; want settled as current with no update", got, fake.updates) + } + }) + + t.Run("write-back gate refusal makes no call at all", func(t *testing.T) { + fake := &staleRedRefreshFake{behindBy: 3} + srv := httptest.NewServer(fake.handler(t)) + defer srv.Close() + c := newTestClient(t, srv, "acme", []string{"widgets"}) + c.SetWritebackGate(writeback.New(writeback.Config{PerRepo: map[string]writeback.RepoOverride{ + "acme/widgets": {Mode: writeback.RepoModeOff}, + }}, writeback.Deps{}, "")) + got, err := c.RefreshStaleRedPR(context.Background(), pr, base) + if err != nil { + t.Fatalf("RefreshStaleRedPR: %v", err) + } + if got.Updated || !got.Settled || got.Skipped != writeback.ReasonRepoOff { + t.Fatalf("outcome = %+v, want settled and skipped for %q", got, writeback.ReasonRepoOff) + } + if len(fake.compares)+len(fake.updates) != 0 { + t.Fatalf("gate refused, but called compare %v / update %v", fake.compares, fake.updates) + } + }) + + t.Run("a foreign PR without the opt-in label is refused by default", func(t *testing.T) { + fake := &staleRedRefreshFake{behindBy: 3} + srv := httptest.NewServer(fake.handler(t)) + defer srv.Close() + c := newTestClient(t, srv, "acme", []string{"widgets"}) + c.SetWritebackGate(writeback.New(writeback.Config{}, writeback.Deps{LevelFor: func(string) int { return 6 }}, "")) + got, err := c.RefreshStaleRedPR(context.Background(), pr, base) + if err != nil { + t.Fatalf("RefreshStaleRedPR: %v", err) + } + if got.Updated || got.Skipped != writeback.ReasonNeedsLabel || len(fake.updates) != 0 { + t.Fatalf("outcome = %+v, updates %v; want refused for the opt-in label", got, fake.updates) + } + }) + + t.Run("the automerge preparation front owns the repo", func(t *testing.T) { + fake := &staleRedRefreshFake{behindBy: 3} + srv := httptest.NewServer(fake.handler(t)) + defer srv.Close() + c := newTestClient(t, srv, "acme", []string{"widgets"}) + c.SetWritebackGate(fullGate()) + c.SetScheduledBaseRefresh(func(string) bool { return true }) + got, err := c.RefreshStaleRedPR(context.Background(), pr, base) + if err != nil { + t.Fatalf("RefreshStaleRedPR: %v", err) + } + if got.Updated || got.Skipped != "automerge-owns-base-refresh" || len(fake.compares)+len(fake.updates) != 0 { + t.Fatalf("outcome = %+v, calls %v/%v; want it left to automerge", got, fake.compares, fake.updates) + } + }) + + t.Run("a merge conflict settles without retrying", func(t *testing.T) { + fake := &staleRedRefreshFake{behindBy: 3, updateCode: http.StatusUnprocessableEntity, updateMsg: "merge conflict between base and head"} + srv := httptest.NewServer(fake.handler(t)) + defer srv.Close() + c := newTestClient(t, srv, "acme", []string{"widgets"}) + c.SetWritebackGate(fullGate()) + got, err := c.RefreshStaleRedPR(context.Background(), pr, base) + if err != nil { + t.Fatalf("RefreshStaleRedPR: %v", err) + } + if got.Updated || !got.Settled || got.Skipped != "conflict" { + t.Fatalf("outcome = %+v, want settled as conflict", got) + } + }) + + t.Run("a head that moved under the request stays open for the next pass", func(t *testing.T) { + fake := &staleRedRefreshFake{behindBy: 3, updateCode: http.StatusUnprocessableEntity, updateMsg: "expected head sha didn't match current head ref."} + srv := httptest.NewServer(fake.handler(t)) + defer srv.Close() + c := newTestClient(t, srv, "acme", []string{"widgets"}) + c.SetWritebackGate(fullGate()) + got, err := c.RefreshStaleRedPR(context.Background(), pr, base) + if err != nil { + t.Fatalf("RefreshStaleRedPR: %v", err) + } + if got.Updated || got.Settled || got.Skipped != "head-moved" { + t.Fatalf("outcome = %+v, want unsettled head-moved", got) + } + }) +} diff --git a/src/pkg/writeback/dedupe.go b/src/pkg/writeback/dedupe.go index 8ae6a4fc89..755d656e6a 100644 --- a/src/pkg/writeback/dedupe.go +++ b/src/pkg/writeback/dedupe.go @@ -71,7 +71,7 @@ func IsContentReason(reason string) bool { // ContentDedupeKinds are the kinds whose preview is a comment body the // content layer may dedupe. Kinds that re-run their gate check every tick // with a constant preview and keep their own idempotence (hold_gate_notice, -// the closes, parent_issue_edit, ci_rerun), the label kinds and issue_file +// the closes, parent_issue_edit, ci_rerun, base_refresh), the label kinds and issue_file // (no existing item) are deliberately absent. var ContentDedupeKinds = map[Kind]bool{ KindSentinel: true, diff --git a/src/pkg/writeback/dedupe_test.go b/src/pkg/writeback/dedupe_test.go index e4dd4b2eae..2463cc4ae1 100644 --- a/src/pkg/writeback/dedupe_test.go +++ b/src/pkg/writeback/dedupe_test.go @@ -195,7 +195,7 @@ func TestKindSets(t *testing.T) { if !k.RateCapExempt() { t.Errorf("%s: label kinds must be exempt from the hourly cap", k) } - case k.IsIssueFileKind(), k == KindSupersessionClose, k == KindIssueClose, k == KindHoldGateNotice, k == KindParentIssueEdit, k == KindCIRerun: + case k.IsIssueFileKind(), k == KindSupersessionClose, k == KindIssueClose, k == KindHoldGateNotice, k == KindParentIssueEdit, k == KindCIRerun, k == KindBaseRefresh: if k.ContentDeduped() { t.Errorf("%s re-checks the gate with a constant preview; it must not be content-deduped", k) } diff --git a/src/pkg/writeback/kinds.go b/src/pkg/writeback/kinds.go index 97ed107f5f..437116aec7 100644 --- a/src/pkg/writeback/kinds.go +++ b/src/pkg/writeback/kinds.go @@ -86,6 +86,11 @@ const ( KindIssueFile Kind = "issue_file" // KindCIRerun re-runs a workflow or posts `/ok-to-test` on a PR. KindCIRerun Kind = "ci_rerun" + // KindBaseRefresh merges the base branch into a red PR that is behind + // it, when every failing check on the PR is green on the base tip: the + // base has fixed what the PR's CI tripped on, and only a new head runs + // that check again. It pushes a merge commit and posts nothing. + KindBaseRefresh Kind = "base_refresh" ) // KindPolicy is the per-kind rule. A write of this kind on an item the hive @@ -142,6 +147,7 @@ func DefaultKinds() map[Kind]KindPolicy { KindWelcomeComment: {MinLevel: 5, RequireLabel: true, UnlabeledMinLevel: 6}, KindIssueFile: {MinLevel: 3}, KindCIRerun: {MinLevel: 6, RequireLabel: true}, + KindBaseRefresh: {MinLevel: 6, RequireLabel: true}, } }