diff --git a/changelog.d/fixed-transient-ci-rerun.md b/changelog.d/fixed-transient-ci-rerun.md new file mode 100644 index 0000000000..cbce1dc8c2 --- /dev/null +++ b/changelog.d/fixed-transient-ci-rerun.md @@ -0,0 +1 @@ +- A red hive pull request whose failure an agent triaged as infrastructure is now re-run instead of sent back to a fixer. On [projectbluefin/common#1647](https://github.com/projectbluefin/common/pull/1647) the agent said "infrastructure, not this change" four times, the fix loop still counted six fix re-dispatches on that one commit and escalated the PR to a person, and a lane then pushed an empty "retrigger CI" commit. When every failing check run on the head carries a `hive-ci-transient` marker, Hive now re-runs the failed jobs once for that head (`rerun-failed-jobs`, through the write-back gate as `ci_rerun`) and sends no fix kick while that re-run is owed. A head still red after the re-run goes back to the normal fix loop. The fix kick now tells the agent to stamp the marker and never push to retrigger CI, and the reviewer lane's DE-ESCALATE verdict no longer asks for a push when the branch is already current. diff --git a/src/cmd/hive/main_eval_cycle.go b/src/cmd/hive/main_eval_cycle.go index 873befe1a5..5c1e3540b3 100644 --- a/src/cmd/hive/main_eval_cycle.go +++ b/src/cmd/hive/main_eval_cycle.go @@ -798,7 +798,23 @@ func runEvalCycle( // re-engagement budget only for kicks that were accepted. Composes with the // merge watcher (#2): both go through the same per-red-SHA cap, so the same // PR is never double-dispatched within a red-SHA's budget. - reapStuckRedPRs(cfg, actionable, escalatedPRs, agentKicker{mgr: agentMgr}, + // + // A head an agent triaged as transient infrastructure (hive-ci-transient + // markers) is re-run once here instead: the reaper and the merge watcher + // hold their fix kicks while that re-run is owed + // (projectbluefin/common#1647). + reapSkip := escalatedPRs + if ghClient != nil { + var audit func(string, int, string) + if dashSrv != nil { + audit = func(repo string, number int, detail string) { + dashSrv.AuditLogRecord("system", "transient-ci-rerun", detail, "", repo, number) + } + } + reran := rerunTransientTriagedPRs(ctx, cfg, ghClient, actionable, escalatedPRs, audit, logger) + reapSkip = reaperSkipAfterRerun(escalatedPRs, reran) + } + reapStuckRedPRs(cfg, actionable, reapSkip, agentKicker{mgr: agentMgr}, agentAvailability(cfg, agentMgr), logger) // Release sentinel (hivecommons/hive#9585), opt-in and default OFF: when diff --git a/src/cmd/hive/main_sweeps.go b/src/cmd/hive/main_sweeps.go index 3e00d1e9af..9bf87b3cc9 100644 --- a/src/cmd/hive/main_sweeps.go +++ b/src/cmd/hive/main_sweeps.go @@ -123,6 +123,7 @@ func escalationObservation(cfg *config.Config, pr github.PullRequest, owners map FailingChecks: pr.FailingChecks, Agent: owners[escalation.Key(repo, pr.Number)], IndependentControl: isIndependentCIControl(cfg, pr), + TransientTriaged: red && pr.TransientCITriaged, } } @@ -360,8 +361,17 @@ func redPRFixKick(o escalation.Observation, rerouted bool) string { fmt.Fprintf(&b, " gh pr checkout %d --repo %s → fix → git commit -s → git push\n", o.Number, o.Repo) b.WriteString("Push to the SAME branch. Do NOT open a replacement PR.\n") b.WriteString("After the push, STOP: do not watch, poll, or sleep on CI (no gh run watch/view loops).\n") - b.WriteString("The automerge sweep merges the PR once it is green; if the failure is infrastructure\n") - b.WriteString("(runner lost, no space left, checks still queued) leave one comment and end the turn.\n") + b.WriteString("The automerge sweep merges the PR once it is green; if checks are still queued, end the turn.\n") + // An infra verdict must reach the hive as a marker, not prose: four + // "infrastructure, not this change" comments on projectbluefin/common#1647 + // were invisible to the fix loop, which kicked again until the PR + // escalated, and a lane then pushed an empty "retrigger CI" commit. + b.WriteString("If the failure is infrastructure, not this change (runner lost, no space left, a package\n") + b.WriteString("mirror or download dropping the connection): do NOT push anything — no empty, no-op or\n") + b.WriteString("\"retrigger CI\" commit; a push re-arms review fences and can dismiss approvals. Leave ONE\n") + fmt.Fprintf(&b, "comment with the evidence that ends with the transient_marker printed by\n hive-baseline-check.sh %s '' %d --json\n", o.Repo, o.Number) + b.WriteString("(one per failing check), then end the turn. Once every failing check carries one, Hive\n") + b.WriteString("re-runs the failed jobs itself, once.\n") if o.HeadSHA != "" { fmt.Fprintf(&b, "\nhead: %s\n", o.HeadSHA) } @@ -436,6 +446,17 @@ func deliverReEngagement( available func(string) bool, logger *slog.Logger, ) bool { + if store.TransientRerunOwed(o.Repo, o.Number, headSHA) { + // An agent already triaged this head's failure as infrastructure; + // a fix kick could only produce the same comment again and spend + // the budget that escalates the PR (projectbluefin/common#1647). + // rerunTransientTriagedPRs re-runs the failed jobs instead. + if logger != nil { + logger.Debug("re-engagement held: failure triaged as transient, hive re-runs the failed jobs", + "repo", o.Repo, "pr", o.Number) + } + return false + } fixer := resolveRedPRFixer(cfg, o.Agent, available) if fixer == "" { // Nobody can be asked right now. Charging budget here is what escalated diff --git a/src/cmd/hive/transient_ci_rerun.go b/src/cmd/hive/transient_ci_rerun.go new file mode 100644 index 0000000000..417b6947e0 --- /dev/null +++ b/src/cmd/hive/transient_ci_rerun.go @@ -0,0 +1,125 @@ +package main + +import ( + "context" + "fmt" + "log/slog" + + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/escalation" + "github.com/hivecommons/hive/pkg/github" +) + +// transientCIRerunMaxPerPass bounds the re-run calls one eval cycle makes. +const transientCIRerunMaxPerPass = 10 + +// transientCIRerunner re-runs a transient-triaged PR's failed jobs; +// *github.Client implements it. +type transientCIRerunner interface { + RerunTransientCIFailures(ctx context.Context, pr github.PullRequest) (github.TransientCIRerunOutcome, error) +} + +// rerunTransientTriagedPRs is the fix loop's answer to a red hive-authored PR +// whose every failing check run an agent triaged as infrastructure, not this +// change (`hive-ci-transient` markers, github.PullRequest.TransientCITriaged): +// it re-runs the failed jobs once per head through rerun-failed-jobs, and +// never kicks a fixer or pushes a commit for it. On +// projectbluefin/common#1647 the fix loop instead kicked the owning agent six +// times on one head, each kick answered with "infrastructure, not this +// change", escalated the PR to a person, and a lane then pushed an empty +// "retrigger CI" commit. +// +// It runs after recordRedStaleness (which records the triaged head in the +// ledger) and before reapStuckRedPRs, whose kicks deliverReEngagement holds +// while a re-run is owed. audit, when non-nil, receives one line per PR the +// hive re-ran. +// +// It returns the escalation keys of the PRs whose failed jobs it re-ran this +// pass. The reaper runs next in the same cycle, on the same snapshot, where +// those PRs still read red and stale, and the re-run is no longer owed; the +// caller skips them there (reaperSkipAfterRerun) so a re-run and a fix kick +// never go out together for one head (hivecommons/hive#17002 review). A PR +// whose re-run was skipped (no failed runs, re-run forbidden) is not +// returned: the fix loop resumes on it, as intended. +func rerunTransientTriagedPRs( + ctx context.Context, + cfg *config.Config, + rerunner transientCIRerunner, + actionable *github.ActionableResult, + escalatedPRs map[string]bool, + audit func(repo string, number int, detail string), + logger *slog.Logger, +) map[string]bool { + if cfg == nil || cfg.Escalation.Disabled || rerunner == nil || actionable == nil { + return nil + } + store := getEscalationStore() + attempted := 0 + var reran map[string]bool + for _, pr := range actionable.PRs.Items { + if attempted >= transientCIRerunMaxPerPass { + break + } + if !isHiveAgentAuthor(cfg, pr.Author) || !pr.TransientCITriaged || pr.CIChecksRunning || !pr.HasFailingRequiredCheck() { + continue + } + o := escalationObservation(cfg, pr, nil) + key := escalation.Key(o.Repo, o.Number) + if escalatedPRs[key] || escalation.HasNeedsHumanLabel(pr.Labels) { + continue + } + if !store.TransientRerunOwed(o.Repo, o.Number, o.HeadSHA) { + continue + } + attempted++ + outcome, err := rerunner.RerunTransientCIFailures(ctx, pr) + if err != nil { + if logger != nil { + logger.Warn("transient CI rerun: could not list failed runs; retrying next pass", + "repo", o.Repo, "pr", o.Number, "head", o.HeadSHA, "error", err) + } + continue + } + if !outcome.Settled { + continue + } + store.RecordTransientRerun(o.Repo, o.Number, o.HeadSHA) + if len(outcome.RunIDs) == 0 { + if logger != nil { + logger.Info("transient CI rerun skipped; the fix loop resumes on this head", + "repo", o.Repo, "pr", o.Number, "head", o.HeadSHA, "reason", outcome.Skipped) + } + continue + } + if logger != nil { + logger.Info("transient CI rerun: re-ran failed jobs instead of a fix kick", + "repo", o.Repo, "pr", o.Number, "head", o.HeadSHA, "runs", outcome.RunIDs) + } + if audit != nil { + audit(o.Repo, o.Number, fmt.Sprintf("repo=%s, pr=%d, head=%s, runs=%d, reason=transient CI triage", + o.Repo, o.Number, o.HeadSHA, len(outcome.RunIDs))) + } + if reran == nil { + reran = map[string]bool{} + } + reran[key] = true + } + return reran +} + +// reaperSkipAfterRerun is the set the reaper skips this cycle: the escalated +// PRs, plus the PRs whose failed jobs rerunTransientTriagedPRs just re-ran. +// escalated is returned unchanged when nothing was re-ran. +func reaperSkipAfterRerun(escalated, reran map[string]bool) map[string]bool { + if len(reran) == 0 { + return escalated + } + skip := make(map[string]bool, len(escalated)+len(reran)) + for k, v := range escalated { + skip[k] = v + } + for k := range reran { + skip[k] = true + } + return skip +} diff --git a/src/cmd/hive/transient_ci_rerun_test.go b/src/cmd/hive/transient_ci_rerun_test.go new file mode 100644 index 0000000000..886c6b063f --- /dev/null +++ b/src/cmd/hive/transient_ci_rerun_test.go @@ -0,0 +1,221 @@ +package main + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/escalation" + "github.com/hivecommons/hive/pkg/github" +) + +type fakeTransientRerunner struct { + calls []github.PullRequest + out github.TransientCIRerunOutcome + err error +} + +func (f *fakeTransientRerunner) RerunTransientCIFailures(_ context.Context, pr github.PullRequest) (github.TransientCIRerunOutcome, error) { + f.calls = append(f.calls, pr) + return f.out, f.err +} + +func transientRedPR() github.PullRequest { + pr := redPR("widgets", 7, "hive-agent", "abc") + pr.TransientCITriaged = true + return pr +} + +// projectbluefin/common#1647: an agent said "infrastructure, not this change" +// four times, yet the fix loop delivered six fix re-dispatches on the same +// head and escalated the PR. A head whose every failing run carries a +// hive-ci-transient marker must draw no fix kick and spend no budget, from +// the reaper or the merge watcher's hook. +func TestTransientTriagedHeadDrawsNoFixKick(t *testing.T) { + store, clock := newTestEscalationStore(t) + cfg := escalationTestConfig() + actionable := actionableWith(transientRedPR()) + recordRedStaleness(cfg, actionable) + *clock = clock.Add(escalation.RedPRStaleAfter + time.Minute) + kicker := &fakeKicker{} + + for i := 0; i < escalation.MaxReEngagements+2; i++ { + reapStuckRedPRs(cfg, actionable, map[string]bool{}, kicker, nil, discardLogger()) + *clock = clock.Add(escalation.MaxReEngageCooldown + time.Minute) + } + if hook := mergeReEngageHook(cfg, kicker, nil, discardLogger()); hook("widgets", 7) { + t.Error("merge watcher re-engaged a transient-triaged head") + } + if len(kicker.kicks) != 0 { + t.Fatalf("transient-triaged head got %d fix kicks, want 0", len(kicker.kicks)) + } + if got := store.ReEngagements("acme/widgets", 7); got != 0 { + t.Fatalf("transient-triaged head spent %d re-engagements, want 0", got) + } +} + +// The hive re-runs the failed jobs once per head; a head still red after +// that re-run (new run ids, so the markers are stale) is back in the +// ordinary fix loop. +func TestRerunTransientTriagedPRs_OncePerHeadThenFixLoopResumes(t *testing.T) { + store, clock := newTestEscalationStore(t) + cfg := escalationTestConfig() + pr := transientRedPR() + actionable := actionableWith(pr) + recordRedStaleness(cfg, actionable) + rerunner := &fakeTransientRerunner{out: github.TransientCIRerunOutcome{Settled: true, RunIDs: []int64{42}}} + var audited []string + audit := func(repo string, number int, detail string) { audited = append(audited, detail) } + + rerunTransientTriagedPRs(context.Background(), cfg, rerunner, actionable, map[string]bool{}, audit, discardLogger()) + rerunTransientTriagedPRs(context.Background(), cfg, rerunner, actionable, map[string]bool{}, audit, discardLogger()) + if len(rerunner.calls) != 1 { + t.Fatalf("re-run calls = %d, want exactly one per head", len(rerunner.calls)) + } + if len(audited) != 1 || !strings.Contains(audited[0], "runs=1") { + t.Fatalf("audit = %v, want one line naming the re-run", audited) + } + if store.TransientRerunOwed("acme/widgets", 7, "abc") { + t.Fatal("re-run still owed after it was spent") + } + + // Red again after the re-run: the re-run gave the checks new run ids, + // so the markers no longer cover the head. + pr.TransientCITriaged = false + actionable = actionableWith(pr) + recordRedStaleness(cfg, actionable) + *clock = clock.Add(escalation.RedPRStaleAfter + time.Minute) + kicker := &fakeKicker{} + reapStuckRedPRs(cfg, actionable, map[string]bool{}, kicker, nil, discardLogger()) + if len(kicker.kicks) != 1 { + t.Fatalf("after the one re-run the fix loop must resume, got %d kicks", len(kicker.kicks)) + } + + // A new head with a fresh transient triage earns its own re-run. + pr.HeadSHA, pr.TransientCITriaged = "def", true + actionable = actionableWith(pr) + recordRedStaleness(cfg, actionable) + rerunTransientTriagedPRs(context.Background(), cfg, rerunner, actionable, map[string]bool{}, nil, discardLogger()) + if len(rerunner.calls) != 2 || rerunner.calls[1].HeadSHA != "def" { + t.Fatalf("new head was not re-run: %+v", rerunner.calls) + } +} + +// hivecommons/hive#17002 review: the reaper runs right after the re-run step, +// in the same cycle and on the same snapshot, where the head still reads red +// and stale and the re-run is no longer owed. With no earlier kick (the agent +// stamped the marker in its own session), no cooldown stands in the way, so +// the cycle that re-ran the jobs must not also kick a fixer. +func TestRerunTransientTriagedPRs_SameCycleDrawsNoFixKick(t *testing.T) { + sameCycle := func(t *testing.T, skip func(escalated, reran map[string]bool) map[string]bool) int { + t.Helper() + _, clock := newTestEscalationStore(t) + cfg := escalationTestConfig() + actionable := actionableWith(transientRedPR()) + recordRedStaleness(cfg, actionable) + *clock = clock.Add(escalation.RedPRStaleAfter + time.Minute) + rerunner := &fakeTransientRerunner{out: github.TransientCIRerunOutcome{Settled: true, RunIDs: []int64{42}}} + escalated := map[string]bool{} + reran := rerunTransientTriagedPRs(context.Background(), cfg, rerunner, actionable, escalated, nil, discardLogger()) + if len(rerunner.calls) != 1 || !reran["acme/widgets#7"] { + t.Fatalf("re-run calls = %d, reran = %v; want the head re-run and reported", len(rerunner.calls), reran) + } + kicker := &fakeKicker{} + reapStuckRedPRs(cfg, actionable, skip(escalated, reran), kicker, nil, discardLogger()) + return len(kicker.kicks) + } + if got := sameCycle(t, func(escalated, _ map[string]bool) map[string]bool { return escalated }); got != 1 { + t.Fatalf("test premise: reaping with the escalated set alone kicks the re-run head (got %d kicks, want 1)", got) + } + if got := sameCycle(t, reaperSkipAfterRerun); got != 0 { + t.Fatalf("the cycle that re-ran the jobs also sent %d fix kicks, want 0", got) + } + + // A skipped re-run (no failed runs) is not reported: the fix loop resumes. + _, clock := newTestEscalationStore(t) + cfg := escalationTestConfig() + actionable := actionableWith(transientRedPR()) + recordRedStaleness(cfg, actionable) + *clock = clock.Add(escalation.RedPRStaleAfter + time.Minute) + skipped := &fakeTransientRerunner{out: github.TransientCIRerunOutcome{Settled: true, Skipped: "no-failed-runs"}} + if reran := rerunTransientTriagedPRs(context.Background(), cfg, skipped, actionable, map[string]bool{}, nil, discardLogger()); len(reran) != 0 { + t.Fatalf("a skipped re-run was reported as re-ran: %v", reran) + } +} + +func TestRerunTransientTriagedPRs_Skips(t *testing.T) { + t.Run("escalated PR is a person's", func(t *testing.T) { + newTestEscalationStore(t) + cfg := escalationTestConfig() + pr := transientRedPR() + pr.Labels = []string{escalation.NeedsHumanLabel} + actionable := actionableWith(pr) + recordRedStaleness(cfg, actionable) + rerunner := &fakeTransientRerunner{out: github.TransientCIRerunOutcome{Settled: true}} + rerunTransientTriagedPRs(context.Background(), cfg, rerunner, actionable, map[string]bool{"acme/widgets#7": true}, nil, discardLogger()) + if len(rerunner.calls) != 0 { + t.Fatal("re-ran an escalated PR") + } + }) + t.Run("human-authored PR", func(t *testing.T) { + newTestEscalationStore(t) + cfg := escalationTestConfig() + pr := transientRedPR() + pr.Author = "someone" + actionable := actionableWith(pr) + recordRedStaleness(cfg, actionable) + rerunner := &fakeTransientRerunner{out: github.TransientCIRerunOutcome{Settled: true}} + rerunTransientTriagedPRs(context.Background(), cfg, rerunner, actionable, map[string]bool{}, nil, discardLogger()) + if len(rerunner.calls) != 0 { + t.Fatal("re-ran a human-authored PR") + } + }) + t.Run("untriaged red PR", func(t *testing.T) { + newTestEscalationStore(t) + cfg := escalationTestConfig() + actionable := actionableWith(redPR("widgets", 7, "hive-agent", "abc")) + recordRedStaleness(cfg, actionable) + rerunner := &fakeTransientRerunner{out: github.TransientCIRerunOutcome{Settled: true}} + rerunTransientTriagedPRs(context.Background(), cfg, rerunner, actionable, map[string]bool{}, nil, discardLogger()) + if len(rerunner.calls) != 0 { + t.Fatal("re-ran a PR nobody triaged as transient") + } + }) + t.Run("listing error is retried next pass", func(t *testing.T) { + store, _ := newTestEscalationStore(t) + cfg := escalationTestConfig() + actionable := actionableWith(transientRedPR()) + recordRedStaleness(cfg, actionable) + rerunner := &fakeTransientRerunner{err: errors.New("502")} + rerunTransientTriagedPRs(context.Background(), cfg, rerunner, actionable, map[string]bool{}, nil, discardLogger()) + if !store.TransientRerunOwed("acme/widgets", 7, "abc") { + t.Fatal("an API error spent the head's re-run") + } + rerunner.err, rerunner.out = nil, github.TransientCIRerunOutcome{Settled: true, Skipped: "repo-mode-off"} + rerunTransientTriagedPRs(context.Background(), cfg, rerunner, actionable, map[string]bool{}, nil, discardLogger()) + if len(rerunner.calls) != 2 || store.TransientRerunOwed("acme/widgets", 7, "abc") { + t.Fatalf("a settled refusal must spend the re-run: calls=%d", len(rerunner.calls)) + } + }) +} + +// The fix kick must tell the agent how to make an infra verdict visible to +// the hive (the transient marker) and forbid a retrigger push; on +// projectbluefin/common#1647 the prose-only verdict was invisible and a lane +// pushed an empty "retrigger CI" commit. +func TestRedPRFixKickInfraVerdictUsesTransientMarker(t *testing.T) { + got := redPRFixKick(escalation.Observation{Repo: "projectbluefin/common", Number: 1647, HeadSHA: "6772623"}, false) + for _, want := range []string{ + "do NOT push anything", + "\"retrigger CI\" commit", + "transient_marker", + "hive-baseline-check.sh projectbluefin/common '' 1647 --json", + "Hive\nre-runs the failed jobs itself, once.", + } { + if !strings.Contains(got, want) { + t.Errorf("fix kick missing %q in:\n%s", want, got) + } + } +} diff --git a/src/docs/adr/0010-escalation-circuit-breaker.md b/src/docs/adr/0010-escalation-circuit-breaker.md index a8beb7407b..5ece932454 100644 --- a/src/docs/adr/0010-escalation-circuit-breaker.md +++ b/src/docs/adr/0010-escalation-circuit-breaker.md @@ -50,6 +50,27 @@ cadence window. The staleness clock starts when CI SETTLES — a head red on one check while others are still running is not yet stuck. The escalation comment quotes the number of kicks actually delivered. +### Re-run, do not re-dispatch, a transient-triaged head + +A fix kick cannot repair an infrastructure failure. When every failing check +run on a hive-authored PR's head carries an agent's `hive-ci-transient` triage +marker, the eval cycle re-runs that head's failed jobs once +(`rerun-failed-jobs`, through the write-back gate as `ci_rerun`), and the +reaper and merge watcher send no fix kick and charge no budget while that +re-run is owed. Only markers in the hive's own comments count, because a +marker now causes a write: a comment from anyone else cannot spend Actions +minutes or pause the fix loop. (A hive that cannot name any of its own +accounts, meaning no App login, `ai_author` or bot login configured and `/user` +unreachable, falls back to reading every comment, as the other +hive-authorship checks do.) The cycle that re-runs a head's jobs sends +it no fix kick either: the reaper, which runs next on the same snapshot, +skips the PRs just re-run. The re-run gives the checks new run ids, so the markers go +stale on their own: a head still red afterwards is back in the ordinary loop +above. The re-run is recorded per head in the ledger; a person or the reviewer +lane returning an escalated PR to the lane allows one more. Before this, +projectbluefin/common#1647 spent all six kicks on one head whose agent had +answered each one with "infrastructure, not this change". + ### Exclude independently confirmed shared failures Shared CI breakage is not a fix attempt. A check is shared only when it fails diff --git a/src/docs/github-app-setup.md b/src/docs/github-app-setup.md index 57de9096cf..09c45cff24 100644 --- a/src/docs/github-app-setup.md +++ b/src/docs/github-app-setup.md @@ -113,6 +113,13 @@ retries no more than once per repository per permission cooldown (one hour). An unchanged PR remains eligible after the cooldown; a successful rerun or observing no failed runs clears the recovery block. +The transient-CI re-run (a red hive PR whose failing runs all carry +`hive-ci-transient`, see ADR-0010) uses the same `rerun-failed-jobs` call and +the same per-repository refusal backoff. Without Actions write it logs +`transient CI rerun forbidden` and posts no note, and the head goes back to +the ordinary fix loop. It is turned off together with the fix loop by +`escalation.disabled`. + A maintainer may re-run the named workflow with their already-authorized identity. That manual recovery does **not** change the App's permissions. For automatic recovery, the App owner must explicitly request Actions read and diff --git a/src/docs/github-write-surface.md b/src/docs/github-write-surface.md index 13af6c5f3d..f9ff0af45f 100644 --- a/src/docs/github-write-surface.md +++ b/src/docs/github-write-surface.md @@ -157,6 +157,7 @@ writes no entry. | Task-list sweep closes | `task_list_sweep.go` | yes (`task-list-sweep-closed`) | yes | | Supersession sweep (notice, `hive/superseded` close, own-PR update-branch + body `Closes`→`Refs` rewrite, `needs-human` + `Parked` fact on escalation) | `pr_supersession_sweep.go`, `pr_supersession_autoclose.go`, `pr_supersession_resolve.go` | yes (`supersession-sweep-*`) | yes | | Shared-CI rerun (`rerun-failed-jobs`, `hive-shared-ci-rerun-` comment) | `shared_ci_rerun.go` | yes (`shared-ci-rerun`) | yes | +| Transient-CI rerun (`rerun-failed-jobs` on a hive PR whose failing runs carry `hive-ci-transient`, once per head, no comment) | `transient_ci_rerun.go` | yes (`transient-ci-rerun`) | yes | | Open-PR dispatch guard (`has-open-pr` label on a refused issue) and duplicate-fix sweep (one `hive:duplicate-fix` comment; `hive/superseded` close of the later hive PR, or `needs-human` + `Parked` fact on escalation) | `open_pr_links.go`, `pr_duplicate_fix_sweep.go` | yes (`duplicate-fix-*`) | yes | | Duplicate-PR sweep suggestions | `duplicate_sweep.go` | yes (`duplicate-sweep-suggested`) | yes (target = surviving PR) | | Hold-label migration (one-time) | `hive_hold_migration.go` | yes (`hold_migration_label_added`, one per item labeled; the migration report file is still written) | yes | diff --git a/src/docs/labels-and-control-signals/glossary-and-operations.md b/src/docs/labels-and-control-signals/glossary-and-operations.md index 9c6c87dbe5..4c9f0bbfd3 100644 --- a/src/docs/labels-and-control-signals/glossary-and-operations.md +++ b/src/docs/labels-and-control-signals/glossary-and-operations.md @@ -233,7 +233,7 @@ The dashboard renders every open issue and PR's stage as one unified tracker str - **Human comments and assignees** can acknowledge hive-filed directions for #5117; assignees also affect ranking. - **Hidden comment markers** carry state that no label does. `` records an issue claim, `` dedupes a published finding, `` marks an overlap notice, `` starts the question auto-close clock, and `` is stamped on the single comment a fix lane leaves on a PR it defers to shared incident `#` (`DEFER_TO_INCIDENT` from `bin/hive-baseline-check.sh`), so "every PR incident `#` broke" stays greppable after the fix lands ([#10441](https://github.com/hivecommons/hive/issues/10441)). One marker per PR per incident; it is never edited or removed while the incident is open. While incident `#` is open, the governor records it as `deferred_incident` on the PR's `ci-failing.json` row and the kick builders keep that PR out of every CI-FAILING / FIX-BEFORE-NEW repair list, naming it only as deferred; once the incident closes the PR is listed for repair again ([#10528](https://github.com/hivecommons/hive/issues/10528)). When the incident closes (not as "not planned"), the shared-CI rerun sweep re-runs the failed jobs of each still-open PR carrying that marker — the newest failed or timed-out run per workflow on the PR head — using the hive's own credential, then stamps `` on the PR so it is re-run at most once per incident; disable with `shared_ci_rerun.enabled: false` ([#12253](https://github.com/hivecommons/hive/issues/12253)). - **``** is stamped by a fix lane on the one comment it leaves when `bin/hive-baseline-check.sh` returns `DEFER_EXTERNAL`: the repository syncs from an upstream project (its GitHub fork parent, or `--upstream`) and the same check is red on the upstream default branch, so no change here can fix it. The helper prints the exact string as `external_marker`. While every failing check run on the PR's current head carries a marker for that head and run, the governor records `deferred_external` on the PR's `ci-failing.json` row and the kick builders name the PR only as deferred under `DEFERRED AS FAILING UPSTREAM`; a push, a newly failing check or a re-run of the check (a new run id — e.g. it went green and failed again at the same head) makes the markers stale and the PR is listed for repair again ([#11609](https://github.com/hivecommons/hive/issues/11609)). -- **``** is stamped by an agent on its triage comment when it diagnoses a red check on a PR as transient or infrastructure-related (a flaky job, a runner outage, a race against a cache the default branch was still publishing) rather than caused by the diff, and cannot re-run the check itself. The fields and pinning match `hive-ci-external`; `bin/hive-baseline-check.sh --json` with the PR number prints the exact string as `transient_marker` whenever the PR's newest run of the check is red, on any verdict, and the agent copies it verbatim. While every failing check run on the PR's current head carries a marker for that head and run, the PR's `transient_ci_triaged` is set and the dependency-bot red-CI hand-off still adds `needs-human` after its grace period (nothing re-runs a single PR's check automatically), but its comment says Hive's triage found the failure was not caused by the bump and asks for a re-run, instead of telling the owner to fix, pin or close the bump. A push or a re-run of the check (a new run id) makes the marker stale ([#12977](https://github.com/hivecommons/hive/issues/12977)). +- **``** is stamped by an agent on its triage comment when it diagnoses a red check on a PR as transient or infrastructure-related (a flaky job, a runner outage, a race against a cache the default branch was still publishing) rather than caused by the diff, and cannot re-run the check itself. The fields and pinning match `hive-ci-external`; `bin/hive-baseline-check.sh --json` with the PR number prints the exact string as `transient_marker` whenever the PR's newest run of the check is red, on any verdict, and the agent copies it verbatim. While every failing check run on the PR's current head carries a marker for that head and run, the PR's `transient_ci_triaged` is set. On a hive-authored PR the fix loop then re-runs the failed jobs once for that head (`rerun-failed-jobs`, gated as `ci_rerun`) and sends no FIX-BEFORE-NEW kick while the re-run is owed; a head still red after it is back in the ordinary fix loop ([projectbluefin/common#1647](https://github.com/projectbluefin/common/pull/1647)). Only markers in the hive's own comments count here, so a comment from anyone else cannot trigger the re-run or hold the fix loop. A hive that cannot name any of its own accounts falls back to every comment. The dependency-bot red-CI hand-off still adds `needs-human` after its grace period (dependency-bot PRs are not re-run automatically), but its comment says Hive's triage found the failure was not caused by the bump and asks for a re-run, instead of telling the owner to fix, pin or close the bump. A push or a re-run of the check (a new run id) makes the marker stale ([#12977](https://github.com/hivecommons/hive/issues/12977)). - **Paused repos** (`project.paused_repos`) stop whole-repo write/merge/enumeration paths without labels. - **Contributor queue holds** such as active leases, cooldowns, dependencies, and quota guard holds suppress contributor offers without touching GitHub labels. - **Jira/Linear labels** map only where the work-source adapter supports them: Linear skips held work using substring hold labels; Jira skips exact configured hold labels; GitHub Projects currently imports labels but does not use a hold gate. diff --git a/src/docs/operator-reference/sweeps-and-operations.md b/src/docs/operator-reference/sweeps-and-operations.md index 281a1d45d5..f0ad490d8e 100644 --- a/src/docs/operator-reference/sweeps-and-operations.md +++ b/src/docs/operator-reference/sweeps-and-operations.md @@ -454,6 +454,7 @@ that level, the power the write-back exists to govern? | Welcome / outreach (`pkg/gfi`, `gfi_forge.go`, `pr_clanker_requested.go`) | good-first-issue selection | welcome comment, `good first issue` label | issues hive advertises | `welcome_comment` → L5 with label, L6 without | | Issue filing (`pkg/upstreamwatch/github_filer.go`, `advisory.go`, scanner/deps/regression agents, `dedupe_advisory_close.go`) | agents, advisory digest, upstream watch | **new** issues hive authors, pinned advisory issue + comments | `project.repos` | `issue_file` → L3, no label (this is what listing a repo opts in to; `per_repo.mode: issues-only` keeps only this) | | Shared-CI rerun (`shared_ci_rerun.go`) | CI failure watcher | `rerun-failed-jobs`, `hive-shared-ci-rerun-` comment | PRs with failed shared workflows | `ci_rerun` → L6 + label | +| Transient-CI rerun (`transient_ci_rerun.go`) | eval cycle, before the stuck-PR reaper | `rerun-failed-jobs`, once per head, no comment | hive-authored red PRs whose every failing run carries `hive-ci-transient` | `ci_rerun` (own items: allowed unless the repo is paused, off or issues-only) | | Level hold, self-authorization & reporter-trust releases (`pr_level_hold.go`, `automerge/self_authorization_release.go`, `reporter_trust_release.go`) | PR watcher | `hold` label add/remove | agent PRs | own items (hive-authored) — not gated | | Migrations & retirements (`hive_hold_migration.go`, `hive_id_label_retire.go`, `claimed_by_human_migration.go`, `origin_labels.go`) | boot, one-shot | label renames | items carrying hive labels | opted in by carrying hive labels | diff --git a/src/pkg/escalation/escalation.go b/src/pkg/escalation/escalation.go index 63ab2c6632..d3b5994340 100644 --- a/src/pkg/escalation/escalation.go +++ b/src/pkg/escalation/escalation.go @@ -199,6 +199,15 @@ type Entry struct { // RunWaitEscalatedGen records the run generation whose human wait timeout // already paged a human. It makes run-wait escalation once-per-generation. RunWaitEscalatedGen uint64 `json:"run_wait_escalated_gen,omitempty"` + + // TransientTriagedSHA is the red head on which every failing check run + // carries a `hive-ci-transient` triage marker (Observation. + // TransientTriaged), as of the last ObserveRed pass; "" otherwise. + // TransientRerunSHA is the head whose failed jobs the hive already + // re-ran because of such a triage. Together they make the re-run once + // per head and hold fix kicks while it is owed (transient_rerun.go). + TransientTriagedSHA string `json:"transient_triaged_sha,omitempty"` + TransientRerunSHA string `json:"transient_rerun_sha,omitempty"` } // Store is the on-PVC attempt ledger. All methods are safe for concurrent use. @@ -303,6 +312,12 @@ type Observation struct { // SharedCIIncident is the open [shared-ci] incident the PR is deferred // to, 0 when none; named in the hold comment. SharedCIIncident int + // TransientTriaged reports that every failing check run on this head + // carries an agent's `hive-ci-transient` marker for this head and run + // (github.PullRequest.TransientCITriaged): the failure was diagnosed as + // infrastructure, not this change. The fix loop re-runs the failed jobs + // once instead of kicking a fixer (transient_rerun.go). + TransientTriaged bool } // MarkSharedFailures classifies each red observation's failing checks as @@ -599,6 +614,9 @@ func (s *Store) Sweep(obs []Observation, threshold int) map[string]Result { e.RedSHAs = nil e.ReEngagements = 0 e.LastReEngagedAt = time.Time{} + // A person returned the PR to the lane: a fresh transient + // triage on the same head earns one more re-run. + e.TransientRerunSHA = "" e.UpdatedAt = s.now() } @@ -683,6 +701,7 @@ func (s *Store) Sweep(obs []Observation, threshold int) map[string]Result { e.RedSHAs = nil e.RedTrees = nil e.ReEngagements = 0 + e.TransientRerunSHA = "" // Remember WHAT the reviewer left on the branch and WHEN, so a // later re-escalation can hand the human the reviewer's context // instead of a bare label (#5617 item 3). Keyed on the SHA so a @@ -1088,6 +1107,7 @@ func (s *Store) ObserveRed(obs []Observation) { e.FirstRedAt = time.Time{} e.ReEngagements = 0 e.LastReEngagedAt = time.Time{} + e.TransientTriagedSHA = "" } continue } @@ -1113,6 +1133,10 @@ func (s *Store) ObserveRed(obs []Observation) { if o.Excerpt != "" { e.LastExcerpt = o.Excerpt } + e.TransientTriagedSHA = "" + if o.TransientTriaged { + e.TransientTriagedSHA = o.HeadSHA + } e.UpdatedAt = now } s.saveLocked() diff --git a/src/pkg/escalation/transient_rerun.go b/src/pkg/escalation/transient_rerun.go new file mode 100644 index 0000000000..e879f1158e --- /dev/null +++ b/src/pkg/escalation/transient_rerun.go @@ -0,0 +1,58 @@ +package escalation + +// Transient-triage re-run (projectbluefin/common#1647). +// +// When an agent diagnoses a red check as infrastructure, not the change (a +// runner's apt mirror dropping the connection, a lost runner), the repair is +// a re-run of the failed jobs, not a commit. Before this the fix loop kept +// kicking the owning agent: each kick was a delivered re-dispatch, the agent +// answered each one with the same "infrastructure, not this change" comment, +// and the sixth kick escalated the PR to a person. A lane then pushed an +// empty "retrigger CI" commit, which re-arms review fences and can dismiss +// approvals. +// +// The agent's triage reaches the hive as the `hive-ci-transient` marker +// (github.PullRequest.TransientCITriaged). ObserveRed records the head it +// covers; while a re-run of that head is owed, the fix-kick paths stand down +// (TransientRerunOwed) and the eval cycle re-runs the failed jobs once +// (RecordTransientRerun). The re-run gives the failing checks new run ids, so +// the markers go stale by themselves: a head that is red again afterwards is +// back in the ordinary fix loop, with its usual budget and escalation. + +// TransientRerunOwed reports that the PR's red head is transient-triaged and +// the hive has not yet re-run its failed jobs. headSHA "" means the head the +// ledger last saw red (the merge watcher's hook knows only the PR number). +// An escalated PR is a person's, so nothing is owed on it. +func (s *Store) TransientRerunOwed(repo string, number int, headSHA string) bool { + s.mu.Lock() + defer s.mu.Unlock() + e := s.entries[Key(repo, number)] + if e == nil || e.Escalated || e.ReporterTrustReason != "" { + return false + } + if headSHA == "" { + headSHA = e.CurRedSHA + } + return headSHA != "" && e.TransientTriagedSHA == headSHA && e.TransientRerunSHA != headSHA +} + +// RecordTransientRerun marks the head's one transient re-run as spent. The +// caller records it once the attempt is settled: re-run, refused by the +// write-back gate or by GitHub, or nothing left to re-run. An attempt that +// failed for a reason a later pass can clear (an API error) is not recorded. +func (s *Store) RecordTransientRerun(repo string, number int, headSHA string) { + if headSHA == "" { + return + } + s.mu.Lock() + defer s.mu.Unlock() + key := Key(repo, number) + e := s.entries[key] + if e == nil { + e = &Entry{Machinery: MachineryVersion} + s.entries[key] = e + } + e.TransientRerunSHA = headSHA + e.UpdatedAt = s.now() + s.saveLocked() +} diff --git a/src/pkg/escalation/transient_rerun_test.go b/src/pkg/escalation/transient_rerun_test.go new file mode 100644 index 0000000000..3a4790194a --- /dev/null +++ b/src/pkg/escalation/transient_rerun_test.go @@ -0,0 +1,69 @@ +package escalation + +import ( + "path/filepath" + "testing" + "time" +) + +func transientObs(head string, triaged bool, labels ...string) Observation { + return Observation{Repo: "o/r", Number: 1, HeadSHA: head, Red: true, FailingChecks: []string{"build"}, TransientTriaged: triaged, Labels: labels, Labeled: HasNeedsHumanLabel(labels)} +} + +func TestTransientRerunOwedOncePerHead(t *testing.T) { + s := Load(filepath.Join(t.TempDir(), "l.json")) + if s.TransientRerunOwed("o/r", 1, "a") { + t.Fatal("owed with no observation") + } + s.ObserveRed([]Observation{transientObs("a", false)}) + if s.TransientRerunOwed("o/r", 1, "a") { + t.Fatal("owed on an untriaged head") + } + s.ObserveRed([]Observation{transientObs("a", true)}) + if !s.TransientRerunOwed("o/r", 1, "a") || !s.TransientRerunOwed("o/r", 1, "") { + t.Fatal("triaged head must owe a re-run (by sha and by the ledger's current head)") + } + if s.TransientRerunOwed("o/r", 1, "b") { + t.Fatal("owed for a head the triage does not cover") + } + s.RecordTransientRerun("o/r", 1, "a") + if s.TransientRerunOwed("o/r", 1, "a") { + t.Fatal("still owed after the re-run") + } + // Survives a restart: the ledger is on the PVC. + s2 := Load(s.path) + s2.ObserveRed([]Observation{transientObs("a", true)}) + if s2.TransientRerunOwed("o/r", 1, "a") { + t.Fatal("a restart re-armed the spent re-run") + } + // A new head with its own triage owes its own re-run. + s2.ObserveRed([]Observation{transientObs("b", true)}) + if !s2.TransientRerunOwed("o/r", 1, "b") { + t.Fatal("new triaged head owes no re-run") + } + // Markers gone stale (the re-run made new run ids): nothing owed. + s2.ObserveRed([]Observation{transientObs("b", false)}) + if s2.TransientRerunOwed("o/r", 1, "b") { + t.Fatal("owed after the triage went stale") + } +} + +// A person (or the reviewer lane) returning an escalated PR to the lane +// gives a still-triaged head one more re-run. +func TestTransientRerunReArmedWhenUnparked(t *testing.T) { + s := Load(filepath.Join(t.TempDir(), "l.json")) + now := time.Date(2026, 11, 6, 21, 0, 0, 0, time.UTC) + s.SetClock(func() time.Time { return now }) + s.ObserveRed([]Observation{transientObs("a", true)}) + s.RecordTransientRerun("o/r", 1, "a") + s.Sweep([]Observation{transientObs("a", true, NeedsHumanLabel)}, DefaultThreshold) + if s.TransientRerunOwed("o/r", 1, "a") { + t.Fatal("an escalated PR must owe nothing") + } + now = now.Add(LabelUnparkGrace + time.Minute) + s.ObserveRed([]Observation{transientObs("a", true, ReviewerPassedLabel)}) + s.Sweep([]Observation{transientObs("a", true, ReviewerPassedLabel)}, DefaultThreshold) + if !s.TransientRerunOwed("o/r", 1, "a") { + t.Fatal("un-parked PR with a fresh transient triage must owe one more re-run") + } +} diff --git a/src/pkg/github/client.go b/src/pkg/github/client.go index 55f6d0c0f2..9dc362ec8b 100644 --- a/src/pkg/github/client.go +++ b/src/pkg/github/client.go @@ -2748,10 +2748,10 @@ func (c *Client) EnrichCIStatus(ctx context.Context, prs []PullRequest) { prs[i].CIActivity.WorkflowApproval = c.workflowApprovalWithLedger(prs[i].Repo, prs[i].Number, prs[i].HeadSHA, c.workflowRunsAwaitingApproval(ctx, prs[i].Repo, prs[i].HeadSHA)) } if prs[i].CIStatus == "failure" { - if texts, ok := c.redPRCommentTexts(ctx, &prs[i]); ok { + if texts, hiveTexts, ok := c.redPRCommentTexts(ctx, &prs[i]); ok { prs[i].SharedCIIncident = c.openSharedCIIncident(ctx, &prs[i], texts, incidents) prs[i].ExternalCIDeferred = externalCIDeferred(texts, prs[i].HeadSHA, prs[i].FailingCheckRunIDs) - prs[i].TransientCITriaged = transientCITriaged(texts, prs[i].HeadSHA, prs[i].FailingCheckRunIDs) + prs[i].TransientCITriaged = transientCITriaged(hiveTexts, prs[i].HeadSHA, prs[i].FailingCheckRunIDs) } } facts.attach(ctx, &prs[i], reported) diff --git a/src/pkg/github/shared_ci_incident.go b/src/pkg/github/shared_ci_incident.go index 4fe20f801e..e48b835419 100644 --- a/src/pkg/github/shared_ci_incident.go +++ b/src/pkg/github/shared_ci_incident.go @@ -37,19 +37,30 @@ func newestSharedCIMarker(texts []string) int { // incident marker (hivecommons/hive#10528) and the external-CI marker // (hivecommons/hive#11609). One listing serves both. ok is false when the // comments cannot be read; the PR then stays listed. -func (c *Client) redPRCommentTexts(ctx context.Context, pr *PullRequest) (texts []string, ok bool) { +// +// hiveTexts are the bodies the hive's own accounts wrote. The transient-CI +// marker is read from those alone: it re-runs the failed jobs and holds the +// fix loop until they finish, so a comment from anyone else must not be able +// to spend Actions minutes or pause repairs (hivecommons/hive#17002 review). +// A hive that cannot name its own accounts falls back to every body, as the +// other hive-authorship checks do. +func (c *Client) redPRCommentTexts(ctx context.Context, pr *PullRequest) (texts, hiveTexts []string, ok bool) { owner, repo := c.splitRepo(pr.Repo) ctx = WithRESTCaller(ctx, "hive:shared_ci_incident") comments, err := c.listIssueComments(ctx, owner, repo, pr.Number) if err != nil { c.logger.Debug("CI deferral marker lookup failed; PR stays in the repair list", "repo", pr.Repo, "pr", pr.Number, "error", err) - return nil, false + return nil, nil, false } texts = make([]string, 0, len(comments)) + ownOnly := c.knowsOwnAccounts() for _, comment := range comments { texts = append(texts, comment.GetBody()) + if !ownOnly || c.isHiveAccount(comment.GetUser()) { + hiveTexts = append(hiveTexts, comment.GetBody()) + } } - return texts, true + return texts, hiveTexts, true } // sharedCIRerunAwaitGrace is how long after a [shared-ci] incident closes a diff --git a/src/pkg/github/transient_ci_marker_author_test.go b/src/pkg/github/transient_ci_marker_author_test.go new file mode 100644 index 0000000000..ae745b3559 --- /dev/null +++ b/src/pkg/github/transient_ci_marker_author_test.go @@ -0,0 +1,72 @@ +package github + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "regexp" + "strconv" + "strings" + "testing" +) + +// The transient-CI marker now re-runs the failed jobs and holds the fix loop +// (hivecommons/hive#17002), so only the hive's own comments may set it: a +// marker anyone else posts on a public PR must not spend Actions minutes or +// pause repairs. The external-CI marker keeps its old any-author reading. +func TestEnrichCIStatus_TransientCITriagedOnlyFromHiveComments(t *testing.T) { + head := strings.Repeat("a", 40) + marker := "Infra flake.\n" + type comment struct{ login, body string } + comments := map[int][]comment{ + 1: {{"hivecommons-hive[bot]", marker}}, + 2: {{"mallory", marker}}, + 3: {{"mallory", externalMarker(head, "build", 1)}}, + } + commentsRE := regexp.MustCompile(`^/repos/org/repo/issues/(\d+)/comments$`) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if m := commentsRE.FindStringSubmatch(r.URL.Path); m != nil { + n, _ := strconv.Atoi(m[1]) + out := []map[string]any{} + for _, cm := range comments[n] { + out = append(out, map[string]any{"body": cm.body, "user": map[string]any{"login": cm.login}}) + } + _ = json.NewEncoder(w).Encode(out) + return + } + if regexp.MustCompile(`/commits/[^/]+/check-runs$`).MatchString(r.URL.Path) { + _ = json.NewEncoder(w).Encode(map[string]any{ + "total_count": 1, + "check_runs": []map[string]any{{"id": 1, "name": "build", "status": "completed", "conclusion": "failure"}}, + }) + return + } + w.WriteHeader(http.StatusNotFound) + })) + t.Cleanup(srv.Close) + + c := newTestClient(t, srv, "org", []string{"repo"}) + c.SetAppBotLogin("hivecommons-hive[bot]") + prs := []PullRequest{ + {Repo: "org/repo", Number: 1, HeadSHA: head}, + {Repo: "org/repo", Number: 2, HeadSHA: head}, + {Repo: "org/repo", Number: 3, HeadSHA: head}, + } + c.EnrichCIStatus(context.Background(), prs) + + for _, pr := range prs { + if pr.CIStatus != "failure" { + t.Fatalf("PR %d CIStatus = %q, want failure", pr.Number, pr.CIStatus) + } + } + if !prs[0].TransientCITriaged { + t.Error("the hive's own transient marker did not triage the head") + } + if prs[1].TransientCITriaged { + t.Error("a transient marker from someone outside the hive triaged the head (it would trigger a re-run)") + } + if !prs[2].ExternalCIDeferred { + t.Error("the external-CI marker must keep reading every author's comments") + } +} diff --git a/src/pkg/github/transient_ci_rerun.go b/src/pkg/github/transient_ci_rerun.go new file mode 100644 index 0000000000..5598941d62 --- /dev/null +++ b/src/pkg/github/transient_ci_rerun.go @@ -0,0 +1,78 @@ +package github + +import ( + "context" + "fmt" + + "github.com/hivecommons/hive/pkg/writeback" +) + +// TransientCIRerunOutcome is what RerunTransientCIFailures did for one PR. +type TransientCIRerunOutcome struct { + // Settled means the head's one transient re-run is spent: jobs were + // re-run, the write-back gate or GitHub refused, or nothing failed is + // left to re-run. Unsettled (with an error) means a later pass may retry. + Settled bool + // RunIDs are the workflow runs whose failed jobs GitHub accepted for a + // re-run. + RunIDs []int64 + // Skipped names why nothing was re-run ("" when RunIDs is non-empty): + // a write-back gate reason, "rerun-forbidden" or "no-failed-runs". + Skipped string +} + +// RerunTransientCIFailures re-runs the failed jobs of every failed workflow +// run on pr's head, for a red PR an agent triaged as transient +// (PullRequest.TransientCITriaged). It is the deterministic repair the fix +// loop uses instead of kicking a fixer that can only comment, or pushing a +// "retrigger CI" commit that re-arms fences and can dismiss approvals +// (projectbluefin/common#1647). It never pushes. The re-run is an +// unsolicited write, so it consults the write-back gate (KindCIRerun) first; +// a hive-authored PR passes unless the repository is paused, off or +// issues-only. Once any re-run request reached GitHub the attempt is settled +// whatever came back, so a head gets at most one round of re-run calls. +func (c *Client) RerunTransientCIFailures(ctx context.Context, pr PullRequest) (TransientCIRerunOutcome, error) { + if c == nil || c.client == nil { + return TransientCIRerunOutcome{}, ErrNoGitHubClient + } + owner, name := c.splitRepo(pr.Repo) + repo := owner + "/" + name + preview := fmt.Sprintf("re-run the failed jobs on %s#%d at %s (transient CI triage)", repo, pr.Number, shortSHA(pr.HeadSHA)) + if d := c.WritebackAllowed(repo, c.WritebackItemForPR(pr), writeback.KindCIRerun, preview); !d.Allowed { + return TransientCIRerunOutcome{Settled: true, Skipped: d.Reason}, nil + } + if c.sharedCIRerunRefused(repo) { + return TransientCIRerunOutcome{Settled: true, Skipped: "rerun-forbidden"}, nil + } + ctx = WithRESTCaller(ctx, "hive:transient_ci_rerun") + runs, err := c.latestFailedWorkflowRuns(ctx, owner, name, pr.HeadSHA) + if err != nil { + return TransientCIRerunOutcome{}, err + } + if len(runs) == 0 { + // Failed on a check no Actions run backs (an external CI), or it was + // already re-run by hand: nothing the hive can re-run. + return TransientCIRerunOutcome{Settled: true, Skipped: "no-failed-runs"}, nil + } + out := TransientCIRerunOutcome{Settled: true} + for _, run := range runs { + if _, err := c.client.Actions.RerunFailedJobsByID(ctx, owner, name, run.GetID()); err != nil { // writeback:gated — WritebackAllowed(KindCIRerun) consulted above + if isWriteForbiddenError(err) { + if c.markSharedCIRerunForbidden(repo) { + c.logger.Warn("transient CI rerun forbidden: the App installation lacks Actions write permission for rerun-failed-jobs", "repo", repo, "pr", pr.Number, "run", run.GetID(), "error", err) + } + out.Skipped = "rerun-forbidden" + break + } + c.logger.Warn("transient CI rerun failed", "repo", repo, "pr", pr.Number, "run", run.GetID(), "error", err) + continue + } + out.RunIDs = append(out.RunIDs, run.GetID()) + } + if len(out.RunIDs) > 0 { + out.Skipped = "" + } else if out.Skipped == "" { + out.Skipped = "rerun-failed" + } + return out, nil +} diff --git a/src/pkg/github/transient_ci_rerun_test.go b/src/pkg/github/transient_ci_rerun_test.go new file mode 100644 index 0000000000..b9ad3dd977 --- /dev/null +++ b/src/pkg/github/transient_ci_rerun_test.go @@ -0,0 +1,88 @@ +package github + +import ( + "context" + "reflect" + "testing" + + "github.com/hivecommons/hive/pkg/writeback" +) + +// A transient-triaged red PR gets its failed jobs re-run through +// rerun-failed-jobs, never a push (projectbluefin/common#1647), and the +// re-run is an unsolicited write the write-back gate can refuse. +func TestRerunTransientCIFailures(t *testing.T) { + failedPR := func() []*sharedCIRerunPR { + return []*sharedCIRerunPR{{Number: 1, SHA: "aaaaaaaaaa", Runs: []sharedCIRerunRun{ + failedRun(11, 1), + {ID: 12, WorkflowID: 2, RunNumber: 1, Attempt: 1, Status: "completed", Conclusion: "success"}, + }}} + } + pr := PullRequest{Repo: "org/repo", Number: 1, HeadSHA: "aaaaaaaaaa", AppAuthored: true, CIStatus: "failure", TransientCITriaged: true} + + t.Run("re-runs the failed workflow runs on the head", func(t *testing.T) { + srv, fake := newSharedCIRerunFake(t, nil, failedPR()) + c := newTestClient(t, srv, "org", []string{"org/repo"}) + c.SetWritebackGate(writeback.New(writeback.Config{}, writeback.Deps{}, "")) + got, err := c.RerunTransientCIFailures(context.Background(), pr) + if err != nil { + t.Fatalf("rerun: %v", err) + } + reruns, posted, _ := fake.snap() + if !reflect.DeepEqual(reruns, []int64{11}) { + t.Fatalf("reruns = %v, want [11] (the failed run only)", reruns) + } + if !got.Settled || !reflect.DeepEqual(got.RunIDs, []int64{11}) || got.Skipped != "" { + t.Fatalf("outcome = %+v, want settled with run 11", got) + } + if len(posted) != 0 { + t.Fatalf("a transient re-run must not comment, posted %v", posted) + } + }) + + t.Run("write-back gate refusal makes no GitHub write", func(t *testing.T) { + srv, fake := newSharedCIRerunFake(t, nil, failedPR()) + c := newTestClient(t, srv, "org", []string{"org/repo"}) + c.SetWritebackGate(writeback.New(writeback.Config{PerRepo: map[string]writeback.RepoOverride{ + "org/repo": {Mode: writeback.RepoModeOff}, + }}, writeback.Deps{}, "")) + got, err := c.RerunTransientCIFailures(context.Background(), pr) + if err != nil { + t.Fatalf("rerun: %v", err) + } + if reruns, _, _ := fake.snap(); len(reruns) != 0 { + t.Fatalf("gate refused, but reran %v", reruns) + } + if !got.Settled || len(got.RunIDs) != 0 || got.Skipped != writeback.ReasonRepoOff { + t.Fatalf("outcome = %+v, want settled and skipped for %q", got, writeback.ReasonRepoOff) + } + }) + + t.Run("forbidden re-run settles the attempt", func(t *testing.T) { + srv, fake := newSharedCIRerunFake(t, nil, failedPR()) + fake.forbidRerun = true + c := newTestClient(t, srv, "org", []string{"org/repo"}) + got, err := c.RerunTransientCIFailures(context.Background(), pr) + if err != nil { + t.Fatalf("rerun: %v", err) + } + if !got.Settled || len(got.RunIDs) != 0 || got.Skipped != "rerun-forbidden" { + t.Fatalf("outcome = %+v, want settled rerun-forbidden", got) + } + }) + + t.Run("nothing failed to re-run", func(t *testing.T) { + srv, fake := newSharedCIRerunFake(t, nil, []*sharedCIRerunPR{{Number: 1, SHA: "aaaaaaaaaa"}}) + c := newTestClient(t, srv, "org", []string{"org/repo"}) + got, err := c.RerunTransientCIFailures(context.Background(), pr) + if err != nil { + t.Fatalf("rerun: %v", err) + } + if reruns, _, _ := fake.snap(); len(reruns) != 0 { + t.Fatalf("reran %v with no failed runs", reruns) + } + if !got.Settled || got.Skipped != "no-failed-runs" { + t.Fatalf("outcome = %+v, want settled no-failed-runs", got) + } + }) +} diff --git a/src/pkg/policies/defaults/reviewer-lane.md b/src/pkg/policies/defaults/reviewer-lane.md index 3cc904b82a..70d61338d4 100644 --- a/src/pkg/policies/defaults/reviewer-lane.md +++ b/src/pkg/policies/defaults/reviewer-lane.md @@ -16,8 +16,14 @@ ADJUDICATION CONTRACT — for EACH PR above, deliver EXACTLY ONE verdict: d. After completing the mandatory audit below, return it to the automated lane and mark your pass: gh pr edit --remove-label needs-human --add-label ${REVIEWER_PASSED_LABEL} 2. DE-ESCALATE — the failure was environmental (base-branch regression since - fixed, infra flake): rebase the branch on its base, push, complete the - mandatory audit below, then + fixed, infra flake). If the branch is behind its base, rebase it on the + base and push. If it is current, push NOTHING: no empty, no-op or + "retrigger CI" commit (a push re-arms review fences and can dismiss + approvals). Re-run the failed jobs with gh run rerun --failed, + or, if you cannot, leave one PR comment that ends with the transient_marker + hive-baseline-check.sh '' --json prints for + each failing check, so Hive re-runs them. Then complete the mandatory + audit below, then gh pr edit --remove-label needs-human --add-label ${REVIEWER_PASSED_LABEL} 3. RECOMMEND-CLOSE — duplicate, superseded, or irreparably lossy: complete the mandatory audit below using its recommend-close review body, then mark the diff --git a/src/pkg/scheduler/reviewer_lane.go b/src/pkg/scheduler/reviewer_lane.go index a3e940c4c5..926db252e0 100644 --- a/src/pkg/scheduler/reviewer_lane.go +++ b/src/pkg/scheduler/reviewer_lane.go @@ -312,8 +312,14 @@ func (s *Scheduler) buildReviewerMessageHardcoded(agentName string, level int, w b.WriteString(fmt.Sprintf(" d. After completing the mandatory audit below, return it to the automated lane and mark your pass:\n"+ " gh pr edit --remove-label needs-human --add-label %s\n", ReviewerPassedLabel)) b.WriteString(" 2. DE-ESCALATE — the failure was environmental (base-branch regression since\n") - b.WriteString(" fixed, infra flake): rebase the branch on its base, push, complete the\n") - b.WriteString(" mandatory audit below, then\n") + b.WriteString(" fixed, infra flake). If the branch is behind its base, rebase it on the\n") + b.WriteString(" base and push. If it is current, push NOTHING: no empty, no-op or\n") + b.WriteString(" \"retrigger CI\" commit (a push re-arms review fences and can dismiss\n") + b.WriteString(" approvals). Re-run the failed jobs with gh run rerun --failed,\n") + b.WriteString(" or, if you cannot, leave one PR comment that ends with the transient_marker\n") + b.WriteString(" hive-baseline-check.sh '' --json prints for\n") + b.WriteString(" each failing check, so Hive re-runs them. Then complete the mandatory\n") + b.WriteString(" audit below, then\n") b.WriteString(fmt.Sprintf(" gh pr edit --remove-label needs-human --add-label %s\n", ReviewerPassedLabel)) b.WriteString(" 3. RECOMMEND-CLOSE — duplicate, superseded, or irreparably lossy: complete the\n") b.WriteString(" mandatory audit below using its recommend-close review body, then mark the\n") diff --git a/src/policies/reviewer-lane.md b/src/policies/reviewer-lane.md index 3cc904b82a..70d61338d4 100644 --- a/src/policies/reviewer-lane.md +++ b/src/policies/reviewer-lane.md @@ -16,8 +16,14 @@ ADJUDICATION CONTRACT — for EACH PR above, deliver EXACTLY ONE verdict: d. After completing the mandatory audit below, return it to the automated lane and mark your pass: gh pr edit --remove-label needs-human --add-label ${REVIEWER_PASSED_LABEL} 2. DE-ESCALATE — the failure was environmental (base-branch regression since - fixed, infra flake): rebase the branch on its base, push, complete the - mandatory audit below, then + fixed, infra flake). If the branch is behind its base, rebase it on the + base and push. If it is current, push NOTHING: no empty, no-op or + "retrigger CI" commit (a push re-arms review fences and can dismiss + approvals). Re-run the failed jobs with gh run rerun --failed, + or, if you cannot, leave one PR comment that ends with the transient_marker + hive-baseline-check.sh '' --json prints for + each failing check, so Hive re-runs them. Then complete the mandatory + audit below, then gh pr edit --remove-label needs-human --add-label ${REVIEWER_PASSED_LABEL} 3. RECOMMEND-CLOSE — duplicate, superseded, or irreparably lossy: complete the mandatory audit below using its recommend-close review body, then mark the