From 872eb4886226efb8216d070338e9633b54435422 Mon Sep 17 00:00:00 2001 From: Danathar Date: Sat, 7 Nov 2026 10:23:02 -0500 Subject: [PATCH 1/4] fix(escalation): re-run a transient-triaged red PR instead of re-dispatching fixers On projectbluefin/common#1647 the owning agent answered every fix kick with "infrastructure, not this change" (apt mirror dropped the connection in Setup runner), but only in prose. The fix loop counted six delivered re-dispatches on one head and escalated the PR, and the reviewer lane then pushed an empty "retrigger CI" commit because its DE-ESCALATE verdict said "infra flake: rebase, push". When every failing check run on a hive-authored PR's head carries a hive-ci-transient marker, the eval cycle now re-runs the failed jobs once for that head (rerun-failed-jobs, gated as writeback ci_rerun), and deliverReEngagement holds the reaper's and the merge watcher's fix kicks while that re-run is owed. The re-run is recorded per head in the escalation ledger; a head still red after it returns to the ordinary fix loop. The fix kick now tells the agent to stamp the marker and push nothing, and DE-ESCALATE pushes only when the branch is behind its base. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BQYnPGFJx81c8JXR3o1sg1 Signed-off-by: Danathar --- changelog.d/fixed-transient-ci-rerun.md | 1 + src/cmd/hive/main_eval_cycle.go | 14 ++ src/cmd/hive/main_sweeps.go | 25 ++- src/cmd/hive/transient_ci_rerun.go | 95 ++++++++++ src/cmd/hive/transient_ci_rerun_test.go | 179 ++++++++++++++++++ .../adr/0010-escalation-circuit-breaker.md | 14 ++ src/docs/github-write-surface.md | 1 + .../glossary-and-operations.md | 2 +- .../sweeps-and-operations.md | 1 + src/pkg/escalation/escalation.go | 24 +++ src/pkg/escalation/transient_rerun.go | 58 ++++++ src/pkg/escalation/transient_rerun_test.go | 69 +++++++ src/pkg/github/transient_ci_rerun.go | 78 ++++++++ src/pkg/github/transient_ci_rerun_test.go | 88 +++++++++ src/pkg/policies/defaults/reviewer-lane.md | 10 +- src/pkg/scheduler/reviewer_lane.go | 10 +- src/policies/reviewer-lane.md | 10 +- 17 files changed, 670 insertions(+), 9 deletions(-) create mode 100644 changelog.d/fixed-transient-ci-rerun.md create mode 100644 src/cmd/hive/transient_ci_rerun.go create mode 100644 src/cmd/hive/transient_ci_rerun_test.go create mode 100644 src/pkg/escalation/transient_rerun.go create mode 100644 src/pkg/escalation/transient_rerun_test.go create mode 100644 src/pkg/github/transient_ci_rerun.go create mode 100644 src/pkg/github/transient_ci_rerun_test.go diff --git a/changelog.d/fixed-transient-ci-rerun.md b/changelog.d/fixed-transient-ci-rerun.md new file mode 100644 index 0000000000..cbce1dc8c2 --- /dev/null +++ b/changelog.d/fixed-transient-ci-rerun.md @@ -0,0 +1 @@ +- A red hive pull request whose failure an agent triaged as infrastructure is now re-run instead of sent back to a fixer. On [projectbluefin/common#1647](https://github.com/projectbluefin/common/pull/1647) the agent said "infrastructure, not this change" four times, the fix loop still counted six fix re-dispatches on that one commit and escalated the PR to a person, and a lane then pushed an empty "retrigger CI" commit. When every failing check run on the head carries a `hive-ci-transient` marker, Hive now re-runs the failed jobs once for that head (`rerun-failed-jobs`, through the write-back gate as `ci_rerun`) and sends no fix kick while that re-run is owed. A head still red after the re-run goes back to the normal fix loop. The fix kick now tells the agent to stamp the marker and never push to retrigger CI, and the reviewer lane's DE-ESCALATE verdict no longer asks for a push when the branch is already current. diff --git a/src/cmd/hive/main_eval_cycle.go b/src/cmd/hive/main_eval_cycle.go index 1ce7fa5be5..e40c9d65e9 100644 --- a/src/cmd/hive/main_eval_cycle.go +++ b/src/cmd/hive/main_eval_cycle.go @@ -791,6 +791,20 @@ func runEvalCycle( // re-engagement budget only for kicks that were accepted. Composes with the // merge watcher (#2): both go through the same per-red-SHA cap, so the same // PR is never double-dispatched within a red-SHA's budget. + // + // A head an agent triaged as transient infrastructure (hive-ci-transient + // markers) is re-run once here instead: the reaper and the merge watcher + // hold their fix kicks while that re-run is owed + // (projectbluefin/common#1647). + if ghClient != nil { + var audit func(string, int, string) + if dashSrv != nil { + audit = func(repo string, number int, detail string) { + dashSrv.AuditLogRecord("system", "transient-ci-rerun", detail, "", repo, number) + } + } + rerunTransientTriagedPRs(ctx, cfg, ghClient, actionable, escalatedPRs, audit, logger) + } reapStuckRedPRs(cfg, actionable, escalatedPRs, agentKicker{mgr: agentMgr}, agentAvailability(cfg, agentMgr), logger) diff --git a/src/cmd/hive/main_sweeps.go b/src/cmd/hive/main_sweeps.go index 6cda3319e7..eb9a9c9eb7 100644 --- a/src/cmd/hive/main_sweeps.go +++ b/src/cmd/hive/main_sweeps.go @@ -123,6 +123,7 @@ func escalationObservation(cfg *config.Config, pr github.PullRequest, owners map FailingChecks: pr.FailingChecks, Agent: owners[escalation.Key(repo, pr.Number)], IndependentControl: isIndependentCIControl(cfg, pr), + TransientTriaged: red && pr.TransientCITriaged, } } @@ -373,8 +374,17 @@ func redPRFixKick(o escalation.Observation, rerouted bool) string { fmt.Fprintf(&b, " gh pr checkout %d --repo %s → fix → git commit -s → git push\n", o.Number, o.Repo) b.WriteString("Push to the SAME branch. Do NOT open a replacement PR.\n") b.WriteString("After the push, STOP: do not watch, poll, or sleep on CI (no gh run watch/view loops).\n") - b.WriteString("The automerge sweep merges the PR once it is green; if the failure is infrastructure\n") - b.WriteString("(runner lost, no space left, checks still queued) leave one comment and end the turn.\n") + b.WriteString("The automerge sweep merges the PR once it is green; if checks are still queued, end the turn.\n") + // An infra verdict must reach the hive as a marker, not prose: four + // "infrastructure, not this change" comments on projectbluefin/common#1647 + // were invisible to the fix loop, which kicked again until the PR + // escalated, and a lane then pushed an empty "retrigger CI" commit. + b.WriteString("If the failure is infrastructure, not this change (runner lost, no space left, a package\n") + b.WriteString("mirror or download dropping the connection): do NOT push anything — no empty, no-op or\n") + b.WriteString("\"retrigger CI\" commit; a push re-arms review fences and can dismiss approvals. Leave ONE\n") + fmt.Fprintf(&b, "comment with the evidence that ends with the transient_marker printed by\n hive-baseline-check.sh %s '' %d --json\n", o.Repo, o.Number) + b.WriteString("(one per failing check), then end the turn. Once every failing check carries one, Hive\n") + b.WriteString("re-runs the failed jobs itself, once.\n") if o.HeadSHA != "" { fmt.Fprintf(&b, "\nhead: %s\n", o.HeadSHA) } @@ -449,6 +459,17 @@ func deliverReEngagement( available func(string) bool, logger *slog.Logger, ) bool { + if store.TransientRerunOwed(o.Repo, o.Number, headSHA) { + // An agent already triaged this head's failure as infrastructure; + // a fix kick could only produce the same comment again and spend + // the budget that escalates the PR (projectbluefin/common#1647). + // rerunTransientTriagedPRs re-runs the failed jobs instead. + if logger != nil { + logger.Debug("re-engagement held: failure triaged as transient, hive re-runs the failed jobs", + "repo", o.Repo, "pr", o.Number) + } + return false + } fixer := resolveRedPRFixer(cfg, o.Agent, available) if fixer == "" { // Nobody can be asked right now. Charging budget here is what escalated diff --git a/src/cmd/hive/transient_ci_rerun.go b/src/cmd/hive/transient_ci_rerun.go new file mode 100644 index 0000000000..4495baffc9 --- /dev/null +++ b/src/cmd/hive/transient_ci_rerun.go @@ -0,0 +1,95 @@ +package main + +import ( + "context" + "fmt" + "log/slog" + + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/escalation" + "github.com/hivecommons/hive/pkg/github" +) + +// transientCIRerunMaxPerPass bounds the re-run calls one eval cycle makes. +const transientCIRerunMaxPerPass = 10 + +// transientCIRerunner re-runs a transient-triaged PR's failed jobs; +// *github.Client implements it. +type transientCIRerunner interface { + RerunTransientCIFailures(ctx context.Context, pr github.PullRequest) (github.TransientCIRerunOutcome, error) +} + +// rerunTransientTriagedPRs is the fix loop's answer to a red hive-authored PR +// whose every failing check run an agent triaged as infrastructure, not this +// change (`hive-ci-transient` markers, github.PullRequest.TransientCITriaged): +// it re-runs the failed jobs once per head through rerun-failed-jobs, and +// never kicks a fixer or pushes a commit for it. On +// projectbluefin/common#1647 the fix loop instead kicked the owning agent six +// times on one head, each kick answered with "infrastructure, not this +// change", escalated the PR to a person, and a lane then pushed an empty +// "retrigger CI" commit. +// +// It runs after recordRedStaleness (which records the triaged head in the +// ledger) and before reapStuckRedPRs, whose kicks deliverReEngagement holds +// while a re-run is owed. audit, when non-nil, receives one line per PR the +// hive re-ran. +func rerunTransientTriagedPRs( + ctx context.Context, + cfg *config.Config, + rerunner transientCIRerunner, + actionable *github.ActionableResult, + escalatedPRs map[string]bool, + audit func(repo string, number int, detail string), + logger *slog.Logger, +) int { + if cfg == nil || cfg.Escalation.Disabled || rerunner == nil || actionable == nil { + return 0 + } + store := getEscalationStore() + attempted := 0 + for _, pr := range actionable.PRs.Items { + if attempted >= transientCIRerunMaxPerPass { + break + } + if !isHiveAgentAuthor(cfg, pr.Author) || !pr.TransientCITriaged || pr.CIChecksRunning || !pr.HasFailingRequiredCheck() { + continue + } + o := escalationObservation(cfg, pr, nil) + key := escalation.Key(o.Repo, o.Number) + if escalatedPRs[key] || escalation.HasNeedsHumanLabel(pr.Labels) { + continue + } + if !store.TransientRerunOwed(o.Repo, o.Number, o.HeadSHA) { + continue + } + attempted++ + outcome, err := rerunner.RerunTransientCIFailures(ctx, pr) + if err != nil { + if logger != nil { + logger.Warn("transient CI rerun: could not list failed runs; retrying next pass", + "repo", o.Repo, "pr", o.Number, "head", o.HeadSHA, "error", err) + } + continue + } + if !outcome.Settled { + continue + } + store.RecordTransientRerun(o.Repo, o.Number, o.HeadSHA) + if len(outcome.RunIDs) == 0 { + if logger != nil { + logger.Info("transient CI rerun skipped; the fix loop resumes on this head", + "repo", o.Repo, "pr", o.Number, "head", o.HeadSHA, "reason", outcome.Skipped) + } + continue + } + if logger != nil { + logger.Info("transient CI rerun: re-ran failed jobs instead of a fix kick", + "repo", o.Repo, "pr", o.Number, "head", o.HeadSHA, "runs", outcome.RunIDs) + } + if audit != nil { + audit(o.Repo, o.Number, fmt.Sprintf("repo=%s, pr=%d, head=%s, runs=%d, reason=transient CI triage", + o.Repo, o.Number, o.HeadSHA, len(outcome.RunIDs))) + } + } + return attempted +} diff --git a/src/cmd/hive/transient_ci_rerun_test.go b/src/cmd/hive/transient_ci_rerun_test.go new file mode 100644 index 0000000000..97f46c5473 --- /dev/null +++ b/src/cmd/hive/transient_ci_rerun_test.go @@ -0,0 +1,179 @@ +package main + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/escalation" + "github.com/hivecommons/hive/pkg/github" +) + +type fakeTransientRerunner struct { + calls []github.PullRequest + out github.TransientCIRerunOutcome + err error +} + +func (f *fakeTransientRerunner) RerunTransientCIFailures(_ context.Context, pr github.PullRequest) (github.TransientCIRerunOutcome, error) { + f.calls = append(f.calls, pr) + return f.out, f.err +} + +func transientRedPR() github.PullRequest { + pr := redPR("widgets", 7, "hive-agent", "abc") + pr.TransientCITriaged = true + return pr +} + +// projectbluefin/common#1647: an agent said "infrastructure, not this change" +// four times, yet the fix loop delivered six fix re-dispatches on the same +// head and escalated the PR. A head whose every failing run carries a +// hive-ci-transient marker must draw no fix kick and spend no budget, from +// the reaper or the merge watcher's hook. +func TestTransientTriagedHeadDrawsNoFixKick(t *testing.T) { + store, clock := newTestEscalationStore(t) + cfg := escalationTestConfig() + actionable := actionableWith(transientRedPR()) + recordRedStaleness(cfg, actionable) + *clock = clock.Add(escalation.RedPRStaleAfter + time.Minute) + kicker := &fakeKicker{} + + for i := 0; i < escalation.MaxReEngagements+2; i++ { + reapStuckRedPRs(cfg, actionable, map[string]bool{}, kicker, nil, discardLogger()) + *clock = clock.Add(escalation.MaxReEngageCooldown + time.Minute) + } + if hook := mergeReEngageHook(cfg, kicker, nil, discardLogger()); hook("widgets", 7) { + t.Error("merge watcher re-engaged a transient-triaged head") + } + if len(kicker.kicks) != 0 { + t.Fatalf("transient-triaged head got %d fix kicks, want 0", len(kicker.kicks)) + } + if got := store.ReEngagements("acme/widgets", 7); got != 0 { + t.Fatalf("transient-triaged head spent %d re-engagements, want 0", got) + } +} + +// The hive re-runs the failed jobs once per head; a head still red after +// that re-run (new run ids, so the markers are stale) is back in the +// ordinary fix loop. +func TestRerunTransientTriagedPRs_OncePerHeadThenFixLoopResumes(t *testing.T) { + store, clock := newTestEscalationStore(t) + cfg := escalationTestConfig() + pr := transientRedPR() + actionable := actionableWith(pr) + recordRedStaleness(cfg, actionable) + rerunner := &fakeTransientRerunner{out: github.TransientCIRerunOutcome{Settled: true, RunIDs: []int64{42}}} + var audited []string + audit := func(repo string, number int, detail string) { audited = append(audited, detail) } + + rerunTransientTriagedPRs(context.Background(), cfg, rerunner, actionable, map[string]bool{}, audit, discardLogger()) + rerunTransientTriagedPRs(context.Background(), cfg, rerunner, actionable, map[string]bool{}, audit, discardLogger()) + if len(rerunner.calls) != 1 { + t.Fatalf("re-run calls = %d, want exactly one per head", len(rerunner.calls)) + } + if len(audited) != 1 || !strings.Contains(audited[0], "runs=1") { + t.Fatalf("audit = %v, want one line naming the re-run", audited) + } + if store.TransientRerunOwed("acme/widgets", 7, "abc") { + t.Fatal("re-run still owed after it was spent") + } + + // Red again after the re-run: the re-run gave the checks new run ids, + // so the markers no longer cover the head. + pr.TransientCITriaged = false + actionable = actionableWith(pr) + recordRedStaleness(cfg, actionable) + *clock = clock.Add(escalation.RedPRStaleAfter + time.Minute) + kicker := &fakeKicker{} + reapStuckRedPRs(cfg, actionable, map[string]bool{}, kicker, nil, discardLogger()) + if len(kicker.kicks) != 1 { + t.Fatalf("after the one re-run the fix loop must resume, got %d kicks", len(kicker.kicks)) + } + + // A new head with a fresh transient triage earns its own re-run. + pr.HeadSHA, pr.TransientCITriaged = "def", true + actionable = actionableWith(pr) + recordRedStaleness(cfg, actionable) + rerunTransientTriagedPRs(context.Background(), cfg, rerunner, actionable, map[string]bool{}, nil, discardLogger()) + if len(rerunner.calls) != 2 || rerunner.calls[1].HeadSHA != "def" { + t.Fatalf("new head was not re-run: %+v", rerunner.calls) + } +} + +func TestRerunTransientTriagedPRs_Skips(t *testing.T) { + t.Run("escalated PR is a person's", func(t *testing.T) { + newTestEscalationStore(t) + cfg := escalationTestConfig() + pr := transientRedPR() + pr.Labels = []string{escalation.NeedsHumanLabel} + actionable := actionableWith(pr) + recordRedStaleness(cfg, actionable) + rerunner := &fakeTransientRerunner{out: github.TransientCIRerunOutcome{Settled: true}} + rerunTransientTriagedPRs(context.Background(), cfg, rerunner, actionable, map[string]bool{"acme/widgets#7": true}, nil, discardLogger()) + if len(rerunner.calls) != 0 { + t.Fatal("re-ran an escalated PR") + } + }) + t.Run("human-authored PR", func(t *testing.T) { + newTestEscalationStore(t) + cfg := escalationTestConfig() + pr := transientRedPR() + pr.Author = "someone" + actionable := actionableWith(pr) + recordRedStaleness(cfg, actionable) + rerunner := &fakeTransientRerunner{out: github.TransientCIRerunOutcome{Settled: true}} + rerunTransientTriagedPRs(context.Background(), cfg, rerunner, actionable, map[string]bool{}, nil, discardLogger()) + if len(rerunner.calls) != 0 { + t.Fatal("re-ran a human-authored PR") + } + }) + t.Run("untriaged red PR", func(t *testing.T) { + newTestEscalationStore(t) + cfg := escalationTestConfig() + actionable := actionableWith(redPR("widgets", 7, "hive-agent", "abc")) + recordRedStaleness(cfg, actionable) + rerunner := &fakeTransientRerunner{out: github.TransientCIRerunOutcome{Settled: true}} + rerunTransientTriagedPRs(context.Background(), cfg, rerunner, actionable, map[string]bool{}, nil, discardLogger()) + if len(rerunner.calls) != 0 { + t.Fatal("re-ran a PR nobody triaged as transient") + } + }) + t.Run("listing error is retried next pass", func(t *testing.T) { + store, _ := newTestEscalationStore(t) + cfg := escalationTestConfig() + actionable := actionableWith(transientRedPR()) + recordRedStaleness(cfg, actionable) + rerunner := &fakeTransientRerunner{err: errors.New("502")} + rerunTransientTriagedPRs(context.Background(), cfg, rerunner, actionable, map[string]bool{}, nil, discardLogger()) + if !store.TransientRerunOwed("acme/widgets", 7, "abc") { + t.Fatal("an API error spent the head's re-run") + } + rerunner.err, rerunner.out = nil, github.TransientCIRerunOutcome{Settled: true, Skipped: "repo-mode-off"} + rerunTransientTriagedPRs(context.Background(), cfg, rerunner, actionable, map[string]bool{}, nil, discardLogger()) + if len(rerunner.calls) != 2 || store.TransientRerunOwed("acme/widgets", 7, "abc") { + t.Fatalf("a settled refusal must spend the re-run: calls=%d", len(rerunner.calls)) + } + }) +} + +// The fix kick must tell the agent how to make an infra verdict visible to +// the hive (the transient marker) and forbid a retrigger push; on +// projectbluefin/common#1647 the prose-only verdict was invisible and a lane +// pushed an empty "retrigger CI" commit. +func TestRedPRFixKickInfraVerdictUsesTransientMarker(t *testing.T) { + got := redPRFixKick(escalation.Observation{Repo: "projectbluefin/common", Number: 1647, HeadSHA: "6772623"}, false) + for _, want := range []string{ + "do NOT push anything", + "\"retrigger CI\" commit", + "transient_marker", + "hive-baseline-check.sh projectbluefin/common '' 1647 --json", + "Hive\nre-runs the failed jobs itself, once.", + } { + if !strings.Contains(got, want) { + t.Errorf("fix kick missing %q in:\n%s", want, got) + } + } +} diff --git a/src/docs/adr/0010-escalation-circuit-breaker.md b/src/docs/adr/0010-escalation-circuit-breaker.md index a8beb7407b..ccd913cb00 100644 --- a/src/docs/adr/0010-escalation-circuit-breaker.md +++ b/src/docs/adr/0010-escalation-circuit-breaker.md @@ -50,6 +50,20 @@ cadence window. The staleness clock starts when CI SETTLES — a head red on one check while others are still running is not yet stuck. The escalation comment quotes the number of kicks actually delivered. +### Re-run, do not re-dispatch, a transient-triaged head + +A fix kick cannot repair an infrastructure failure. When every failing check +run on a hive-authored PR's head carries an agent's `hive-ci-transient` triage +marker, the eval cycle re-runs that head's failed jobs once +(`rerun-failed-jobs`, through the write-back gate as `ci_rerun`), and the +reaper and merge watcher send no fix kick and charge no budget while that +re-run is owed. The re-run gives the checks new run ids, so the markers go +stale on their own: a head still red afterwards is back in the ordinary loop +above. The re-run is recorded per head in the ledger; a person or the reviewer +lane returning an escalated PR to the lane allows one more. Before this, +projectbluefin/common#1647 spent all six kicks on one head whose agent had +answered each one with "infrastructure, not this change". + ### Exclude independently confirmed shared failures Shared CI breakage is not a fix attempt. A check is shared only when it fails diff --git a/src/docs/github-write-surface.md b/src/docs/github-write-surface.md index 13af6c5f3d..f9ff0af45f 100644 --- a/src/docs/github-write-surface.md +++ b/src/docs/github-write-surface.md @@ -157,6 +157,7 @@ writes no entry. | Task-list sweep closes | `task_list_sweep.go` | yes (`task-list-sweep-closed`) | yes | | Supersession sweep (notice, `hive/superseded` close, own-PR update-branch + body `Closes`→`Refs` rewrite, `needs-human` + `Parked` fact on escalation) | `pr_supersession_sweep.go`, `pr_supersession_autoclose.go`, `pr_supersession_resolve.go` | yes (`supersession-sweep-*`) | yes | | Shared-CI rerun (`rerun-failed-jobs`, `hive-shared-ci-rerun-` comment) | `shared_ci_rerun.go` | yes (`shared-ci-rerun`) | yes | +| Transient-CI rerun (`rerun-failed-jobs` on a hive PR whose failing runs carry `hive-ci-transient`, once per head, no comment) | `transient_ci_rerun.go` | yes (`transient-ci-rerun`) | yes | | Open-PR dispatch guard (`has-open-pr` label on a refused issue) and duplicate-fix sweep (one `hive:duplicate-fix` comment; `hive/superseded` close of the later hive PR, or `needs-human` + `Parked` fact on escalation) | `open_pr_links.go`, `pr_duplicate_fix_sweep.go` | yes (`duplicate-fix-*`) | yes | | Duplicate-PR sweep suggestions | `duplicate_sweep.go` | yes (`duplicate-sweep-suggested`) | yes (target = surviving PR) | | Hold-label migration (one-time) | `hive_hold_migration.go` | yes (`hold_migration_label_added`, one per item labeled; the migration report file is still written) | yes | diff --git a/src/docs/labels-and-control-signals/glossary-and-operations.md b/src/docs/labels-and-control-signals/glossary-and-operations.md index 9c6c87dbe5..144e254b7f 100644 --- a/src/docs/labels-and-control-signals/glossary-and-operations.md +++ b/src/docs/labels-and-control-signals/glossary-and-operations.md @@ -233,7 +233,7 @@ The dashboard renders every open issue and PR's stage as one unified tracker str - **Human comments and assignees** can acknowledge hive-filed directions for #5117; assignees also affect ranking. - **Hidden comment markers** carry state that no label does. `` records an issue claim, `` dedupes a published finding, `` marks an overlap notice, `` starts the question auto-close clock, and `` is stamped on the single comment a fix lane leaves on a PR it defers to shared incident `#` (`DEFER_TO_INCIDENT` from `bin/hive-baseline-check.sh`), so "every PR incident `#` broke" stays greppable after the fix lands ([#10441](https://github.com/hivecommons/hive/issues/10441)). One marker per PR per incident; it is never edited or removed while the incident is open. While incident `#` is open, the governor records it as `deferred_incident` on the PR's `ci-failing.json` row and the kick builders keep that PR out of every CI-FAILING / FIX-BEFORE-NEW repair list, naming it only as deferred; once the incident closes the PR is listed for repair again ([#10528](https://github.com/hivecommons/hive/issues/10528)). When the incident closes (not as "not planned"), the shared-CI rerun sweep re-runs the failed jobs of each still-open PR carrying that marker — the newest failed or timed-out run per workflow on the PR head — using the hive's own credential, then stamps `` on the PR so it is re-run at most once per incident; disable with `shared_ci_rerun.enabled: false` ([#12253](https://github.com/hivecommons/hive/issues/12253)). - **``** is stamped by a fix lane on the one comment it leaves when `bin/hive-baseline-check.sh` returns `DEFER_EXTERNAL`: the repository syncs from an upstream project (its GitHub fork parent, or `--upstream`) and the same check is red on the upstream default branch, so no change here can fix it. The helper prints the exact string as `external_marker`. While every failing check run on the PR's current head carries a marker for that head and run, the governor records `deferred_external` on the PR's `ci-failing.json` row and the kick builders name the PR only as deferred under `DEFERRED AS FAILING UPSTREAM`; a push, a newly failing check or a re-run of the check (a new run id — e.g. it went green and failed again at the same head) makes the markers stale and the PR is listed for repair again ([#11609](https://github.com/hivecommons/hive/issues/11609)). -- **``** is stamped by an agent on its triage comment when it diagnoses a red check on a PR as transient or infrastructure-related (a flaky job, a runner outage, a race against a cache the default branch was still publishing) rather than caused by the diff, and cannot re-run the check itself. The fields and pinning match `hive-ci-external`; `bin/hive-baseline-check.sh --json` with the PR number prints the exact string as `transient_marker` whenever the PR's newest run of the check is red, on any verdict, and the agent copies it verbatim. While every failing check run on the PR's current head carries a marker for that head and run, the PR's `transient_ci_triaged` is set and the dependency-bot red-CI hand-off still adds `needs-human` after its grace period (nothing re-runs a single PR's check automatically), but its comment says Hive's triage found the failure was not caused by the bump and asks for a re-run, instead of telling the owner to fix, pin or close the bump. A push or a re-run of the check (a new run id) makes the marker stale ([#12977](https://github.com/hivecommons/hive/issues/12977)). +- **``** is stamped by an agent on its triage comment when it diagnoses a red check on a PR as transient or infrastructure-related (a flaky job, a runner outage, a race against a cache the default branch was still publishing) rather than caused by the diff, and cannot re-run the check itself. The fields and pinning match `hive-ci-external`; `bin/hive-baseline-check.sh --json` with the PR number prints the exact string as `transient_marker` whenever the PR's newest run of the check is red, on any verdict, and the agent copies it verbatim. While every failing check run on the PR's current head carries a marker for that head and run, the PR's `transient_ci_triaged` is set. On a hive-authored PR the fix loop then re-runs the failed jobs once for that head (`rerun-failed-jobs`, gated as `ci_rerun`) and sends no FIX-BEFORE-NEW kick while the re-run is owed; a head still red after it is back in the ordinary fix loop ([projectbluefin/common#1647](https://github.com/projectbluefin/common/pull/1647)). The dependency-bot red-CI hand-off still adds `needs-human` after its grace period (dependency-bot PRs are not re-run automatically), but its comment says Hive's triage found the failure was not caused by the bump and asks for a re-run, instead of telling the owner to fix, pin or close the bump. A push or a re-run of the check (a new run id) makes the marker stale ([#12977](https://github.com/hivecommons/hive/issues/12977)). - **Paused repos** (`project.paused_repos`) stop whole-repo write/merge/enumeration paths without labels. - **Contributor queue holds** such as active leases, cooldowns, dependencies, and quota guard holds suppress contributor offers without touching GitHub labels. - **Jira/Linear labels** map only where the work-source adapter supports them: Linear skips held work using substring hold labels; Jira skips exact configured hold labels; GitHub Projects currently imports labels but does not use a hold gate. diff --git a/src/docs/operator-reference/sweeps-and-operations.md b/src/docs/operator-reference/sweeps-and-operations.md index d85ce01b74..fb33aca1ea 100644 --- a/src/docs/operator-reference/sweeps-and-operations.md +++ b/src/docs/operator-reference/sweeps-and-operations.md @@ -445,6 +445,7 @@ that level, the power the write-back exists to govern? | Welcome / outreach (`pkg/gfi`, `gfi_forge.go`, `pr_clanker_requested.go`) | good-first-issue selection | welcome comment, `good first issue` label | issues hive advertises | `welcome_comment` → L5 with label, L6 without | | Issue filing (`pkg/upstreamwatch/github_filer.go`, `advisory.go`, scanner/deps/regression agents, `dedupe_advisory_close.go`) | agents, advisory digest, upstream watch | **new** issues hive authors, pinned advisory issue + comments | `project.repos` | `issue_file` → L3, no label (this is what listing a repo opts in to; `per_repo.mode: issues-only` keeps only this) | | Shared-CI rerun (`shared_ci_rerun.go`) | CI failure watcher | `rerun-failed-jobs`, `hive-shared-ci-rerun-` comment | PRs with failed shared workflows | `ci_rerun` → L6 + label | +| Transient-CI rerun (`transient_ci_rerun.go`) | eval cycle, before the stuck-PR reaper | `rerun-failed-jobs`, once per head, no comment | hive-authored red PRs whose every failing run carries `hive-ci-transient` | `ci_rerun` (own items: allowed unless the repo is paused, off or issues-only) | | Level hold, self-authorization & reporter-trust releases (`pr_level_hold.go`, `automerge/self_authorization_release.go`, `reporter_trust_release.go`) | PR watcher | `hold` label add/remove | agent PRs | own items (hive-authored) — not gated | | Migrations & retirements (`hive_hold_migration.go`, `hive_id_label_retire.go`, `claimed_by_human_migration.go`, `origin_labels.go`) | boot, one-shot | label renames | items carrying hive labels | opted in by carrying hive labels | diff --git a/src/pkg/escalation/escalation.go b/src/pkg/escalation/escalation.go index 63ab2c6632..d3b5994340 100644 --- a/src/pkg/escalation/escalation.go +++ b/src/pkg/escalation/escalation.go @@ -199,6 +199,15 @@ type Entry struct { // RunWaitEscalatedGen records the run generation whose human wait timeout // already paged a human. It makes run-wait escalation once-per-generation. RunWaitEscalatedGen uint64 `json:"run_wait_escalated_gen,omitempty"` + + // TransientTriagedSHA is the red head on which every failing check run + // carries a `hive-ci-transient` triage marker (Observation. + // TransientTriaged), as of the last ObserveRed pass; "" otherwise. + // TransientRerunSHA is the head whose failed jobs the hive already + // re-ran because of such a triage. Together they make the re-run once + // per head and hold fix kicks while it is owed (transient_rerun.go). + TransientTriagedSHA string `json:"transient_triaged_sha,omitempty"` + TransientRerunSHA string `json:"transient_rerun_sha,omitempty"` } // Store is the on-PVC attempt ledger. All methods are safe for concurrent use. @@ -303,6 +312,12 @@ type Observation struct { // SharedCIIncident is the open [shared-ci] incident the PR is deferred // to, 0 when none; named in the hold comment. SharedCIIncident int + // TransientTriaged reports that every failing check run on this head + // carries an agent's `hive-ci-transient` marker for this head and run + // (github.PullRequest.TransientCITriaged): the failure was diagnosed as + // infrastructure, not this change. The fix loop re-runs the failed jobs + // once instead of kicking a fixer (transient_rerun.go). + TransientTriaged bool } // MarkSharedFailures classifies each red observation's failing checks as @@ -599,6 +614,9 @@ func (s *Store) Sweep(obs []Observation, threshold int) map[string]Result { e.RedSHAs = nil e.ReEngagements = 0 e.LastReEngagedAt = time.Time{} + // A person returned the PR to the lane: a fresh transient + // triage on the same head earns one more re-run. + e.TransientRerunSHA = "" e.UpdatedAt = s.now() } @@ -683,6 +701,7 @@ func (s *Store) Sweep(obs []Observation, threshold int) map[string]Result { e.RedSHAs = nil e.RedTrees = nil e.ReEngagements = 0 + e.TransientRerunSHA = "" // Remember WHAT the reviewer left on the branch and WHEN, so a // later re-escalation can hand the human the reviewer's context // instead of a bare label (#5617 item 3). Keyed on the SHA so a @@ -1088,6 +1107,7 @@ func (s *Store) ObserveRed(obs []Observation) { e.FirstRedAt = time.Time{} e.ReEngagements = 0 e.LastReEngagedAt = time.Time{} + e.TransientTriagedSHA = "" } continue } @@ -1113,6 +1133,10 @@ func (s *Store) ObserveRed(obs []Observation) { if o.Excerpt != "" { e.LastExcerpt = o.Excerpt } + e.TransientTriagedSHA = "" + if o.TransientTriaged { + e.TransientTriagedSHA = o.HeadSHA + } e.UpdatedAt = now } s.saveLocked() diff --git a/src/pkg/escalation/transient_rerun.go b/src/pkg/escalation/transient_rerun.go new file mode 100644 index 0000000000..e879f1158e --- /dev/null +++ b/src/pkg/escalation/transient_rerun.go @@ -0,0 +1,58 @@ +package escalation + +// Transient-triage re-run (projectbluefin/common#1647). +// +// When an agent diagnoses a red check as infrastructure, not the change (a +// runner's apt mirror dropping the connection, a lost runner), the repair is +// a re-run of the failed jobs, not a commit. Before this the fix loop kept +// kicking the owning agent: each kick was a delivered re-dispatch, the agent +// answered each one with the same "infrastructure, not this change" comment, +// and the sixth kick escalated the PR to a person. A lane then pushed an +// empty "retrigger CI" commit, which re-arms review fences and can dismiss +// approvals. +// +// The agent's triage reaches the hive as the `hive-ci-transient` marker +// (github.PullRequest.TransientCITriaged). ObserveRed records the head it +// covers; while a re-run of that head is owed, the fix-kick paths stand down +// (TransientRerunOwed) and the eval cycle re-runs the failed jobs once +// (RecordTransientRerun). The re-run gives the failing checks new run ids, so +// the markers go stale by themselves: a head that is red again afterwards is +// back in the ordinary fix loop, with its usual budget and escalation. + +// TransientRerunOwed reports that the PR's red head is transient-triaged and +// the hive has not yet re-run its failed jobs. headSHA "" means the head the +// ledger last saw red (the merge watcher's hook knows only the PR number). +// An escalated PR is a person's, so nothing is owed on it. +func (s *Store) TransientRerunOwed(repo string, number int, headSHA string) bool { + s.mu.Lock() + defer s.mu.Unlock() + e := s.entries[Key(repo, number)] + if e == nil || e.Escalated || e.ReporterTrustReason != "" { + return false + } + if headSHA == "" { + headSHA = e.CurRedSHA + } + return headSHA != "" && e.TransientTriagedSHA == headSHA && e.TransientRerunSHA != headSHA +} + +// RecordTransientRerun marks the head's one transient re-run as spent. The +// caller records it once the attempt is settled: re-run, refused by the +// write-back gate or by GitHub, or nothing left to re-run. An attempt that +// failed for a reason a later pass can clear (an API error) is not recorded. +func (s *Store) RecordTransientRerun(repo string, number int, headSHA string) { + if headSHA == "" { + return + } + s.mu.Lock() + defer s.mu.Unlock() + key := Key(repo, number) + e := s.entries[key] + if e == nil { + e = &Entry{Machinery: MachineryVersion} + s.entries[key] = e + } + e.TransientRerunSHA = headSHA + e.UpdatedAt = s.now() + s.saveLocked() +} diff --git a/src/pkg/escalation/transient_rerun_test.go b/src/pkg/escalation/transient_rerun_test.go new file mode 100644 index 0000000000..3a4790194a --- /dev/null +++ b/src/pkg/escalation/transient_rerun_test.go @@ -0,0 +1,69 @@ +package escalation + +import ( + "path/filepath" + "testing" + "time" +) + +func transientObs(head string, triaged bool, labels ...string) Observation { + return Observation{Repo: "o/r", Number: 1, HeadSHA: head, Red: true, FailingChecks: []string{"build"}, TransientTriaged: triaged, Labels: labels, Labeled: HasNeedsHumanLabel(labels)} +} + +func TestTransientRerunOwedOncePerHead(t *testing.T) { + s := Load(filepath.Join(t.TempDir(), "l.json")) + if s.TransientRerunOwed("o/r", 1, "a") { + t.Fatal("owed with no observation") + } + s.ObserveRed([]Observation{transientObs("a", false)}) + if s.TransientRerunOwed("o/r", 1, "a") { + t.Fatal("owed on an untriaged head") + } + s.ObserveRed([]Observation{transientObs("a", true)}) + if !s.TransientRerunOwed("o/r", 1, "a") || !s.TransientRerunOwed("o/r", 1, "") { + t.Fatal("triaged head must owe a re-run (by sha and by the ledger's current head)") + } + if s.TransientRerunOwed("o/r", 1, "b") { + t.Fatal("owed for a head the triage does not cover") + } + s.RecordTransientRerun("o/r", 1, "a") + if s.TransientRerunOwed("o/r", 1, "a") { + t.Fatal("still owed after the re-run") + } + // Survives a restart: the ledger is on the PVC. + s2 := Load(s.path) + s2.ObserveRed([]Observation{transientObs("a", true)}) + if s2.TransientRerunOwed("o/r", 1, "a") { + t.Fatal("a restart re-armed the spent re-run") + } + // A new head with its own triage owes its own re-run. + s2.ObserveRed([]Observation{transientObs("b", true)}) + if !s2.TransientRerunOwed("o/r", 1, "b") { + t.Fatal("new triaged head owes no re-run") + } + // Markers gone stale (the re-run made new run ids): nothing owed. + s2.ObserveRed([]Observation{transientObs("b", false)}) + if s2.TransientRerunOwed("o/r", 1, "b") { + t.Fatal("owed after the triage went stale") + } +} + +// A person (or the reviewer lane) returning an escalated PR to the lane +// gives a still-triaged head one more re-run. +func TestTransientRerunReArmedWhenUnparked(t *testing.T) { + s := Load(filepath.Join(t.TempDir(), "l.json")) + now := time.Date(2026, 11, 6, 21, 0, 0, 0, time.UTC) + s.SetClock(func() time.Time { return now }) + s.ObserveRed([]Observation{transientObs("a", true)}) + s.RecordTransientRerun("o/r", 1, "a") + s.Sweep([]Observation{transientObs("a", true, NeedsHumanLabel)}, DefaultThreshold) + if s.TransientRerunOwed("o/r", 1, "a") { + t.Fatal("an escalated PR must owe nothing") + } + now = now.Add(LabelUnparkGrace + time.Minute) + s.ObserveRed([]Observation{transientObs("a", true, ReviewerPassedLabel)}) + s.Sweep([]Observation{transientObs("a", true, ReviewerPassedLabel)}, DefaultThreshold) + if !s.TransientRerunOwed("o/r", 1, "a") { + t.Fatal("un-parked PR with a fresh transient triage must owe one more re-run") + } +} diff --git a/src/pkg/github/transient_ci_rerun.go b/src/pkg/github/transient_ci_rerun.go new file mode 100644 index 0000000000..5598941d62 --- /dev/null +++ b/src/pkg/github/transient_ci_rerun.go @@ -0,0 +1,78 @@ +package github + +import ( + "context" + "fmt" + + "github.com/hivecommons/hive/pkg/writeback" +) + +// TransientCIRerunOutcome is what RerunTransientCIFailures did for one PR. +type TransientCIRerunOutcome struct { + // Settled means the head's one transient re-run is spent: jobs were + // re-run, the write-back gate or GitHub refused, or nothing failed is + // left to re-run. Unsettled (with an error) means a later pass may retry. + Settled bool + // RunIDs are the workflow runs whose failed jobs GitHub accepted for a + // re-run. + RunIDs []int64 + // Skipped names why nothing was re-run ("" when RunIDs is non-empty): + // a write-back gate reason, "rerun-forbidden" or "no-failed-runs". + Skipped string +} + +// RerunTransientCIFailures re-runs the failed jobs of every failed workflow +// run on pr's head, for a red PR an agent triaged as transient +// (PullRequest.TransientCITriaged). It is the deterministic repair the fix +// loop uses instead of kicking a fixer that can only comment, or pushing a +// "retrigger CI" commit that re-arms fences and can dismiss approvals +// (projectbluefin/common#1647). It never pushes. The re-run is an +// unsolicited write, so it consults the write-back gate (KindCIRerun) first; +// a hive-authored PR passes unless the repository is paused, off or +// issues-only. Once any re-run request reached GitHub the attempt is settled +// whatever came back, so a head gets at most one round of re-run calls. +func (c *Client) RerunTransientCIFailures(ctx context.Context, pr PullRequest) (TransientCIRerunOutcome, error) { + if c == nil || c.client == nil { + return TransientCIRerunOutcome{}, ErrNoGitHubClient + } + owner, name := c.splitRepo(pr.Repo) + repo := owner + "/" + name + preview := fmt.Sprintf("re-run the failed jobs on %s#%d at %s (transient CI triage)", repo, pr.Number, shortSHA(pr.HeadSHA)) + if d := c.WritebackAllowed(repo, c.WritebackItemForPR(pr), writeback.KindCIRerun, preview); !d.Allowed { + return TransientCIRerunOutcome{Settled: true, Skipped: d.Reason}, nil + } + if c.sharedCIRerunRefused(repo) { + return TransientCIRerunOutcome{Settled: true, Skipped: "rerun-forbidden"}, nil + } + ctx = WithRESTCaller(ctx, "hive:transient_ci_rerun") + runs, err := c.latestFailedWorkflowRuns(ctx, owner, name, pr.HeadSHA) + if err != nil { + return TransientCIRerunOutcome{}, err + } + if len(runs) == 0 { + // Failed on a check no Actions run backs (an external CI), or it was + // already re-run by hand: nothing the hive can re-run. + return TransientCIRerunOutcome{Settled: true, Skipped: "no-failed-runs"}, nil + } + out := TransientCIRerunOutcome{Settled: true} + for _, run := range runs { + if _, err := c.client.Actions.RerunFailedJobsByID(ctx, owner, name, run.GetID()); err != nil { // writeback:gated — WritebackAllowed(KindCIRerun) consulted above + if isWriteForbiddenError(err) { + if c.markSharedCIRerunForbidden(repo) { + c.logger.Warn("transient CI rerun forbidden: the App installation lacks Actions write permission for rerun-failed-jobs", "repo", repo, "pr", pr.Number, "run", run.GetID(), "error", err) + } + out.Skipped = "rerun-forbidden" + break + } + c.logger.Warn("transient CI rerun failed", "repo", repo, "pr", pr.Number, "run", run.GetID(), "error", err) + continue + } + out.RunIDs = append(out.RunIDs, run.GetID()) + } + if len(out.RunIDs) > 0 { + out.Skipped = "" + } else if out.Skipped == "" { + out.Skipped = "rerun-failed" + } + return out, nil +} diff --git a/src/pkg/github/transient_ci_rerun_test.go b/src/pkg/github/transient_ci_rerun_test.go new file mode 100644 index 0000000000..b9ad3dd977 --- /dev/null +++ b/src/pkg/github/transient_ci_rerun_test.go @@ -0,0 +1,88 @@ +package github + +import ( + "context" + "reflect" + "testing" + + "github.com/hivecommons/hive/pkg/writeback" +) + +// A transient-triaged red PR gets its failed jobs re-run through +// rerun-failed-jobs, never a push (projectbluefin/common#1647), and the +// re-run is an unsolicited write the write-back gate can refuse. +func TestRerunTransientCIFailures(t *testing.T) { + failedPR := func() []*sharedCIRerunPR { + return []*sharedCIRerunPR{{Number: 1, SHA: "aaaaaaaaaa", Runs: []sharedCIRerunRun{ + failedRun(11, 1), + {ID: 12, WorkflowID: 2, RunNumber: 1, Attempt: 1, Status: "completed", Conclusion: "success"}, + }}} + } + pr := PullRequest{Repo: "org/repo", Number: 1, HeadSHA: "aaaaaaaaaa", AppAuthored: true, CIStatus: "failure", TransientCITriaged: true} + + t.Run("re-runs the failed workflow runs on the head", func(t *testing.T) { + srv, fake := newSharedCIRerunFake(t, nil, failedPR()) + c := newTestClient(t, srv, "org", []string{"org/repo"}) + c.SetWritebackGate(writeback.New(writeback.Config{}, writeback.Deps{}, "")) + got, err := c.RerunTransientCIFailures(context.Background(), pr) + if err != nil { + t.Fatalf("rerun: %v", err) + } + reruns, posted, _ := fake.snap() + if !reflect.DeepEqual(reruns, []int64{11}) { + t.Fatalf("reruns = %v, want [11] (the failed run only)", reruns) + } + if !got.Settled || !reflect.DeepEqual(got.RunIDs, []int64{11}) || got.Skipped != "" { + t.Fatalf("outcome = %+v, want settled with run 11", got) + } + if len(posted) != 0 { + t.Fatalf("a transient re-run must not comment, posted %v", posted) + } + }) + + t.Run("write-back gate refusal makes no GitHub write", func(t *testing.T) { + srv, fake := newSharedCIRerunFake(t, nil, failedPR()) + c := newTestClient(t, srv, "org", []string{"org/repo"}) + c.SetWritebackGate(writeback.New(writeback.Config{PerRepo: map[string]writeback.RepoOverride{ + "org/repo": {Mode: writeback.RepoModeOff}, + }}, writeback.Deps{}, "")) + got, err := c.RerunTransientCIFailures(context.Background(), pr) + if err != nil { + t.Fatalf("rerun: %v", err) + } + if reruns, _, _ := fake.snap(); len(reruns) != 0 { + t.Fatalf("gate refused, but reran %v", reruns) + } + if !got.Settled || len(got.RunIDs) != 0 || got.Skipped != writeback.ReasonRepoOff { + t.Fatalf("outcome = %+v, want settled and skipped for %q", got, writeback.ReasonRepoOff) + } + }) + + t.Run("forbidden re-run settles the attempt", func(t *testing.T) { + srv, fake := newSharedCIRerunFake(t, nil, failedPR()) + fake.forbidRerun = true + c := newTestClient(t, srv, "org", []string{"org/repo"}) + got, err := c.RerunTransientCIFailures(context.Background(), pr) + if err != nil { + t.Fatalf("rerun: %v", err) + } + if !got.Settled || len(got.RunIDs) != 0 || got.Skipped != "rerun-forbidden" { + t.Fatalf("outcome = %+v, want settled rerun-forbidden", got) + } + }) + + t.Run("nothing failed to re-run", func(t *testing.T) { + srv, fake := newSharedCIRerunFake(t, nil, []*sharedCIRerunPR{{Number: 1, SHA: "aaaaaaaaaa"}}) + c := newTestClient(t, srv, "org", []string{"org/repo"}) + got, err := c.RerunTransientCIFailures(context.Background(), pr) + if err != nil { + t.Fatalf("rerun: %v", err) + } + if reruns, _, _ := fake.snap(); len(reruns) != 0 { + t.Fatalf("reran %v with no failed runs", reruns) + } + if !got.Settled || got.Skipped != "no-failed-runs" { + t.Fatalf("outcome = %+v, want settled no-failed-runs", got) + } + }) +} diff --git a/src/pkg/policies/defaults/reviewer-lane.md b/src/pkg/policies/defaults/reviewer-lane.md index 3cc904b82a..70d61338d4 100644 --- a/src/pkg/policies/defaults/reviewer-lane.md +++ b/src/pkg/policies/defaults/reviewer-lane.md @@ -16,8 +16,14 @@ ADJUDICATION CONTRACT — for EACH PR above, deliver EXACTLY ONE verdict: d. After completing the mandatory audit below, return it to the automated lane and mark your pass: gh pr edit --remove-label needs-human --add-label ${REVIEWER_PASSED_LABEL} 2. DE-ESCALATE — the failure was environmental (base-branch regression since - fixed, infra flake): rebase the branch on its base, push, complete the - mandatory audit below, then + fixed, infra flake). If the branch is behind its base, rebase it on the + base and push. If it is current, push NOTHING: no empty, no-op or + "retrigger CI" commit (a push re-arms review fences and can dismiss + approvals). Re-run the failed jobs with gh run rerun --failed, + or, if you cannot, leave one PR comment that ends with the transient_marker + hive-baseline-check.sh '' --json prints for + each failing check, so Hive re-runs them. Then complete the mandatory + audit below, then gh pr edit --remove-label needs-human --add-label ${REVIEWER_PASSED_LABEL} 3. RECOMMEND-CLOSE — duplicate, superseded, or irreparably lossy: complete the mandatory audit below using its recommend-close review body, then mark the diff --git a/src/pkg/scheduler/reviewer_lane.go b/src/pkg/scheduler/reviewer_lane.go index a3e940c4c5..926db252e0 100644 --- a/src/pkg/scheduler/reviewer_lane.go +++ b/src/pkg/scheduler/reviewer_lane.go @@ -312,8 +312,14 @@ func (s *Scheduler) buildReviewerMessageHardcoded(agentName string, level int, w b.WriteString(fmt.Sprintf(" d. After completing the mandatory audit below, return it to the automated lane and mark your pass:\n"+ " gh pr edit --remove-label needs-human --add-label %s\n", ReviewerPassedLabel)) b.WriteString(" 2. DE-ESCALATE — the failure was environmental (base-branch regression since\n") - b.WriteString(" fixed, infra flake): rebase the branch on its base, push, complete the\n") - b.WriteString(" mandatory audit below, then\n") + b.WriteString(" fixed, infra flake). If the branch is behind its base, rebase it on the\n") + b.WriteString(" base and push. If it is current, push NOTHING: no empty, no-op or\n") + b.WriteString(" \"retrigger CI\" commit (a push re-arms review fences and can dismiss\n") + b.WriteString(" approvals). Re-run the failed jobs with gh run rerun --failed,\n") + b.WriteString(" or, if you cannot, leave one PR comment that ends with the transient_marker\n") + b.WriteString(" hive-baseline-check.sh '' --json prints for\n") + b.WriteString(" each failing check, so Hive re-runs them. Then complete the mandatory\n") + b.WriteString(" audit below, then\n") b.WriteString(fmt.Sprintf(" gh pr edit --remove-label needs-human --add-label %s\n", ReviewerPassedLabel)) b.WriteString(" 3. RECOMMEND-CLOSE — duplicate, superseded, or irreparably lossy: complete the\n") b.WriteString(" mandatory audit below using its recommend-close review body, then mark the\n") diff --git a/src/policies/reviewer-lane.md b/src/policies/reviewer-lane.md index 3cc904b82a..70d61338d4 100644 --- a/src/policies/reviewer-lane.md +++ b/src/policies/reviewer-lane.md @@ -16,8 +16,14 @@ ADJUDICATION CONTRACT — for EACH PR above, deliver EXACTLY ONE verdict: d. After completing the mandatory audit below, return it to the automated lane and mark your pass: gh pr edit --remove-label needs-human --add-label ${REVIEWER_PASSED_LABEL} 2. DE-ESCALATE — the failure was environmental (base-branch regression since - fixed, infra flake): rebase the branch on its base, push, complete the - mandatory audit below, then + fixed, infra flake). If the branch is behind its base, rebase it on the + base and push. If it is current, push NOTHING: no empty, no-op or + "retrigger CI" commit (a push re-arms review fences and can dismiss + approvals). Re-run the failed jobs with gh run rerun --failed, + or, if you cannot, leave one PR comment that ends with the transient_marker + hive-baseline-check.sh '' --json prints for + each failing check, so Hive re-runs them. Then complete the mandatory + audit below, then gh pr edit --remove-label needs-human --add-label ${REVIEWER_PASSED_LABEL} 3. RECOMMEND-CLOSE — duplicate, superseded, or irreparably lossy: complete the mandatory audit below using its recommend-close review body, then mark the From 6e2d8233ad8a2265ac6157b161ac563c910a23db Mon Sep 17 00:00:00 2001 From: Danathar Date: Sat, 7 Nov 2026 10:56:53 -0500 Subject: [PATCH 2/4] fix(escalation): only the hive's own comments can mark a red head transient The hive reviewer on #17002 pointed out that the transient-CI marker was read from every comment on the PR. Before this PR the marker only deferred a PR. Now it re-runs the failed jobs and holds the fix loop, so a comment from anyone on a public repo could spend Actions minutes and pause repairs once per head. redPRCommentTexts now also returns the bodies the hive's own accounts wrote, and TransientCITriaged is read from those alone. A hive that cannot name its own accounts falls back to every body, as the other hive-authorship checks do. The shared-CI and external-CI markers keep their existing any-author reading. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BQYnPGFJx81c8JXR3o1sg1 Signed-off-by: Danathar --- .../adr/0010-escalation-circuit-breaker.md | 4 +- .../glossary-and-operations.md | 2 +- src/pkg/github/client.go | 4 +- src/pkg/github/shared_ci_incident.go | 17 ++++- .../github/transient_ci_marker_author_test.go | 72 +++++++++++++++++++ 5 files changed, 92 insertions(+), 7 deletions(-) create mode 100644 src/pkg/github/transient_ci_marker_author_test.go diff --git a/src/docs/adr/0010-escalation-circuit-breaker.md b/src/docs/adr/0010-escalation-circuit-breaker.md index ccd913cb00..ba3fd4f96a 100644 --- a/src/docs/adr/0010-escalation-circuit-breaker.md +++ b/src/docs/adr/0010-escalation-circuit-breaker.md @@ -57,7 +57,9 @@ run on a hive-authored PR's head carries an agent's `hive-ci-transient` triage marker, the eval cycle re-runs that head's failed jobs once (`rerun-failed-jobs`, through the write-back gate as `ci_rerun`), and the reaper and merge watcher send no fix kick and charge no budget while that -re-run is owed. The re-run gives the checks new run ids, so the markers go +re-run is owed. Only markers in the hive's own comments count, because a +marker now causes a write: a comment from anyone else cannot spend Actions +minutes or pause the fix loop. The re-run gives the checks new run ids, so the markers go stale on their own: a head still red afterwards is back in the ordinary loop above. The re-run is recorded per head in the ledger; a person or the reviewer lane returning an escalated PR to the lane allows one more. Before this, diff --git a/src/docs/labels-and-control-signals/glossary-and-operations.md b/src/docs/labels-and-control-signals/glossary-and-operations.md index 144e254b7f..257d68d448 100644 --- a/src/docs/labels-and-control-signals/glossary-and-operations.md +++ b/src/docs/labels-and-control-signals/glossary-and-operations.md @@ -233,7 +233,7 @@ The dashboard renders every open issue and PR's stage as one unified tracker str - **Human comments and assignees** can acknowledge hive-filed directions for #5117; assignees also affect ranking. - **Hidden comment markers** carry state that no label does. `` records an issue claim, `` dedupes a published finding, `` marks an overlap notice, `` starts the question auto-close clock, and `` is stamped on the single comment a fix lane leaves on a PR it defers to shared incident `#` (`DEFER_TO_INCIDENT` from `bin/hive-baseline-check.sh`), so "every PR incident `#` broke" stays greppable after the fix lands ([#10441](https://github.com/hivecommons/hive/issues/10441)). One marker per PR per incident; it is never edited or removed while the incident is open. While incident `#` is open, the governor records it as `deferred_incident` on the PR's `ci-failing.json` row and the kick builders keep that PR out of every CI-FAILING / FIX-BEFORE-NEW repair list, naming it only as deferred; once the incident closes the PR is listed for repair again ([#10528](https://github.com/hivecommons/hive/issues/10528)). When the incident closes (not as "not planned"), the shared-CI rerun sweep re-runs the failed jobs of each still-open PR carrying that marker — the newest failed or timed-out run per workflow on the PR head — using the hive's own credential, then stamps `` on the PR so it is re-run at most once per incident; disable with `shared_ci_rerun.enabled: false` ([#12253](https://github.com/hivecommons/hive/issues/12253)). - **``** is stamped by a fix lane on the one comment it leaves when `bin/hive-baseline-check.sh` returns `DEFER_EXTERNAL`: the repository syncs from an upstream project (its GitHub fork parent, or `--upstream`) and the same check is red on the upstream default branch, so no change here can fix it. The helper prints the exact string as `external_marker`. While every failing check run on the PR's current head carries a marker for that head and run, the governor records `deferred_external` on the PR's `ci-failing.json` row and the kick builders name the PR only as deferred under `DEFERRED AS FAILING UPSTREAM`; a push, a newly failing check or a re-run of the check (a new run id — e.g. it went green and failed again at the same head) makes the markers stale and the PR is listed for repair again ([#11609](https://github.com/hivecommons/hive/issues/11609)). -- **``** is stamped by an agent on its triage comment when it diagnoses a red check on a PR as transient or infrastructure-related (a flaky job, a runner outage, a race against a cache the default branch was still publishing) rather than caused by the diff, and cannot re-run the check itself. The fields and pinning match `hive-ci-external`; `bin/hive-baseline-check.sh --json` with the PR number prints the exact string as `transient_marker` whenever the PR's newest run of the check is red, on any verdict, and the agent copies it verbatim. While every failing check run on the PR's current head carries a marker for that head and run, the PR's `transient_ci_triaged` is set. On a hive-authored PR the fix loop then re-runs the failed jobs once for that head (`rerun-failed-jobs`, gated as `ci_rerun`) and sends no FIX-BEFORE-NEW kick while the re-run is owed; a head still red after it is back in the ordinary fix loop ([projectbluefin/common#1647](https://github.com/projectbluefin/common/pull/1647)). The dependency-bot red-CI hand-off still adds `needs-human` after its grace period (dependency-bot PRs are not re-run automatically), but its comment says Hive's triage found the failure was not caused by the bump and asks for a re-run, instead of telling the owner to fix, pin or close the bump. A push or a re-run of the check (a new run id) makes the marker stale ([#12977](https://github.com/hivecommons/hive/issues/12977)). +- **``** is stamped by an agent on its triage comment when it diagnoses a red check on a PR as transient or infrastructure-related (a flaky job, a runner outage, a race against a cache the default branch was still publishing) rather than caused by the diff, and cannot re-run the check itself. The fields and pinning match `hive-ci-external`; `bin/hive-baseline-check.sh --json` with the PR number prints the exact string as `transient_marker` whenever the PR's newest run of the check is red, on any verdict, and the agent copies it verbatim. While every failing check run on the PR's current head carries a marker for that head and run, the PR's `transient_ci_triaged` is set. On a hive-authored PR the fix loop then re-runs the failed jobs once for that head (`rerun-failed-jobs`, gated as `ci_rerun`) and sends no FIX-BEFORE-NEW kick while the re-run is owed; a head still red after it is back in the ordinary fix loop ([projectbluefin/common#1647](https://github.com/projectbluefin/common/pull/1647)). Only markers in the hive's own comments count here, so a comment from anyone else cannot trigger the re-run or hold the fix loop. The dependency-bot red-CI hand-off still adds `needs-human` after its grace period (dependency-bot PRs are not re-run automatically), but its comment says Hive's triage found the failure was not caused by the bump and asks for a re-run, instead of telling the owner to fix, pin or close the bump. A push or a re-run of the check (a new run id) makes the marker stale ([#12977](https://github.com/hivecommons/hive/issues/12977)). - **Paused repos** (`project.paused_repos`) stop whole-repo write/merge/enumeration paths without labels. - **Contributor queue holds** such as active leases, cooldowns, dependencies, and quota guard holds suppress contributor offers without touching GitHub labels. - **Jira/Linear labels** map only where the work-source adapter supports them: Linear skips held work using substring hold labels; Jira skips exact configured hold labels; GitHub Projects currently imports labels but does not use a hold gate. diff --git a/src/pkg/github/client.go b/src/pkg/github/client.go index 30240daaff..a22490cd86 100644 --- a/src/pkg/github/client.go +++ b/src/pkg/github/client.go @@ -2746,10 +2746,10 @@ func (c *Client) EnrichCIStatus(ctx context.Context, prs []PullRequest) { prs[i].CIActivity.WorkflowApproval = c.workflowApprovalWithLedger(prs[i].Repo, prs[i].Number, prs[i].HeadSHA, c.workflowRunsAwaitingApproval(ctx, prs[i].Repo, prs[i].HeadSHA)) } if prs[i].CIStatus == "failure" { - if texts, ok := c.redPRCommentTexts(ctx, &prs[i]); ok { + if texts, hiveTexts, ok := c.redPRCommentTexts(ctx, &prs[i]); ok { prs[i].SharedCIIncident = c.openSharedCIIncident(ctx, &prs[i], texts, incidents) prs[i].ExternalCIDeferred = externalCIDeferred(texts, prs[i].HeadSHA, prs[i].FailingCheckRunIDs) - prs[i].TransientCITriaged = transientCITriaged(texts, prs[i].HeadSHA, prs[i].FailingCheckRunIDs) + prs[i].TransientCITriaged = transientCITriaged(hiveTexts, prs[i].HeadSHA, prs[i].FailingCheckRunIDs) } } facts.attach(ctx, &prs[i], reported) diff --git a/src/pkg/github/shared_ci_incident.go b/src/pkg/github/shared_ci_incident.go index b844c49dd4..3c61302f53 100644 --- a/src/pkg/github/shared_ci_incident.go +++ b/src/pkg/github/shared_ci_incident.go @@ -35,19 +35,30 @@ func newestSharedCIMarker(texts []string) int { // incident marker (hivecommons/hive#10528) and the external-CI marker // (hivecommons/hive#11609). One listing serves both. ok is false when the // comments cannot be read; the PR then stays listed. -func (c *Client) redPRCommentTexts(ctx context.Context, pr *PullRequest) (texts []string, ok bool) { +// +// hiveTexts are the bodies the hive's own accounts wrote. The transient-CI +// marker is read from those alone: it re-runs the failed jobs and holds the +// fix loop until they finish, so a comment from anyone else must not be able +// to spend Actions minutes or pause repairs (hivecommons/hive#17002 review). +// A hive that cannot name its own accounts falls back to every body, as the +// other hive-authorship checks do. +func (c *Client) redPRCommentTexts(ctx context.Context, pr *PullRequest) (texts, hiveTexts []string, ok bool) { owner, repo := c.splitRepo(pr.Repo) ctx = WithRESTCaller(ctx, "hive:shared_ci_incident") comments, err := c.listIssueComments(ctx, owner, repo, pr.Number) if err != nil { c.logger.Debug("CI deferral marker lookup failed; PR stays in the repair list", "repo", pr.Repo, "pr", pr.Number, "error", err) - return nil, false + return nil, nil, false } texts = make([]string, 0, len(comments)) + ownOnly := c.knowsOwnAccounts() for _, comment := range comments { texts = append(texts, comment.GetBody()) + if !ownOnly || c.isHiveAccount(comment.GetUser()) { + hiveTexts = append(hiveTexts, comment.GetBody()) + } } - return texts, true + return texts, hiveTexts, true } // openSharedCIIncident returns the shared-CI incident a red PR has already diff --git a/src/pkg/github/transient_ci_marker_author_test.go b/src/pkg/github/transient_ci_marker_author_test.go new file mode 100644 index 0000000000..ae745b3559 --- /dev/null +++ b/src/pkg/github/transient_ci_marker_author_test.go @@ -0,0 +1,72 @@ +package github + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "regexp" + "strconv" + "strings" + "testing" +) + +// The transient-CI marker now re-runs the failed jobs and holds the fix loop +// (hivecommons/hive#17002), so only the hive's own comments may set it: a +// marker anyone else posts on a public PR must not spend Actions minutes or +// pause repairs. The external-CI marker keeps its old any-author reading. +func TestEnrichCIStatus_TransientCITriagedOnlyFromHiveComments(t *testing.T) { + head := strings.Repeat("a", 40) + marker := "Infra flake.\n" + type comment struct{ login, body string } + comments := map[int][]comment{ + 1: {{"hivecommons-hive[bot]", marker}}, + 2: {{"mallory", marker}}, + 3: {{"mallory", externalMarker(head, "build", 1)}}, + } + commentsRE := regexp.MustCompile(`^/repos/org/repo/issues/(\d+)/comments$`) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if m := commentsRE.FindStringSubmatch(r.URL.Path); m != nil { + n, _ := strconv.Atoi(m[1]) + out := []map[string]any{} + for _, cm := range comments[n] { + out = append(out, map[string]any{"body": cm.body, "user": map[string]any{"login": cm.login}}) + } + _ = json.NewEncoder(w).Encode(out) + return + } + if regexp.MustCompile(`/commits/[^/]+/check-runs$`).MatchString(r.URL.Path) { + _ = json.NewEncoder(w).Encode(map[string]any{ + "total_count": 1, + "check_runs": []map[string]any{{"id": 1, "name": "build", "status": "completed", "conclusion": "failure"}}, + }) + return + } + w.WriteHeader(http.StatusNotFound) + })) + t.Cleanup(srv.Close) + + c := newTestClient(t, srv, "org", []string{"repo"}) + c.SetAppBotLogin("hivecommons-hive[bot]") + prs := []PullRequest{ + {Repo: "org/repo", Number: 1, HeadSHA: head}, + {Repo: "org/repo", Number: 2, HeadSHA: head}, + {Repo: "org/repo", Number: 3, HeadSHA: head}, + } + c.EnrichCIStatus(context.Background(), prs) + + for _, pr := range prs { + if pr.CIStatus != "failure" { + t.Fatalf("PR %d CIStatus = %q, want failure", pr.Number, pr.CIStatus) + } + } + if !prs[0].TransientCITriaged { + t.Error("the hive's own transient marker did not triage the head") + } + if prs[1].TransientCITriaged { + t.Error("a transient marker from someone outside the hive triaged the head (it would trigger a re-run)") + } + if !prs[2].ExternalCIDeferred { + t.Error("the external-CI marker must keep reading every author's comments") + } +} From 7404c8e12e7c0995b1cc4ac509805f9f673ca78e Mon Sep 17 00:00:00 2001 From: Danathar Date: Sat, 7 Nov 2026 11:26:22 -0500 Subject: [PATCH 3/4] fix(escalation): the cycle that re-runs a transient head sends it no fix kick The hive reviewer on #17002 found a same-cycle gap. rerunTransientTriagedPRs records the re-run, and then reapStuckRedPRs runs on the same snapshot, where the head still reads red and stale and the re-run is no longer owed. When the agent stamped the marker in its own session there is no earlier kick, so no cooldown applies. The cycle that re-ran the jobs could then also kick a fixer and charge a re-engagement. rerunTransientTriagedPRs now returns the PRs whose failed jobs it re-ran, and the reaper skips those for the cycle (reaperSkipAfterRerun). A skipped re-run (no failed runs, re-run forbidden) is not returned, so the fix loop resumes on it as before. ADR-0010 and the glossary now note the fallback: a hive that can't name its own accounts reads every comment for the marker. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BQYnPGFJx81c8JXR3o1sg1 Signed-off-by: Danathar --- src/cmd/hive/main_eval_cycle.go | 6 ++- src/cmd/hive/transient_ci_rerun.go | 36 ++++++++++++++-- src/cmd/hive/transient_ci_rerun_test.go | 42 +++++++++++++++++++ .../adr/0010-escalation-circuit-breaker.md | 7 +++- .../glossary-and-operations.md | 2 +- 5 files changed, 86 insertions(+), 7 deletions(-) diff --git a/src/cmd/hive/main_eval_cycle.go b/src/cmd/hive/main_eval_cycle.go index 44fbeabedb..5c1e3540b3 100644 --- a/src/cmd/hive/main_eval_cycle.go +++ b/src/cmd/hive/main_eval_cycle.go @@ -803,6 +803,7 @@ func runEvalCycle( // markers) is re-run once here instead: the reaper and the merge watcher // hold their fix kicks while that re-run is owed // (projectbluefin/common#1647). + reapSkip := escalatedPRs if ghClient != nil { var audit func(string, int, string) if dashSrv != nil { @@ -810,9 +811,10 @@ func runEvalCycle( dashSrv.AuditLogRecord("system", "transient-ci-rerun", detail, "", repo, number) } } - rerunTransientTriagedPRs(ctx, cfg, ghClient, actionable, escalatedPRs, audit, logger) + reran := rerunTransientTriagedPRs(ctx, cfg, ghClient, actionable, escalatedPRs, audit, logger) + reapSkip = reaperSkipAfterRerun(escalatedPRs, reran) } - reapStuckRedPRs(cfg, actionable, escalatedPRs, agentKicker{mgr: agentMgr}, + reapStuckRedPRs(cfg, actionable, reapSkip, agentKicker{mgr: agentMgr}, agentAvailability(cfg, agentMgr), logger) // Release sentinel (hivecommons/hive#9585), opt-in and default OFF: when diff --git a/src/cmd/hive/transient_ci_rerun.go b/src/cmd/hive/transient_ci_rerun.go index 4495baffc9..417b6947e0 100644 --- a/src/cmd/hive/transient_ci_rerun.go +++ b/src/cmd/hive/transient_ci_rerun.go @@ -33,6 +33,14 @@ type transientCIRerunner interface { // ledger) and before reapStuckRedPRs, whose kicks deliverReEngagement holds // while a re-run is owed. audit, when non-nil, receives one line per PR the // hive re-ran. +// +// It returns the escalation keys of the PRs whose failed jobs it re-ran this +// pass. The reaper runs next in the same cycle, on the same snapshot, where +// those PRs still read red and stale, and the re-run is no longer owed; the +// caller skips them there (reaperSkipAfterRerun) so a re-run and a fix kick +// never go out together for one head (hivecommons/hive#17002 review). A PR +// whose re-run was skipped (no failed runs, re-run forbidden) is not +// returned: the fix loop resumes on it, as intended. func rerunTransientTriagedPRs( ctx context.Context, cfg *config.Config, @@ -41,12 +49,13 @@ func rerunTransientTriagedPRs( escalatedPRs map[string]bool, audit func(repo string, number int, detail string), logger *slog.Logger, -) int { +) map[string]bool { if cfg == nil || cfg.Escalation.Disabled || rerunner == nil || actionable == nil { - return 0 + return nil } store := getEscalationStore() attempted := 0 + var reran map[string]bool for _, pr := range actionable.PRs.Items { if attempted >= transientCIRerunMaxPerPass { break @@ -90,6 +99,27 @@ func rerunTransientTriagedPRs( audit(o.Repo, o.Number, fmt.Sprintf("repo=%s, pr=%d, head=%s, runs=%d, reason=transient CI triage", o.Repo, o.Number, o.HeadSHA, len(outcome.RunIDs))) } + if reran == nil { + reran = map[string]bool{} + } + reran[key] = true + } + return reran +} + +// reaperSkipAfterRerun is the set the reaper skips this cycle: the escalated +// PRs, plus the PRs whose failed jobs rerunTransientTriagedPRs just re-ran. +// escalated is returned unchanged when nothing was re-ran. +func reaperSkipAfterRerun(escalated, reran map[string]bool) map[string]bool { + if len(reran) == 0 { + return escalated + } + skip := make(map[string]bool, len(escalated)+len(reran)) + for k, v := range escalated { + skip[k] = v + } + for k := range reran { + skip[k] = true } - return attempted + return skip } diff --git a/src/cmd/hive/transient_ci_rerun_test.go b/src/cmd/hive/transient_ci_rerun_test.go index 97f46c5473..886c6b063f 100644 --- a/src/cmd/hive/transient_ci_rerun_test.go +++ b/src/cmd/hive/transient_ci_rerun_test.go @@ -103,6 +103,48 @@ func TestRerunTransientTriagedPRs_OncePerHeadThenFixLoopResumes(t *testing.T) { } } +// hivecommons/hive#17002 review: the reaper runs right after the re-run step, +// in the same cycle and on the same snapshot, where the head still reads red +// and stale and the re-run is no longer owed. With no earlier kick (the agent +// stamped the marker in its own session), no cooldown stands in the way, so +// the cycle that re-ran the jobs must not also kick a fixer. +func TestRerunTransientTriagedPRs_SameCycleDrawsNoFixKick(t *testing.T) { + sameCycle := func(t *testing.T, skip func(escalated, reran map[string]bool) map[string]bool) int { + t.Helper() + _, clock := newTestEscalationStore(t) + cfg := escalationTestConfig() + actionable := actionableWith(transientRedPR()) + recordRedStaleness(cfg, actionable) + *clock = clock.Add(escalation.RedPRStaleAfter + time.Minute) + rerunner := &fakeTransientRerunner{out: github.TransientCIRerunOutcome{Settled: true, RunIDs: []int64{42}}} + escalated := map[string]bool{} + reran := rerunTransientTriagedPRs(context.Background(), cfg, rerunner, actionable, escalated, nil, discardLogger()) + if len(rerunner.calls) != 1 || !reran["acme/widgets#7"] { + t.Fatalf("re-run calls = %d, reran = %v; want the head re-run and reported", len(rerunner.calls), reran) + } + kicker := &fakeKicker{} + reapStuckRedPRs(cfg, actionable, skip(escalated, reran), kicker, nil, discardLogger()) + return len(kicker.kicks) + } + if got := sameCycle(t, func(escalated, _ map[string]bool) map[string]bool { return escalated }); got != 1 { + t.Fatalf("test premise: reaping with the escalated set alone kicks the re-run head (got %d kicks, want 1)", got) + } + if got := sameCycle(t, reaperSkipAfterRerun); got != 0 { + t.Fatalf("the cycle that re-ran the jobs also sent %d fix kicks, want 0", got) + } + + // A skipped re-run (no failed runs) is not reported: the fix loop resumes. + _, clock := newTestEscalationStore(t) + cfg := escalationTestConfig() + actionable := actionableWith(transientRedPR()) + recordRedStaleness(cfg, actionable) + *clock = clock.Add(escalation.RedPRStaleAfter + time.Minute) + skipped := &fakeTransientRerunner{out: github.TransientCIRerunOutcome{Settled: true, Skipped: "no-failed-runs"}} + if reran := rerunTransientTriagedPRs(context.Background(), cfg, skipped, actionable, map[string]bool{}, nil, discardLogger()); len(reran) != 0 { + t.Fatalf("a skipped re-run was reported as re-ran: %v", reran) + } +} + func TestRerunTransientTriagedPRs_Skips(t *testing.T) { t.Run("escalated PR is a person's", func(t *testing.T) { newTestEscalationStore(t) diff --git a/src/docs/adr/0010-escalation-circuit-breaker.md b/src/docs/adr/0010-escalation-circuit-breaker.md index ba3fd4f96a..5ece932454 100644 --- a/src/docs/adr/0010-escalation-circuit-breaker.md +++ b/src/docs/adr/0010-escalation-circuit-breaker.md @@ -59,7 +59,12 @@ marker, the eval cycle re-runs that head's failed jobs once reaper and merge watcher send no fix kick and charge no budget while that re-run is owed. Only markers in the hive's own comments count, because a marker now causes a write: a comment from anyone else cannot spend Actions -minutes or pause the fix loop. The re-run gives the checks new run ids, so the markers go +minutes or pause the fix loop. (A hive that cannot name any of its own +accounts, meaning no App login, `ai_author` or bot login configured and `/user` +unreachable, falls back to reading every comment, as the other +hive-authorship checks do.) The cycle that re-runs a head's jobs sends +it no fix kick either: the reaper, which runs next on the same snapshot, +skips the PRs just re-run. The re-run gives the checks new run ids, so the markers go stale on their own: a head still red afterwards is back in the ordinary loop above. The re-run is recorded per head in the ledger; a person or the reviewer lane returning an escalated PR to the lane allows one more. Before this, diff --git a/src/docs/labels-and-control-signals/glossary-and-operations.md b/src/docs/labels-and-control-signals/glossary-and-operations.md index 257d68d448..4c9f0bbfd3 100644 --- a/src/docs/labels-and-control-signals/glossary-and-operations.md +++ b/src/docs/labels-and-control-signals/glossary-and-operations.md @@ -233,7 +233,7 @@ The dashboard renders every open issue and PR's stage as one unified tracker str - **Human comments and assignees** can acknowledge hive-filed directions for #5117; assignees also affect ranking. - **Hidden comment markers** carry state that no label does. `` records an issue claim, `` dedupes a published finding, `` marks an overlap notice, `` starts the question auto-close clock, and `` is stamped on the single comment a fix lane leaves on a PR it defers to shared incident `#` (`DEFER_TO_INCIDENT` from `bin/hive-baseline-check.sh`), so "every PR incident `#` broke" stays greppable after the fix lands ([#10441](https://github.com/hivecommons/hive/issues/10441)). One marker per PR per incident; it is never edited or removed while the incident is open. While incident `#` is open, the governor records it as `deferred_incident` on the PR's `ci-failing.json` row and the kick builders keep that PR out of every CI-FAILING / FIX-BEFORE-NEW repair list, naming it only as deferred; once the incident closes the PR is listed for repair again ([#10528](https://github.com/hivecommons/hive/issues/10528)). When the incident closes (not as "not planned"), the shared-CI rerun sweep re-runs the failed jobs of each still-open PR carrying that marker — the newest failed or timed-out run per workflow on the PR head — using the hive's own credential, then stamps `` on the PR so it is re-run at most once per incident; disable with `shared_ci_rerun.enabled: false` ([#12253](https://github.com/hivecommons/hive/issues/12253)). - **``** is stamped by a fix lane on the one comment it leaves when `bin/hive-baseline-check.sh` returns `DEFER_EXTERNAL`: the repository syncs from an upstream project (its GitHub fork parent, or `--upstream`) and the same check is red on the upstream default branch, so no change here can fix it. The helper prints the exact string as `external_marker`. While every failing check run on the PR's current head carries a marker for that head and run, the governor records `deferred_external` on the PR's `ci-failing.json` row and the kick builders name the PR only as deferred under `DEFERRED AS FAILING UPSTREAM`; a push, a newly failing check or a re-run of the check (a new run id — e.g. it went green and failed again at the same head) makes the markers stale and the PR is listed for repair again ([#11609](https://github.com/hivecommons/hive/issues/11609)). -- **``** is stamped by an agent on its triage comment when it diagnoses a red check on a PR as transient or infrastructure-related (a flaky job, a runner outage, a race against a cache the default branch was still publishing) rather than caused by the diff, and cannot re-run the check itself. The fields and pinning match `hive-ci-external`; `bin/hive-baseline-check.sh --json` with the PR number prints the exact string as `transient_marker` whenever the PR's newest run of the check is red, on any verdict, and the agent copies it verbatim. While every failing check run on the PR's current head carries a marker for that head and run, the PR's `transient_ci_triaged` is set. On a hive-authored PR the fix loop then re-runs the failed jobs once for that head (`rerun-failed-jobs`, gated as `ci_rerun`) and sends no FIX-BEFORE-NEW kick while the re-run is owed; a head still red after it is back in the ordinary fix loop ([projectbluefin/common#1647](https://github.com/projectbluefin/common/pull/1647)). Only markers in the hive's own comments count here, so a comment from anyone else cannot trigger the re-run or hold the fix loop. The dependency-bot red-CI hand-off still adds `needs-human` after its grace period (dependency-bot PRs are not re-run automatically), but its comment says Hive's triage found the failure was not caused by the bump and asks for a re-run, instead of telling the owner to fix, pin or close the bump. A push or a re-run of the check (a new run id) makes the marker stale ([#12977](https://github.com/hivecommons/hive/issues/12977)). +- **``** is stamped by an agent on its triage comment when it diagnoses a red check on a PR as transient or infrastructure-related (a flaky job, a runner outage, a race against a cache the default branch was still publishing) rather than caused by the diff, and cannot re-run the check itself. The fields and pinning match `hive-ci-external`; `bin/hive-baseline-check.sh --json` with the PR number prints the exact string as `transient_marker` whenever the PR's newest run of the check is red, on any verdict, and the agent copies it verbatim. While every failing check run on the PR's current head carries a marker for that head and run, the PR's `transient_ci_triaged` is set. On a hive-authored PR the fix loop then re-runs the failed jobs once for that head (`rerun-failed-jobs`, gated as `ci_rerun`) and sends no FIX-BEFORE-NEW kick while the re-run is owed; a head still red after it is back in the ordinary fix loop ([projectbluefin/common#1647](https://github.com/projectbluefin/common/pull/1647)). Only markers in the hive's own comments count here, so a comment from anyone else cannot trigger the re-run or hold the fix loop. A hive that cannot name any of its own accounts falls back to every comment. The dependency-bot red-CI hand-off still adds `needs-human` after its grace period (dependency-bot PRs are not re-run automatically), but its comment says Hive's triage found the failure was not caused by the bump and asks for a re-run, instead of telling the owner to fix, pin or close the bump. A push or a re-run of the check (a new run id) makes the marker stale ([#12977](https://github.com/hivecommons/hive/issues/12977)). - **Paused repos** (`project.paused_repos`) stop whole-repo write/merge/enumeration paths without labels. - **Contributor queue holds** such as active leases, cooldowns, dependencies, and quota guard holds suppress contributor offers without touching GitHub labels. - **Jira/Linear labels** map only where the work-source adapter supports them: Linear skips held work using substring hold labels; Jira skips exact configured hold labels; GitHub Projects currently imports labels but does not use a hold gate. From f0f51137d7d7b0447e70aa63832391aafcdd0e12 Mon Sep 17 00:00:00 2001 From: Danathar Date: Sat, 7 Nov 2026 12:00:01 -0500 Subject: [PATCH 4/4] docs(github-app-setup): the transient-CI re-run also needs Actions write The hive reviewer on #17002 noted that the setup page named only the shared-CI rerun sweep as a consumer of Actions: Read and write. The transient re-run uses the same rerun-failed-jobs call and backoff. Without the permission it logs and returns the head to the fix loop instead of posting a note, so an operator would not otherwise learn it is degraded. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BQYnPGFJx81c8JXR3o1sg1 Signed-off-by: Danathar --- src/docs/github-app-setup.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/docs/github-app-setup.md b/src/docs/github-app-setup.md index 57de9096cf..09c45cff24 100644 --- a/src/docs/github-app-setup.md +++ b/src/docs/github-app-setup.md @@ -113,6 +113,13 @@ retries no more than once per repository per permission cooldown (one hour). An unchanged PR remains eligible after the cooldown; a successful rerun or observing no failed runs clears the recovery block. +The transient-CI re-run (a red hive PR whose failing runs all carry +`hive-ci-transient`, see ADR-0010) uses the same `rerun-failed-jobs` call and +the same per-repository refusal backoff. Without Actions write it logs +`transient CI rerun forbidden` and posts no note, and the head goes back to +the ordinary fix loop. It is turned off together with the fix loop by +`escalation.disabled`. + A maintainer may re-run the named workflow with their already-authorized identity. That manual recovery does **not** change the App's permissions. For automatic recovery, the App owner must explicitly request Actions read and