Skip to content
1 change: 1 addition & 0 deletions changelog.d/fixed-transient-ci-rerun.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- A red hive pull request whose failure an agent triaged as infrastructure is now re-run instead of sent back to a fixer. On [projectbluefin/common#1647](https://github.com/projectbluefin/common/pull/1647) the agent said "infrastructure, not this change" four times, the fix loop still counted six fix re-dispatches on that one commit and escalated the PR to a person, and a lane then pushed an empty "retrigger CI" commit. When every failing check run on the head carries a `hive-ci-transient` marker, Hive now re-runs the failed jobs once for that head (`rerun-failed-jobs`, through the write-back gate as `ci_rerun`) and sends no fix kick while that re-run is owed. A head still red after the re-run goes back to the normal fix loop. The fix kick now tells the agent to stamp the marker and never push to retrigger CI, and the reviewer lane's DE-ESCALATE verdict no longer asks for a push when the branch is already current.
18 changes: 17 additions & 1 deletion src/cmd/hive/main_eval_cycle.go
Original file line number Diff line number Diff line change
Expand Up @@ -798,7 +798,23 @@ func runEvalCycle(
// re-engagement budget only for kicks that were accepted. Composes with the
// merge watcher (#2): both go through the same per-red-SHA cap, so the same
// PR is never double-dispatched within a red-SHA's budget.
reapStuckRedPRs(cfg, actionable, escalatedPRs, agentKicker{mgr: agentMgr},
//
// A head an agent triaged as transient infrastructure (hive-ci-transient
// markers) is re-run once here instead: the reaper and the merge watcher
// hold their fix kicks while that re-run is owed
// (projectbluefin/common#1647).
reapSkip := escalatedPRs
if ghClient != nil {
var audit func(string, int, string)
if dashSrv != nil {
audit = func(repo string, number int, detail string) {
dashSrv.AuditLogRecord("system", "transient-ci-rerun", detail, "", repo, number)
}
}
reran := rerunTransientTriagedPRs(ctx, cfg, ghClient, actionable, escalatedPRs, audit, logger)
reapSkip = reaperSkipAfterRerun(escalatedPRs, reran)
}
reapStuckRedPRs(cfg, actionable, reapSkip, agentKicker{mgr: agentMgr},
agentAvailability(cfg, agentMgr), logger)

// Release sentinel (hivecommons/hive#9585), opt-in and default OFF: when
Expand Down
25 changes: 23 additions & 2 deletions src/cmd/hive/main_sweeps.go
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,7 @@ func escalationObservation(cfg *config.Config, pr github.PullRequest, owners map
FailingChecks: pr.FailingChecks,
Agent: owners[escalation.Key(repo, pr.Number)],
IndependentControl: isIndependentCIControl(cfg, pr),
TransientTriaged: red && pr.TransientCITriaged,
}
}

Expand Down Expand Up @@ -360,8 +361,17 @@ func redPRFixKick(o escalation.Observation, rerouted bool) string {
fmt.Fprintf(&b, " gh pr checkout %d --repo %s → fix → git commit -s → git push\n", o.Number, o.Repo)
b.WriteString("Push to the SAME branch. Do NOT open a replacement PR.\n")
b.WriteString("After the push, STOP: do not watch, poll, or sleep on CI (no gh run watch/view loops).\n")
b.WriteString("The automerge sweep merges the PR once it is green; if the failure is infrastructure\n")
b.WriteString("(runner lost, no space left, checks still queued) leave one comment and end the turn.\n")
b.WriteString("The automerge sweep merges the PR once it is green; if checks are still queued, end the turn.\n")
// An infra verdict must reach the hive as a marker, not prose: four
// "infrastructure, not this change" comments on projectbluefin/common#1647
// were invisible to the fix loop, which kicked again until the PR
// escalated, and a lane then pushed an empty "retrigger CI" commit.
b.WriteString("If the failure is infrastructure, not this change (runner lost, no space left, a package\n")
b.WriteString("mirror or download dropping the connection): do NOT push anything — no empty, no-op or\n")
b.WriteString("\"retrigger CI\" commit; a push re-arms review fences and can dismiss approvals. Leave ONE\n")
fmt.Fprintf(&b, "comment with the evidence that ends with the transient_marker printed by\n hive-baseline-check.sh %s '<failing check>' %d --json\n", o.Repo, o.Number)
b.WriteString("(one per failing check), then end the turn. Once every failing check carries one, Hive\n")
b.WriteString("re-runs the failed jobs itself, once.\n")
if o.HeadSHA != "" {
fmt.Fprintf(&b, "\nhead: %s\n", o.HeadSHA)
}
Expand Down Expand Up @@ -436,6 +446,17 @@ func deliverReEngagement(
available func(string) bool,
logger *slog.Logger,
) bool {
if store.TransientRerunOwed(o.Repo, o.Number, headSHA) {
// An agent already triaged this head's failure as infrastructure;
// a fix kick could only produce the same comment again and spend
// the budget that escalates the PR (projectbluefin/common#1647).
// rerunTransientTriagedPRs re-runs the failed jobs instead.
if logger != nil {
logger.Debug("re-engagement held: failure triaged as transient, hive re-runs the failed jobs",
"repo", o.Repo, "pr", o.Number)
}
return false
}
fixer := resolveRedPRFixer(cfg, o.Agent, available)
if fixer == "" {
// Nobody can be asked right now. Charging budget here is what escalated
Expand Down
125 changes: 125 additions & 0 deletions src/cmd/hive/transient_ci_rerun.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,125 @@
package main

import (
"context"
"fmt"
"log/slog"

"github.com/hivecommons/hive/pkg/config"
"github.com/hivecommons/hive/pkg/escalation"
"github.com/hivecommons/hive/pkg/github"
)

// transientCIRerunMaxPerPass bounds the re-run calls one eval cycle makes.
const transientCIRerunMaxPerPass = 10

// transientCIRerunner re-runs a transient-triaged PR's failed jobs;
// *github.Client implements it.
type transientCIRerunner interface {
RerunTransientCIFailures(ctx context.Context, pr github.PullRequest) (github.TransientCIRerunOutcome, error)
}

// rerunTransientTriagedPRs is the fix loop's answer to a red hive-authored PR
// whose every failing check run an agent triaged as infrastructure, not this
// change (`hive-ci-transient` markers, github.PullRequest.TransientCITriaged):
// it re-runs the failed jobs once per head through rerun-failed-jobs, and
// never kicks a fixer or pushes a commit for it. On
// projectbluefin/common#1647 the fix loop instead kicked the owning agent six
// times on one head, each kick answered with "infrastructure, not this
// change", escalated the PR to a person, and a lane then pushed an empty
// "retrigger CI" commit.
//
// It runs after recordRedStaleness (which records the triaged head in the
// ledger) and before reapStuckRedPRs, whose kicks deliverReEngagement holds
// while a re-run is owed. audit, when non-nil, receives one line per PR the
// hive re-ran.
//
// It returns the escalation keys of the PRs whose failed jobs it re-ran this
// pass. The reaper runs next in the same cycle, on the same snapshot, where
// those PRs still read red and stale, and the re-run is no longer owed; the
// caller skips them there (reaperSkipAfterRerun) so a re-run and a fix kick
// never go out together for one head (hivecommons/hive#17002 review). A PR
// whose re-run was skipped (no failed runs, re-run forbidden) is not
// returned: the fix loop resumes on it, as intended.
func rerunTransientTriagedPRs(
ctx context.Context,
cfg *config.Config,
rerunner transientCIRerunner,
actionable *github.ActionableResult,
escalatedPRs map[string]bool,
audit func(repo string, number int, detail string),
logger *slog.Logger,
) map[string]bool {
if cfg == nil || cfg.Escalation.Disabled || rerunner == nil || actionable == nil {
return nil
}
store := getEscalationStore()
attempted := 0
var reran map[string]bool
for _, pr := range actionable.PRs.Items {
if attempted >= transientCIRerunMaxPerPass {
break
}
if !isHiveAgentAuthor(cfg, pr.Author) || !pr.TransientCITriaged || pr.CIChecksRunning || !pr.HasFailingRequiredCheck() {
continue
}
o := escalationObservation(cfg, pr, nil)
key := escalation.Key(o.Repo, o.Number)
if escalatedPRs[key] || escalation.HasNeedsHumanLabel(pr.Labels) {
continue
}
if !store.TransientRerunOwed(o.Repo, o.Number, o.HeadSHA) {
continue
}
attempted++
outcome, err := rerunner.RerunTransientCIFailures(ctx, pr)
if err != nil {
if logger != nil {
logger.Warn("transient CI rerun: could not list failed runs; retrying next pass",
"repo", o.Repo, "pr", o.Number, "head", o.HeadSHA, "error", err)
}
continue
}
if !outcome.Settled {
continue
}
store.RecordTransientRerun(o.Repo, o.Number, o.HeadSHA)
if len(outcome.RunIDs) == 0 {
if logger != nil {
logger.Info("transient CI rerun skipped; the fix loop resumes on this head",
"repo", o.Repo, "pr", o.Number, "head", o.HeadSHA, "reason", outcome.Skipped)
}
continue
}
if logger != nil {
logger.Info("transient CI rerun: re-ran failed jobs instead of a fix kick",
"repo", o.Repo, "pr", o.Number, "head", o.HeadSHA, "runs", outcome.RunIDs)
}
if audit != nil {
audit(o.Repo, o.Number, fmt.Sprintf("repo=%s, pr=%d, head=%s, runs=%d, reason=transient CI triage",
o.Repo, o.Number, o.HeadSHA, len(outcome.RunIDs)))
}
if reran == nil {
reran = map[string]bool{}
}
reran[key] = true
}
return reran
}

// reaperSkipAfterRerun is the set the reaper skips this cycle: the escalated
// PRs, plus the PRs whose failed jobs rerunTransientTriagedPRs just re-ran.
// escalated is returned unchanged when nothing was re-ran.
func reaperSkipAfterRerun(escalated, reran map[string]bool) map[string]bool {
if len(reran) == 0 {
return escalated
}
skip := make(map[string]bool, len(escalated)+len(reran))
for k, v := range escalated {
skip[k] = v
}
for k := range reran {
skip[k] = true
}
return skip
}
Loading
Loading