Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/fixed-16023.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- An adopted (or otherwise hive-owned) conflicting PR no longer sits on "Rebasing · attempt 1/2" while the fixer is busy: each rebase attempt is routed once and gets `governor.labels.stage.adoption.rebase_attempt_timeout` (default 15m) to push before it counts as failed and the next attempt is dispatched, the row shows the wait ("Rebasing · attempt 1/2 · waiting for scanner 12m"), and an in-place adoption whose attempts are spent re-homes to a hive-owned superseding PR at once instead of after the 3h dwell ([#16023](https://github.com/hivecommons/hive/issues/16023)).
82 changes: 82 additions & 0 deletions src/cmd/hive/adopted_pr_fixer_test.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
package main

import (
"bytes"
"log/slog"
"path/filepath"
"strings"
"testing"
"time"
Expand Down Expand Up @@ -229,3 +232,82 @@ func TestWriteMergeEligible_UntrustedConflictIsNotHiveRebaseWork(t *testing.T) {
t.Fatalf("ci_failing = %+v, want nothing when the trusted-author tier is off", failing)
}
}

// sweepRebaseLogged runs one writeMergeEligible sweep at now and returns the
// ci-failing rows and the number of "routed to fixer for rebase" lines.
func sweepRebaseLogged(t *testing.T, prs []github.PullRequest, now time.Time) ([]failingEntry, int) {
t.Helper()
dir := t.TempDir()
origMerge, origFail, origNow := mergeEligiblePath, ciFailingPath, rebaseAttemptNow
mergeEligiblePath = filepath.Join(dir, "merge-eligible.json")
ciFailingPath = filepath.Join(dir, "ci-failing.json")
rebaseAttemptNow = func() time.Time { return now }
defer func() { mergeEligiblePath, ciFailingPath, rebaseAttemptNow = origMerge, origFail, origNow }()
var logs bytes.Buffer
actionable := &github.ActionableResult{PRs: github.PRResult{Items: prs}}
writeMergeEligible(actionable, github.HoldResult{}, "hivecommons", nil, false, nil, false, nil, nil, nil, slog.New(slog.NewTextHandler(&logs, nil)))
var payload struct {
CIFailing []failingEntry `json:"ci_failing"`
}
readJSON(t, ciFailingPath, &payload)
return payload.CIFailing, strings.Count(logs.String(), "PR routed to fixer for rebase")
}

// hivecommons/hive#16023: an adopted conflicting PR whose fixer never
// pushes must not sit on attempt 1 forever. The attempt is routed (and
// logged) once; past the per-attempt deadline it counts as failed and
// attempt 2 is routed with one line; past the cap the row leaves the fixer
// list so the adoption lane can re-home it.
func TestWriteMergeEligible_AdoptedRebaseAttemptTimesOutWithoutPush(t *testing.T) {
store := useMemoryRebaseAttempts(t)
const repo = "hivecommons/hive"
adoptInPlace(t, repo, 15998, "f00dfac")
pr := github.PullRequest{Number: 15998, Repo: "hive", Author: "clanker-bot", HeadSHA: "f00dfac",
HeadRef: "feat/x", HeadRepo: "hivecommons/hive", BaseRef: "v5",
Mergeable: github.MergeableNo, MergeableState: "dirty", CIStatus: "pending",
Labels: []string{"needs-rebase", "stage/074-adopted-by-hive", "author/clanker"}}
t0 := time.Date(2026, 10, 31, 9, 0, 0, 0, time.UTC)
timeout := config.DefaultAdoptRebaseAttemptTimeout

failing, routed := sweepRebaseLogged(t, []github.PullRequest{pr}, t0)
if len(failing) != 1 || failing[0].RebaseAttempt != 1 || routed != 1 {
t.Fatalf("first sweep: rows=%+v routed=%d, want attempt 1 routed once", failing, routed)
}
// Later sweeps inside the deadline keep the row listed, not re-routed.
failing, routed = sweepRebaseLogged(t, []github.PullRequest{pr}, t0.Add(5*time.Minute))
if len(failing) != 1 || failing[0].RebaseAttempt != 1 || routed != 0 {
t.Fatalf("second sweep: rows=%+v routed=%d, want attempt 1 listed but not re-routed", failing, routed)
}
if a, _ := store.Get(repo, 15998); !a.DispatchedAt.Equal(t0) || a.Fixer != "scanner" {
t.Fatalf("dispatch = %+v, want the first sweep's dispatch to scanner kept", a)
}
if got := rebasemerge.AttemptTextAt(mustAttempt(t, store, repo, 15998), t0.Add(12*time.Minute)); got != "Rebasing · attempt 1/2 · waiting for scanner 12m" {
t.Fatalf("caption = %q", got)
}
// No push by the deadline: attempt 1 failed, attempt 2 routed once.
failing, routed = sweepRebaseLogged(t, []github.PullRequest{pr}, t0.Add(timeout))
if len(failing) != 1 || failing[0].RebaseAttempt != 2 || routed != 1 {
t.Fatalf("deadline sweep: rows=%+v routed=%d, want attempt 2 routed once", failing, routed)
}
failing, routed = sweepRebaseLogged(t, []github.PullRequest{pr}, t0.Add(timeout+time.Minute))
if len(failing) != 1 || failing[0].RebaseAttempt != 2 || routed != 0 {
t.Fatalf("attempt 2 re-sweep: rows=%+v routed=%d", failing, routed)
}
// Attempt 2 also times out: exhausted, off the fixer list.
failing, routed = sweepRebaseLogged(t, []github.PullRequest{pr}, t0.Add(2*timeout))
if len(failing) != 0 || routed != 0 {
t.Fatalf("exhausted sweep: rows=%+v routed=%d, want nothing", failing, routed)
}
if a := mustAttempt(t, store, repo, 15998); !a.Exhausted() || a.Stalled != 2 {
t.Fatalf("episode = %+v, want exhausted by two timed-out attempts", a)
}
}

func mustAttempt(t *testing.T, store *rebasemerge.Attempts, repo string, number int) rebasemerge.Attempt {
t.Helper()
a, ok := store.Get(repo, number)
if !ok {
t.Fatalf("no rebase episode for %s#%d", repo, number)
}
return a
}
47 changes: 43 additions & 4 deletions src/cmd/hive/merge_eligibility.go
Original file line number Diff line number Diff line change
Expand Up @@ -830,6 +830,19 @@ func hiveAuthoredPR(pr github.PullRequest) bool {
// rebaseAttempts is the conflict loop-guard ledger; tests swap it.
var rebaseAttempts = rebasemerge.SharedAttempts

// rebaseAttemptNow is the clock for the rebase episode; tests swap it.
var rebaseAttemptNow = time.Now

// rebaseAttemptTimeout is how long one fixer rebase attempt may wait for a
// push before it counts as failed
// (governor.labels.stage.adoption.rebase_attempt_timeout, default 15m).
func rebaseAttemptTimeout(cfg *config.Config) time.Duration {
if cfg == nil {
return config.DefaultAdoptRebaseAttemptTimeout
}
return cfg.Governor.Labels.Stage.Adoption.EffectiveRebaseAttemptTimeout()
}

// mergeTrainRebaseRequester is the RequestedBy the merge train writes on a
// rebase episode it opens for a member it could not update-branch
// (hivecommons/hive#15117 → #15104).
Expand Down Expand Up @@ -1236,13 +1249,30 @@ func writeMergeEligible(actionable *github.ActionableResult, hold github.HoldRes
// Loop guard: attempt n of MaxRebaseAttempts per conflict
// episode; a fixer's pushed head that is still dirty is the
// next attempt, and past the cap the hive stops re-kicking.
attempt := rebaseAttempts().Observe(fullRepo, pr.Number, pr.HeadSHA, time.Now())
// A fixer-owned attempt (adopted, trusted-author, requested)
// that saw no push within its deadline is the next attempt too
// (hivecommons/hive#16023): a busy fixer must not hold the
// episode at attempt 1 forever.
fixerRebase := adopted || trustedRebase || requested
rebaseNow := rebaseAttemptNow()
attempt := rebaseAttempts().Observe(fullRepo, pr.Number, pr.HeadSHA, rebaseNow)
var timedOut *rebasemerge.Attempt
if fixerRebase {
prev := attempt
if next, ok := rebaseAttempts().Expire(fullRepo, pr.Number, rebaseAttemptTimeout(cfg), rebaseNow); ok {
timedOut, attempt = &prev, next
}
}
row.RebaseAttempt = attempt.Count()
row.RebaseAttemptsMax = rebasemerge.MaxRebaseAttempts
row.BaseRef = pr.BaseRef
row.TrustedAuthor = trustedRebase
if attempt.Exhausted() {
logger.Warn("rebase attempts exhausted; leaving the conflict to a person", "repo", fullRepo, "number", pr.Number, "head", pr.HeadSHA, "attempts", attempt.Count()-1, "max", rebasemerge.MaxRebaseAttempts, "mergeable_state", pr.MergeableState)
args := []any{"repo", fullRepo, "number", pr.Number, "head", pr.HeadSHA, "attempts", attempt.Count() - 1, "max", rebasemerge.MaxRebaseAttempts, "timed_out", attempt.Stalled, "mergeable_state", pr.MergeableState}
if timedOut != nil {
args = append(args, "waited_for", timedOut.Fixer, "waited", timedOut.Waiting(rebaseNow).Round(time.Second).String())
}
logger.Warn("rebase attempts exhausted; leaving the conflict to a person", args...)
continue
}
agent := prFixAgent(pr, prAgents[fmt.Sprintf("%s#%d", fullRepo, pr.Number)])
Expand All @@ -1262,8 +1292,17 @@ func writeMergeEligible(actionable *github.ActionableResult, hold github.HoldRes
if trustedRebase || requested {
row.ReachableAction = github.ReachableActionPush
}
if adopted || trustedRebase || requested {
logger.Info("PR routed to fixer for rebase", "repo", fullRepo, "number", pr.Number, "work", "rebase", "adopted", adopted, "trusted_author", trustedRebase, "requested_by", attempt.RequestedBy, "attempt", attempt.Count(), "max", rebasemerge.MaxRebaseAttempts, "mergeable_state", pr.MergeableState, "fixer", agent, "sentinel_alert", row.SentinelAlert)
if fixerRebase {
// The row stays listed every sweep (the fixer's kick overlay
// reads it), but each attempt is routed and announced once;
// the dispatch time is the attempt's deadline clock.
if _, first := rebaseAttempts().Dispatch(fullRepo, pr.Number, agent, rebaseNow); first {
args := []any{"repo", fullRepo, "number", pr.Number, "work", "rebase", "adopted", adopted, "trusted_author", trustedRebase, "requested_by", attempt.RequestedBy, "attempt", attempt.Count(), "max", rebasemerge.MaxRebaseAttempts, "mergeable_state", pr.MergeableState, "fixer", agent, "sentinel_alert", row.SentinelAlert}
if timedOut != nil {
args = append(args, "previous_attempt_timed_out", true, "waited_for", timedOut.Fixer, "waited", timedOut.Waiting(rebaseNow).Round(time.Second).String())
}
logger.Info("PR routed to fixer for rebase", args...)
}
}
failing = append(failing, row)
continue
Expand Down
8 changes: 7 additions & 1 deletion src/docs/labels-and-control-signals/stage-labels.md
Original file line number Diff line number Diff line change
Expand Up @@ -876,7 +876,13 @@ When a contributor-agent PR goes red the hive does not just relabel it (a human-
adopted branch first reaches the normal fixer, which rebases onto the
PR base and pushes to the actual fork with a head-bound lease,
preserving authorship and sign-offs. If the head stays idle past the
074 dwell, supersede in that sweep rather than escalate to 085.
074 dwell, supersede in that sweep rather than escalate to 085. Each
fixer attempt is routed once and gets
`governor.labels.stage.adoption.rebase_attempt_timeout` (default 15m)
to push; one with no push counts as failed and the next attempt is
dispatched (the row reads "Rebasing · attempt 1/2 · waiting for
scanner 12m" meanwhile). Once both attempts are spent the adoption
supersedes at once instead of waiting out the dwell (#16023).
- `reason=in-place-stalled` (always `mode=supersede`) — second takeover:
an in-place-adopted head sat unchanged for the 074 dwell (3h) with CI
idle (not pending) and the review swarm's latest verdict on it
Expand Down
2 changes: 2 additions & 0 deletions src/hive.yaml.example
Original file line number Diff line number Diff line change
Expand Up @@ -525,6 +525,8 @@ governor:
# after_offline: 15m # default 15m; registered relay offline, measured from the later
# # of its last heartbeat and the first stage/072 nudge
# after_unregistered: 15m # default 15m; no contributor profile, wait after the nudge with no new head
# rebase_attempt_timeout: 15m # default 15m; a fixer rebase attempt with no push by then counts as failed;
# # once attempts are spent an in-place adoption re-homes to a superseding PR
# supersede:
# close_first: true # default true; close the original (hive/superseded, one note naming the
# # copy) in the pass that opens the copy; reopened if the copy fails.
Expand Down
19 changes: 16 additions & 3 deletions src/pkg/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -2930,6 +2930,13 @@ type StageAdoptionConfig struct {
// 072/085 for a contributor who cannot act (default 30m;
// hivecommons/hive#15307).
OfflineGrace string `yaml:"offline_grace,omitempty"`
// RebaseAttemptTimeout is how long one fixer rebase attempt on an
// adopted (or otherwise hive-owned) conflicting PR may wait for the
// fixer's push before it counts as failed and the next attempt is
// dispatched; past the attempt cap an in-place adoption re-homes to a
// hive-owned superseding PR at once (default 15m;
// hivecommons/hive#16023).
RebaseAttemptTimeout string `yaml:"rebase_attempt_timeout,omitempty"`
// InPlace lets the hive adopt by pushing to the PR's own branch when
// GitHub allows it (same-repo head or maintainer_can_modify) instead of
// opening a superseding App PR. nil = true.
Expand Down Expand Up @@ -3057,11 +3064,17 @@ func (c StageAdoptionConfig) StrictAttributionEnabled() bool {

// Defaults for the offline-adoption waits (stage 074, hivecommons/hive#12436).
const (
DefaultAdoptAfterOffline = 15 * time.Minute
DefaultAdoptAfterUnregistered = 15 * time.Minute
DefaultAdoptOfflineGrace = 30 * time.Minute
DefaultAdoptAfterOffline = 15 * time.Minute
DefaultAdoptAfterUnregistered = 15 * time.Minute
DefaultAdoptOfflineGrace = 30 * time.Minute
DefaultAdoptRebaseAttemptTimeout = 15 * time.Minute
)

// EffectiveRebaseAttemptTimeout parses RebaseAttemptTimeout (default 15m).
func (c StageAdoptionConfig) EffectiveRebaseAttemptTimeout() time.Duration {
return parseDurationOr(c.RebaseAttemptTimeout, DefaultAdoptRebaseAttemptTimeout)
}

// EffectiveOfflineGrace parses OfflineGrace (default 30m).
func (c StageAdoptionConfig) EffectiveOfflineGrace() time.Duration {
return parseDurationOr(c.OfflineGrace, DefaultAdoptOfflineGrace)
Expand Down
4 changes: 3 additions & 1 deletion src/pkg/config/stage_labels_getters_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ func TestStageLabelsConfigGetterDefaults(t *testing.T) {
}
if s.Adoption.EffectiveAfterOffline() != DefaultAdoptAfterOffline ||
s.Adoption.EffectiveAfterUnregistered() != DefaultAdoptAfterUnregistered ||
s.Adoption.EffectiveRebaseAttemptTimeout() != 15*time.Minute ||
s.EffectiveContributorGraceOnline() != DefaultContributorGraceOnline ||
s.EffectiveContributorGraceOffline() != DefaultContributorGraceOffline ||
s.EffectivePresenceWindow() != DefaultPresenceWindow ||
Expand All @@ -33,7 +34,7 @@ func TestStageLabelsConfigGetterDefaults(t *testing.T) {
ContributorNudge: ContributorNudgeConfig{Enabled: &off, RenudgeInterval: "30m"},
Adoption: StageAdoptionConfig{
Enabled: &off, InPlace: &off, StrictAttribution: &off, AutoMerge: &off,
AfterOffline: "2m", AfterUnregistered: "0s",
AfterOffline: "2m", AfterUnregistered: "0s", RebaseAttemptTimeout: "20m",
},
Stalled: StageStalledConfig{RequireQuiet: &off},
}
Expand All @@ -42,6 +43,7 @@ func TestStageLabelsConfigGetterDefaults(t *testing.T) {
t.Fatal("explicit false turns each toggle off")
}
if s.EffectivePresenceWindow() != time.Minute || s.Adoption.EffectiveAfterOffline() != 2*time.Minute ||
s.Adoption.EffectiveRebaseAttemptTimeout() != 20*time.Minute ||
s.ContributorNudge.EffectiveRenudgeInterval() != 30*time.Minute {
t.Fatal("valid durations parse")
}
Expand Down
1 change: 1 addition & 0 deletions src/pkg/config/testdata/agent_runtime_golden.json
Original file line number Diff line number Diff line change
Expand Up @@ -203,6 +203,7 @@
"AfterOffline": "",
"AfterUnregistered": "",
"OfflineGrace": "",
"RebaseAttemptTimeout": "",
"InPlace": null,
"StrictAttribution": null,
"AutoMerge": null,
Expand Down
7 changes: 4 additions & 3 deletions src/pkg/dashboard/rebase_merge_followthrough.go
Original file line number Diff line number Diff line change
Expand Up @@ -381,8 +381,9 @@ type StageTrackerRebase struct {
}

// applyRebaseAttempts stamps rebase onto PR rows with an open conflict
// episode in the fixer's attempts ledger.
func applyRebaseAttempts(items []StageTrackerItem) {
// episode in the fixer's attempts ledger; a dispatched attempt names the
// fixer and how long it has waited at now (hivecommons/hive#16023).
func applyRebaseAttempts(items []StageTrackerItem, now time.Time) {
attempts := rebaseAttemptsStore()
if attempts == nil {
return
Expand All @@ -396,7 +397,7 @@ func applyRebaseAttempts(items []StageTrackerItem) {
if !ok {
continue
}
it.Rebase = &StageTrackerRebase{Attempt: min(max(a.Count(), 1), rebasemerge.MaxRebaseAttempts), Max: rebasemerge.MaxRebaseAttempts, Exhausted: a.Exhausted(), Head: rebaseMergeShortSHA(a.Head()), Text: rebasemerge.AttemptText(a)}
it.Rebase = &StageTrackerRebase{Attempt: min(max(a.Count(), 1), rebasemerge.MaxRebaseAttempts), Max: rebasemerge.MaxRebaseAttempts, Exhausted: a.Exhausted(), Head: rebaseMergeShortSHA(a.Head()), Text: rebasemerge.AttemptTextAt(a, now)}
}
}

Expand Down
14 changes: 10 additions & 4 deletions src/pkg/dashboard/rebase_merge_followthrough_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -215,18 +215,24 @@ func TestRebaseAttemptsPillAndFollowThroughExhaustion(t *testing.T) {
t.Cleanup(func() { rebaseMergeNow = time.Now })

items := []StageTrackerItem{{Repo: "hivecommons/hive", Number: 15087, Kind: "pr", Stage: 90, Modifiers: []string{modNeedsRebase}}, {Repo: "hivecommons/hive", Number: 1, Kind: "pr", Stage: 90}}
applyRebaseAttempts(items)
applyRebaseAttempts(items, now)
if items[0].Rebase != nil || items[1].Rebase != nil {
t.Fatalf("no episode, no pill: %+v %+v", items[0].Rebase, items[1].Rebase)
}
attempts.Observe("hivecommons/hive", 15087, "aaa1111aaa1111", now)
applyRebaseAttempts(items)
applyRebaseAttempts(items, now)
if r := items[0].Rebase; r == nil || r.Attempt != 1 || r.Max != 2 || r.Exhausted || r.Head != "aaa1111" || r.Text != "Rebasing · attempt 1/2" {
t.Fatalf("attempt 1 pill = %+v", r)
}
// hivecommons/hive#16023: a dispatched attempt shows who it waits on.
attempts.Dispatch("hivecommons/hive", 15087, "scanner", now)
applyRebaseAttempts(items, now.Add(12*time.Minute))
if r := items[0].Rebase; r == nil || r.Attempt != 1 || r.Text != "Rebasing · attempt 1/2 · waiting for scanner 12m" {
t.Fatalf("waiting pill = %+v", r)
}
attempts.Observe("hivecommons/hive", 15087, "bbb2222", now)
attempts.Observe("hivecommons/hive", 15087, "ccc3333", now)
applyRebaseAttempts(items)
applyRebaseAttempts(items, now)
if r := items[0].Rebase; r == nil || !r.Exhausted || r.Attempt != 2 || r.Text != "Rebase failed · 2/2 — needs a person" {
t.Fatalf("exhausted pill = %+v", r)
}
Expand Down Expand Up @@ -299,7 +305,7 @@ func TestRepoItemDecisionApproveOnDirtyPRQueuesRebase(t *testing.T) {
t.Fatalf("a dirty head gets no update-branch: %d", mock.updates)
}
items := []StageTrackerItem{{Repo: "testorg/testrepo", Number: 8, Kind: "pr", Stage: 85}}
applyRebaseAttempts(items)
applyRebaseAttempts(items, time.Now())
if items[0].Rebase == nil || items[0].Rebase.Text != "Rebasing · attempt 1/2" {
t.Fatalf("row = %+v", items[0].Rebase)
}
Expand Down
2 changes: 1 addition & 1 deletion src/pkg/dashboard/stage_tracker_build.go
Original file line number Diff line number Diff line change
Expand Up @@ -490,7 +490,7 @@ func (s *Server) buildStageTracker(status *StatusPayload, store *stagelabels.Sto
resp.Items = stageTrackerDropOperatorClosed(resp.Items, now, operatorActionClosedGrace())
resp.OpenIssues, resp.OpenPRs = stageTrackerOpenCounts(resp.Items)
s.applyDecidedProjection(resp.Items, now)
applyRebaseAttempts(resp.Items)
applyRebaseAttempts(resp.Items, now)
s.applyRebaseMergeFollowThrough(resp.Items, now)
resp.NeedsHuman, resp.OnHold = stageTrackerHumanCounts(resp.Items)
resp.WaitingOnHuman = stageTrackerWaitingOnHuman(resp.Items)
Expand Down
Loading
Loading