diff --git a/changelog.d/fixed-front-only-base-refresh.md b/changelog.d/fixed-front-only-base-refresh.md new file mode 100644 index 0000000000..9c8f7be0c0 --- /dev/null +++ b/changelog.d/fixed-front-only-base-refresh.md @@ -0,0 +1 @@ +- Serialize automatic final base refreshes per repository/base across merge lanes, preserve pending preparation outside ready train seats, and allow already-current, CI-green, reviewed successors to merge without waiting for a preparing front. diff --git a/src/cmd/hive/github_client_config.go b/src/cmd/hive/github_client_config.go index 5e13abb93c..c8cb2b5498 100644 --- a/src/cmd/hive/github_client_config.go +++ b/src/cmd/hive/github_client_config.go @@ -115,6 +115,7 @@ func (b *boot) applyGitHubClientConfigHooks(client *github.Client) { // pause taken in the dashboard narrows the very next enumeration and // automerge sweep without a restart. client.SetRepoPausedFunc(b.cfg.IsRepoPaused) + client.SetScheduledBaseRefresh(func(repo string) bool { return b.cfg.AutoMerge.RequireCurrentBaseFor(repo) }) // Per-repo custom agents (#6204). A live predicate over the shared // config, so the hive-open-pr / hive-merge / hive-open-issue relays // refuse an out-of-scope request without a restart. diff --git a/src/cmd/hive/prfollowupwire.go b/src/cmd/hive/prfollowupwire.go index cfa26f0456..4e3a2a09ef 100644 --- a/src/cmd/hive/prfollowupwire.go +++ b/src/cmd/hive/prfollowupwire.go @@ -165,7 +165,7 @@ func routePRFollowUps(ctx context.Context, cfg *config.Config, client *github.Cl Logger: logger, } opts.BaseSyncEnabled = func(repo string) bool { - return cfg != nil && cfg.Review.ContributorPRs.BaseSync + return cfg != nil && cfg.Review.ContributorPRs.BaseSync && !cfg.AutoMerge.RequireCurrentBaseFor(repo) } if client != nil { opts.BranchUpdater = client diff --git a/src/docs/hive-merge.md b/src/docs/hive-merge.md index e0831daf13..47492d3689 100644 --- a/src/docs/hive-merge.md +++ b/src/docs/hive-merge.md @@ -361,6 +361,65 @@ gate, saying it will wait for a human. The rule exists because a ⚠️-titled PR was auto-merged by the trusted-author lane while its owner was still deciding whether to ship it. +### Final base preparation: one refresh owner, no ready-work blockade + +For repositories governed by `auto_merge.require_current_base` (including +the inherited default), automatic base refreshes share **one preparation +owner per canonical repository and destination branch**, across App, +trusted-author and contributor lanes. Repository names are case-insensitive; +branch names remain case-sensitive. Unrelated destinations progress independently. + +A behind head enters final preparation only after its existing CI, review +and lane admission gates pass, **without first requiring an up-to-date base**. +The owner refreshes with an expected-head SHA and then waits for CI and review +on the new head. Pending runner jobs (including queued sign-off checks) are +pending, not failures. Preparation is outside ready merge-train membership: +`base-refresh:preparing` and `base-refresh:queued-behind#N` remain at CI / Review, +never ready-to-merge. Waiting heads may continue development and review. +The adopted stale-red recovery path can also admit its existing, proven +base-fixes-failure repair; that admission never authorizes a merge. +Explicit trusted-human merge-forward repair shares the same owner and +write-ahead identity. Its conflict-repair allowance lasts only through the +existing mutation-reconciliation TTL; a failed/conflicting result returns to +the normal repair path, never a ready seat. + +**Ownership serializes refreshes, not ready merges.** A successor that is +already current-base, CI-green and reviewed passes every normal merge gate +without waiting for a preparing owner. A behind successor cannot skip testing. +Train sibling refreshes and client-side follow-up/relay/supersession updates +defer to preparation on these repositories; opted-out repositories retain +their previous behavior. + +Ownership and first-admission FIFO tickets live in `/data/base-preparation/`, +with per-destination file locking and atomic durable snapshots. An update +identity (PR/head/base SHA) is recorded before the request, across lanes, +concurrent sweeps and restarts. An unchanged unresolved pair waits; after the +existing mutation-reconciliation TTL it can retry only with an authoritative, +still-identical head and the API's expected-head guard. Accepted updates must +complete their normal current-head gates; no approval is transferred. +Release occurs on merge, closed/missing/retargeted/draft/conflicting/held heads, +failed checks, rejected review, or lost admission. Expected pending CI/review +keeps ownership, bounded by the existing review-pending dwell (its default when +stall detection is disabled), so a genuinely hung owner yields to successors. +Waiting tickets are removed when their head changes or stops qualifying. +Admission provenance prevents a different lane's non-admission from revoking +the owning lane's reservation or FIFO ticket. An old waiting ticket reserves +order only: its head must qualify afresh in its lane before becoming owner. + +Audit logs use `automerge base preparation selected`, `refresh claimed`, +`waiting` and `released`, with repository, base, PR, head/base SHA where relevant, +and release/skip reason. `automerge update-branch` and the sweep's +`updated_branches` count actual successful refresh effects; queued/preparing +reasons do not consume the branch-update budget. Ready-only train seating and +reviewer dispatch scheduling are separate policies. + +Human pushes, direct merges and external automation can still move a base. +Such a move is coalesced until the preparation owner qualifies again; it must +then refresh and re-pass current-head gates if still behind. This scheduler +cannot stop external writers and does not relax strict current-base checks. +Deploying the changed hive binary activates the policy; no agent, memory, +review, CI, hold or authorization setting needs changing. + ### Merge train: ready PRs lead Every lane groups its candidates on one base whose changed files overlap @@ -370,8 +429,9 @@ reason `merge-train:queued-behind#:/:…` and, the moment the front merges, are update-branched (`behind`) or handed to the rebase resolver (`dirty`) so the next front is evaluated on a current base. Hive's own repo also runs the current-base gate (`require_current_base`), which -holds a behind head until the update lands; the two cooperate — the train -refreshes, the gate waits for the refresh. +holds a behind head until the update lands. On current-base-gated repositories +the separate preparation owner above refreshes the next admitted head, rather +than refreshing every waiting sibling when a front merges. The train's order is **readiness first**. Each pass the lane sorts the members by what it last saw of their heads: a head at the merge gate — diff --git a/src/pkg/dashboard/stage_tracker_base_preparation_test.go b/src/pkg/dashboard/stage_tracker_base_preparation_test.go new file mode 100644 index 0000000000..f29e95a95d --- /dev/null +++ b/src/pkg/dashboard/stage_tracker_base_preparation_test.go @@ -0,0 +1,15 @@ +package dashboard + +import "testing" + +func TestBasePreparationCaptionNeverAdvertisesReady(t *testing.T) { + for reason, want := range map[string]string{ + "base-refresh:queued-behind#42": "queued for base refresh behind PR #42", + "base-refresh:queued-behind#0": "waiting for admission to final base preparation", + "base-refresh:preparing": "preparing the current base; current-head CI and review must complete", + } { + if got := humanizeStageReason(reason); got != want { + t.Errorf("%s caption=%q, want %q", reason, got, want) + } + } +} diff --git a/src/pkg/dashboard/stage_tracker_caption.go b/src/pkg/dashboard/stage_tracker_caption.go index 1104669fa1..19e24c2017 100644 --- a/src/pkg/dashboard/stage_tracker_caption.go +++ b/src/pkg/dashboard/stage_tracker_caption.go @@ -314,6 +314,15 @@ func humanizeStageReason(reason string) string { return clause + " — approved by @" + approver } lower := strings.ToLower(r) + if front, ok := strings.CutPrefix(r, "base-refresh:queued-behind#"); ok { + if front == "0" { + return "waiting for admission to final base preparation" + } + return "queued for base refresh behind PR #" + front + } + if r == "base-refresh:preparing" { + return "preparing the current base; current-head CI and review must complete" + } switch lower { case "actionable", "live agent claim", "live issue claim", "open pr covers it": return "" diff --git a/src/pkg/github/automerge/automerge_adoption_recovery.go b/src/pkg/github/automerge/automerge_adoption_recovery.go index 46eaa89f54..62daa9627d 100644 --- a/src/pkg/github/automerge/automerge_adoption_recovery.go +++ b/src/pkg/github/automerge/automerge_adoption_recovery.go @@ -47,6 +47,9 @@ type adoptionRecovery struct { // Attempted means an update-branch or re-run was issued: the escalation // waits for CI on the result. Attempted bool + // Deferred means preparation ownership withheld the request, so no + // recovery attempt or lifetime budget was consumed. + Deferred bool // Reason is the structured outcome: update-branch, rerun-failed-jobs or // skipped:. Reason string @@ -213,14 +216,25 @@ func (c *Engine) recoverStaleRedAdoption(ctx context.Context, store *stagelabels c.info("adopted PR recovery", logArgs("reason", out.Reason, "base_sha", baseSHA, "failing_check", failing)...) return out } - store.MarkAdoptionRecovery(key, obs.HeadSHA, baseSHA) behind := cmp.GetBehindBy() if behind > 0 { + if c.requireCurrentBaseFor(obs.Owner + "/" + obs.Repo) { + // Existing recovery evidence already proved that the new base + // clears this failure. This admits repair, never a merge: the + // normal current-head CI and review gates still run afterward. + if reason, err := c.admitBasePreparation(ctx, obs.Owner, obs.Repo, baseRef, obs.Number, obs.HeadSHA, obs.Lane, false); reason != "" || err != nil { + return adoptionRecoverySkipped(nonEmpty(reason, "base-refresh-state-unavailable")) + } + } out, done := c.recoverByUpdateBranch(ctx, store, key, obs, logArgs("base_sha", baseSHA, "behind_by", behind, "failing_check", failing)) if done { + if !out.Deferred { + store.MarkAdoptionRecovery(key, obs.HeadSHA, baseSHA) + } return out } } + store.MarkAdoptionRecovery(key, obs.HeadSHA, baseSHA) return c.recoverByRerun(ctx, obs, logArgs("base_sha", baseSHA, "behind_by", behind, "failing_check", failing)) } @@ -230,6 +244,9 @@ func (c *Engine) recoverStaleRedAdoption(ctx context.Context, store *stagelabels func (c *Engine) recoverByUpdateBranch(ctx context.Context, store *stagelabels.Store, key string, obs stageObservation, logArgs []any) (adoptionRecovery, bool) { fork := obs.HeadRepo != "" && !strings.EqualFold(obs.HeadRepo, obs.Owner+"/"+obs.Repo) if err := c.executeUpdateBranchEffect(ctx, obs.Owner, obs.Repo, obs.Number, obs.HeadSHA, adoptionRecoveryClaimReason); err != nil { + if wait := preparationWaitReason(err); wait != "" { + return adoptionRecovery{Deferred: true, Reason: adoptionRecoverySkipPrefix + wait}, true + } var out adoptionRecovery switch classifyUpdateBranchError(err, fork) { case updateBranchUpToDate: diff --git a/src/pkg/github/automerge/automerge_adoption_recovery_test.go b/src/pkg/github/automerge/automerge_adoption_recovery_test.go index a1bd1495b6..0ff1fd3f93 100644 --- a/src/pkg/github/automerge/automerge_adoption_recovery_test.go +++ b/src/pkg/github/automerge/automerge_adoption_recovery_test.go @@ -1,6 +1,7 @@ package automerge import ( + "context" "net/http" "testing" "time" @@ -150,6 +151,48 @@ func TestAdoptedStaleRedRecovery(t *testing.T) { } } +func TestAdoptedRecoveryPreparationWaitDoesNotConsumeAttempt(t *testing.T) { + for _, duplicate := range []bool{false, true} { + t.Run(map[bool]string{false: "queued", true: "duplicate"}[duplicate], func(t *testing.T) { + h, key, done := recoveryHarness(t) + defer done() + h.e.requireCurrentBase = func(string) bool { return true } + h.e.basePreparation = NewBasePreparationStore("") + h.adopt.compareBehind = 2 + h.adopt.baseChecks = map[string]string{"coverage": "success"} + front, head, wantReason := 8, "head8", preparationQueued(8) + if duplicate { + front, head, wantReason = 7, "aaa", basePreparationReady + } + h.adopt.superseders[front] = map[string]any{ + "number": front, "state": "open", + "head": map[string]string{"sha": head}, + "base": map[string]string{"ref": "v5", "sha": "base1"}, + } + if reason, err := h.e.admitBasePreparation(context.Background(), "acme", "widget", "v5", front, head, stageLaneContributor, false); reason != "" || err != nil { + t.Fatalf("admit owner: reason=%q error=%v", reason, err) + } + if duplicate { + if err := h.e.claimBasePreparationUpdate(context.Background(), "acme", "widget", front, head); err != nil { + t.Fatal(err) + } + } + rec, _ := h.e.stageStoreOrShared().Get(key) + out := h.e.recoverStaleRedAdoption(context.Background(), h.e.stageStoreOrShared(), key, stageObservation{ + Owner: "acme", Repo: "widget", Display: "acme/widget", Number: 7, + BaseRef: "v5", HeadSHA: "aaa", Lane: stageLaneContributor, Reason: recoveryReason, + }, rec) + if out.Attempted || out.Reason != adoptionRecoverySkipPrefix+wantReason || out.Deferred != duplicate { + t.Fatalf("recovery while waiting: %+v", out) + } + rec, _ = h.e.stageStoreOrShared().Get(key) + if rec.AdoptRecoveryCount != 0 || rec.AdoptRecoveryAttempt != "" || len(h.adopt.branchUpdates) != 0 { + t.Fatalf("waiting recovery consumed attempt: record=%+v updates=%v", rec, h.adopt.branchUpdates) + } + }) + } +} + // The same head stays red after the re-run: one attempt per (head, base), // then the escalation runs; a new hive head starts over. func TestAdoptedStaleRedRecoveryEscalatesAfterOneAttempt(t *testing.T) { diff --git a/src/pkg/github/automerge/automerge_base_preparation.go b/src/pkg/github/automerge/automerge_base_preparation.go new file mode 100644 index 0000000000..1fa2830ea4 --- /dev/null +++ b/src/pkg/github/automerge/automerge_base_preparation.go @@ -0,0 +1,291 @@ +package automerge + +import ( + "context" + "errors" + "fmt" + "net/http" + "path/filepath" + "slices" + "strings" + "time" + + gh "github.com/google/go-github/v72/github" + "github.com/hivecommons/hive/pkg/effects" + hgithub "github.com/hivecommons/hive/pkg/github" + "github.com/hivecommons/hive/pkg/stageengine" + "github.com/hivecommons/hive/pkg/stagelabels" +) + +const ( + basePreparationPrefix = "base-refresh:" + basePreparationReady = basePreparationPrefix + "preparing" +) + +var sharedBasePreparation = NewBasePreparationStore(filepath.Join(stagelabels.DefaultStateDir, "base-preparation")) + +type preparationWait struct{ reason string } + +func (e *preparationWait) Error() string { return e.reason } + +func preparationWaitReason(err error) string { + var wait *preparationWait + if errors.As(err, &wait) { + return wait.reason + } + return "" +} + +func (c *Engine) preparationStore() *BasePreparationStore { + if c.basePreparation != nil { + return c.basePreparation + } + return sharedBasePreparation +} + +func (c *Engine) preparationWindow() time.Duration { + // Use the existing CI/review dwell, including the operator's override. + // A disabled stall clock must not disable bounded ownership. + d := stagelabels.Dwell(stagelabels.ReviewPending, c.stagePolicy().Dwell) + if d <= 0 { + d = stagelabels.ReviewPending.Dwell() + } + return d +} + +// preparationInvalid checks live terminal gates without requiring green CI or +// a new verdict: those are expected to become pending after our refresh. +func (c *Engine) preparationInvalid(ctx context.Context, owner, repo, base string, number int, expectedHead string, repair bool) (string, error) { + pr, _, err := c.gh.PullRequests.Get(hgithub.WithRESTCaller(ctx, "hive:base_preparation"), owner, repo, number) + if isGitHubStatus(err, http.StatusNotFound) { + return "gone", nil + } + if err != nil { + return "", err + } + if expectedHead != "" && pr.GetHead().GetSHA() != expectedHead { + return "head-changed-before-preparation", nil + } + return c.preparationInvalidPR(pr, base, repair), nil +} + +func (c *Engine) preparationInvalidPR(pr *gh.PullRequest, base string, repair bool) string { + switch { + case !strings.EqualFold(pr.GetState(), "open"): + return "closed" + case pr.GetBase().GetRef() != base: + return "retargeted" + case pr.GetDraft(): + return "draft" + case prMergeableStateConflicting(pr) && !repair: + return "conflicting" + case c.isHeld(labelNames(pr.Labels)): + return "held" + case c.transport.IsExemptLabels(labelNames(pr.Labels)): + return "exempt-label" + } + if reason := c.humanMergeRequiredReason(pr.GetTitle(), labelNames(pr.Labels)); reason != "" { + return reason + } + return "" +} + +func preparationQueued(front int) string { + return fmt.Sprintf("%squeued-behind#%d", basePreparationPrefix, front) +} + +// admitBasePreparation is called only after admission gates, not after the +// current-base gate: a green/reviewed behind head can therefore enter. The +// durable owner is not a ready merge-train seat. +func (c *Engine) admitBasePreparation(ctx context.Context, owner, repo, base string, number int, head, lane string, repair bool) (string, error) { + reason := "" + now := c.now() + key := preparationKey(owner, repo, base) + err := c.preparationStore().change(key, func(st *basePreparation) error { + if st.Owner.Number != 0 { + repairing := st.Owner.Repair && now.Sub(st.Owner.Since) < effects.DefaultUnresolvedTTL + why, err := c.preparationInvalid(ctx, owner, repo, base, st.Owner.Number, "", repairing) + if err != nil { + return err + } + if why == "" && now.Sub(st.Owner.Since) >= c.preparationWindow() { + why = "preparation-expired" + } + if why != "" { + c.info("automerge base preparation released", "repo", owner+"/"+repo, "base", base, "pr", st.Owner.Number, "reason", why) + st.Owner = preparationCandidate{} + } + } + if st.Owner.Number == number { + return nil // sticky through expected pending checks and review + } + i := slices.IndexFunc(st.Waiting, func(w preparationCandidate) bool { return w.Number == number }) + if i < 0 { + st.Waiting = append(st.Waiting, preparationCandidate{Number: number, Head: head, Lane: lane, Repair: repair, Since: now}) + } else { + st.Waiting[i].Head = head + } + if st.Owner.Number == 0 { + for len(st.Waiting) > 0 { + next := st.Waiting[0] + why, err := c.preparationInvalid(ctx, owner, repo, base, next.Number, next.Head, next.Repair) + if err != nil { + return err + } + if why != "" || now.Sub(next.Since) >= c.preparationWindow() { + st.Waiting = st.Waiting[1:] + continue + } + if next.Number != number { + // Only the caller has just passed its lane's gates. + // Never turn an old FIFO ticket into a sticky owner. + reason = preparationQueued(next.Number) + break + } + st.Waiting = st.Waiting[1:] + next.Since = now + next.Lane = lane + next.Repair = repair + st.Owner = next + c.info("automerge base preparation selected", "repo", owner+"/"+repo, "base", base, "pr", next.Number, "head", next.Head, "lane", next.Lane) + break + } + } + if st.Owner.Number != number { + if reason == "" { + reason = preparationQueued(st.Owner.Number) + } + front := st.Owner.Number + if front == 0 && len(st.Waiting) > 0 { + front = st.Waiting[0].Number + } + c.info("automerge base preparation waiting", "repo", owner+"/"+repo, "base", base, "pr", number, "front", front, "reason", reason) + } + return nil + }) + return reason, err +} + +// claimBasePreparationUpdate writes the attempt BEFORE issuing update-branch. +// Neither a new Engine, another lane nor a pod restart repeats that head/base +// pair while it is unresolved. After the existing mutation reconciliation TTL, +// only a still-identical authoritative head permits a head-pinned retry. +func (c *Engine) claimBasePreparationUpdate(ctx context.Context, owner, repo string, number int, head string) error { + pr, _, err := c.gh.PullRequests.Get(hgithub.WithRESTCaller(ctx, "hive:base_preparation_update"), owner, repo, number) + if err != nil { + return err + } + base, baseSHA := pr.GetBase().GetRef(), pr.GetBase().GetSHA() + if head == "" || pr.GetHead().GetSHA() != head || base == "" || baseSHA == "" { + return &preparationWait{basePreparationPrefix + "snapshot-changed"} + } + var wait string + err = c.preparationStore().change(preparationKey(owner, repo, base), func(st *basePreparation) error { + if st.Owner.Number != number { + wait = preparationQueued(st.Owner.Number) + return nil + } + if why := c.preparationInvalidPR(pr, base, st.Owner.Repair); why != "" { + wait = why + return nil + } + if st.Attempts == nil { + st.Attempts = map[int]preparationAttempt{} + } + prior := st.Attempts[number] + if prior.Head == head && prior.Base == baseSHA && c.now().Sub(prior.At) < effects.DefaultUnresolvedTTL { + wait = basePreparationReady + return nil + } + st.Attempts[number] = preparationAttempt{Head: head, Base: baseSHA, At: c.now()} + c.info("automerge base preparation refresh claimed", "repo", owner+"/"+repo, "base", base, "pr", number, "head", head, "base_sha", baseSHA) + return nil + }) + if err != nil { + return err + } + if wait != "" { + c.info("automerge base preparation waiting", "repo", owner+"/"+repo, "base", base, "pr", number, "reason", wait) + return &preparationWait{wait} + } + return nil +} + +func (c *Engine) forgetRejectedPreparationUpdate(owner, repo string, number int, head string) { + // A synchronous refusal cannot have applied the write. Forget only our + // reservation; the mutation journal still owns ambiguous reconciliation. + pr, _, err := c.gh.PullRequests.Get(context.Background(), owner, repo, number) + if err != nil { + return + } + err = c.preparationStore().change(preparationKey(owner, repo, pr.GetBase().GetRef()), func(st *basePreparation) error { + if st.Attempts[number].Head == head { + delete(st.Attempts, number) + } + return nil + }) + if err != nil { + c.warn("automerge base preparation persistence failed", "repo", owner+"/"+repo, "pr", number, "error", err) + } +} + +func preparationKeepsOwner(reason string, err error) bool { + if err != nil || reason == "" || strings.HasPrefix(reason, basePreparationPrefix) || + stageengine.ClassifyCIReason(reason) == stageengine.CIReasonPending || + reason == autoMergeReasonReviewGatePrefix+ReviewGateOutcomeAwaiting || + reason == autoMergeReasonReviewGatePrefix+ReviewGateOutcomeNoBundle || + reason == contributorLaneReasonNoVerdict || reason == contributorLaneReasonSoaking || + reason == "head-changed-since-eval" || reason == autoMergeReasonBehindBudgetExhausted { + return true + } + // Another lane declining authorship is not the owning lane withdrawing. + return reason == "not-app-authored" || reason == trustedAuthorReasonAppAuthored || reason == contributorLaneReasonAppAuthored +} + +func (c *Engine) observeBasePreparation(lane, owner, repo string, pr *gh.PullRequest, reason string, merged bool, evalErr error) { + if pr == nil || !c.requireCurrentBaseFor(owner+"/"+repo) { + return + } + base := pr.GetBase().GetRef() + if base == "" { + return + } + if err := c.preparationStore().change(preparationKey(owner, repo, base), func(st *basePreparation) error { + keep := !merged && preparationKeepsOwner(reason, evalErr) + if reason == basePreparationPrefix+"snapshot-changed" && st.Attempts[pr.GetNumber()].Head == "" { + // A push won the race before we issued any refresh. Pending + // work on that unreviewed head is not our preparation. + keep = false + } + // Waiting is not preparation yet: a changed or now-pending waiting + // head loses its admission ticket and must qualify again. + st.Waiting = slices.DeleteFunc(st.Waiting, func(w preparationCandidate) bool { + return w.Lane == lane && w.Number == pr.GetNumber() && (w.Head != pr.GetHead().GetSHA() || !strings.HasPrefix(reason, basePreparationPrefix)) + }) + if !keep && st.Owner.Lane == lane && st.Owner.Number == pr.GetNumber() { + if merged { + reason = "merged" + delete(st.Attempts, pr.GetNumber()) + } + c.info("automerge base preparation released", "repo", owner+"/"+repo, "base", base, "pr", pr.GetNumber(), "reason", reason) + st.Owner = preparationCandidate{} + } + return nil + }); err != nil { + c.warn("automerge base preparation persistence failed", "repo", owner+"/"+repo, "pr", pr.GetNumber(), "error", err) + } +} + +func (c *Engine) preparingBase(owner, repo, base string, number int) bool { + if !c.requireCurrentBaseFor(owner + "/" + repo) { + return false + } + preparing := false + err := c.preparationStore().change(preparationKey(owner, repo, base), func(st *basePreparation) error { + preparing = st.Owner.Number == number + return nil + }) + // A failed ownership read must not let a potentially preparing PR become + // a misleading ready train member. The merge gate still fails closed. + return preparing || err != nil +} diff --git a/src/pkg/github/automerge/automerge_base_preparation_races_test.go b/src/pkg/github/automerge/automerge_base_preparation_races_test.go new file mode 100644 index 0000000000..0036797715 --- /dev/null +++ b/src/pkg/github/automerge/automerge_base_preparation_races_test.go @@ -0,0 +1,53 @@ +package automerge + +import ( + "context" + "testing" + + gh "github.com/google/go-github/v72/github" +) + +func TestBasePreparationProvenanceForEveryAdmittingLane(t *testing.T) { + for _, lane := range []string{stageLaneSelf, stageLaneTrusted, stageLaneContributor} { + t.Run(lane, func(t *testing.T) { + api, srv := newPreparationAPI(t) + c := preparationEngine(srv.URL, NewBasePreparationStore("")) + if ev, _, _, err := c.currentBaseGate(context.Background(), "acme/widget", "acme", "widget", 1, "main", "head1", true, "behind", false, lane); err != nil || !ev.BranchUpdated { + t.Fatalf("admission: %+v %v", ev, err) + } + for _, other := range []string{stageLaneSelf, stageLaneTrusted, stageLaneContributor} { + if other != lane { + c.recordStage(other, "acme/widget", "acme", "widget", api.prs[1], "untrusted-author-role", false, nil) + } + } + if _, reason, _, err := preparationGate(c, 2, "main", "head2"); err != nil || reason != preparationQueued(1) { + t.Fatalf("other lane revoked %s owner: %q %v", lane, reason, err) + } + c.recordStage(lane, "acme/widget", "acme", "widget", api.prs[1], "required-check-failing:build", false, nil) + if ev, _, _, err := preparationGate(c, 2, "main", "head2"); err != nil || !ev.BranchUpdated { + t.Fatalf("owning lane did not release: %+v %v", ev, err) + } + }) + } +} + +func TestBasePreparationPushBeforeRefreshYieldsUnstartedOwner(t *testing.T) { + api, srv := newPreparationAPI(t) + c := preparationEngine(srv.URL, NewBasePreparationStore("")) + if reason, err := c.admitBasePreparation(context.Background(), "acme", "widget", "main", 1, "head1", stageLaneSelf, false); err != nil || reason != "" { + t.Fatalf("admission: %q %v", reason, err) + } + api.prs[1].Head.SHA = gh.Ptr("author-push") + api.pending["author-push"] = true + _, reason, err := c.updateBranchAndSkip(context.Background(), "acme/widget", "acme", "widget", 1, "head1", currentBaseUpdateReason, false) + if err != nil || reason != basePreparationPrefix+"snapshot-changed" { + t.Fatalf("stale refresh snapshot: %q %v", reason, err) + } + c.recordStage(stageLaneSelf, "acme/widget", "acme", "widget", api.prs[1], reason, false, nil) + if ev, _, _, err := preparationGate(c, 2, "main", "head2"); err != nil || !ev.BranchUpdated { + t.Fatalf("unstarted owner blocks qualified successor: %+v %v", ev, err) + } + if api.updates[1] != 0 || api.updates[2] != 1 { + t.Fatalf("unsafe refreshes: %v", api.updates) + } +} diff --git a/src/pkg/github/automerge/automerge_base_preparation_test.go b/src/pkg/github/automerge/automerge_base_preparation_test.go new file mode 100644 index 0000000000..b206617020 --- /dev/null +++ b/src/pkg/github/automerge/automerge_base_preparation_test.go @@ -0,0 +1,406 @@ +package automerge + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strconv" + "strings" + "sync" + "testing" + "time" + + gh "github.com/google/go-github/v72/github" + "github.com/hivecommons/hive/pkg/stagelabels" +) + +type preparationAPI struct { + mu sync.Mutex + prs map[int]*gh.PullRequest + behind map[string]int + pending map[string]bool + updates map[int]int + merges map[int]int +} + +func preparationPR(number int, base, head string) *gh.PullRequest { + return &gh.PullRequest{ + Number: gh.Ptr(number), State: gh.Ptr("open"), Title: gh.Ptr("prepare fixture"), + Mergeable: gh.Ptr(true), MergeableState: gh.Ptr("clean"), + User: &gh.User{Login: gh.Ptr(testHiveAppBotLogin)}, + Head: &gh.PullRequestBranch{SHA: gh.Ptr(head), Repo: &gh.Repository{FullName: gh.Ptr("acme/widget")}}, + Base: &gh.PullRequestBranch{Ref: gh.Ptr(base), SHA: gh.Ptr("base1"), Repo: &gh.Repository{FullName: gh.Ptr("acme/widget")}}, + } +} + +func newPreparationAPI(t *testing.T) (*preparationAPI, *httptest.Server) { + t.Helper() + api := &preparationAPI{ + prs: map[int]*gh.PullRequest{}, behind: map[string]int{}, pending: map[string]bool{}, + updates: map[int]int{}, merges: map[int]int{}, + } + for n := 1; n <= 3; n++ { + head := fmt.Sprintf("head%d", n) + api.prs[n] = preparationPR(n, "main", head) + api.behind[head] = 1 + } + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + api.mu.Lock() + defer api.mu.Unlock() + path := strings.Split(strings.Trim(r.URL.Path, "/"), "/") + if len(path) < 4 || path[0] != "repos" { + t.Errorf("unexpected request %s %s", r.Method, r.URL.Path) + w.WriteHeader(http.StatusNotFound) + return + } + write := func(v any) { _ = json.NewEncoder(w).Encode(v) } + switch path[3] { + case "pulls": + n, _ := strconv.Atoi(path[4]) + if len(path) == 5 { + if api.prs[n] == nil { + w.WriteHeader(http.StatusNotFound) + } else { + write(api.prs[n]) + } + return + } + switch path[5] { + case "update-branch": + var opts gh.PullRequestBranchUpdateOptions + _ = json.NewDecoder(r.Body).Decode(&opts) + if opts.GetExpectedHeadSHA() != api.prs[n].GetHead().GetSHA() { + w.WriteHeader(http.StatusUnprocessableEntity) + write(map[string]string{"message": "expected_head_sha does not match"}) + return + } + api.updates[n]++ + w.WriteHeader(http.StatusAccepted) + write(map[string]string{"message": "Updating pull request branch"}) + case "reviews": + write([]any{}) + case "merge": + api.merges[n]++ + write(map[string]any{"merged": true, "sha": "merged"}) + default: + t.Errorf("unexpected pull request %s", r.URL.Path) + w.WriteHeader(http.StatusNotFound) + } + case "compare": + _, head, _ := strings.Cut(path[4], "...") + write(map[string]any{"behind_by": api.behind[head], "base_commit": map[string]string{"sha": "base1"}}) + case "branches": + if len(path) > 5 { + w.WriteHeader(http.StatusNotFound) + write(map[string]string{"message": "Branch not protected"}) + } else { + write(map[string]any{"name": path[4], "commit": map[string]string{"sha": "base1"}}) + } + case "rules": + write([]any{}) + case "commits": + head := path[4] + state, conclusion := "success", "success" + if api.pending[head] { + state, conclusion = "pending", "" + } + if path[5] == "status" { + write(map[string]any{"state": state, "total_count": 1, "statuses": []map[string]string{{"context": "build", "state": state}}}) + } else { + status := "completed" + if conclusion == "" { + status = "queued" + } + write(map[string]any{"total_count": 1, "check_runs": []map[string]string{{"name": "build", "status": status, "conclusion": conclusion}}}) + } + case "actions": + write(map[string]any{"total_count": 0, "workflow_runs": []any{}}) + default: + t.Errorf("unexpected request %s %s", r.Method, r.URL.Path) + w.WriteHeader(http.StatusNotFound) + } + })) + t.Cleanup(srv.Close) + return api, srv +} + +func preparationEngine(url string, store *BasePreparationStore) *Engine { + c := newAutoMergeSweepClient(url) + c.basePreparation = store + c.requireCurrentBase = func(string) bool { return true } + c.requireCurrentBaseStrict = func(string) bool { return true } + return c +} + +func preparationGate(c *Engine, n int, base, head string) (AutoMergeSweepEvent, string, bool, error) { + return c.currentBaseGate(context.Background(), "acme/widget", "acme", "widget", n, base, head, true, "behind", false) +} + +func TestBasePreparationStickyAcrossLanesSweepsAndRestart(t *testing.T) { + api, srv := newPreparationAPI(t) + dir := filepath.Join(t.TempDir(), "preparation") + c := preparationEngine(srv.URL, NewBasePreparationStore(dir)) + if ev, reason, stop, err := preparationGate(c, 1, "main", "head1"); err != nil || !stop || !ev.BranchUpdated || reason != basePreparationReady { + t.Fatalf("first refresh: %+v %q %v %v", ev, reason, stop, err) + } + for _, lane := range []string{stageLaneSelf, stageLaneTrusted, stageLaneContributor} { + // Fresh stores simulate one-shot lane engines and a pod restart. + c = preparationEngine(srv.URL, NewBasePreparationStore(dir)) + c.recordStage(lane, "acme/widget", "acme", "widget", preparationPR(1, "main", "new-refresh-head"), "required-check-pending:Sign-off survives the squash", false, nil) + for _, n := range []int{2, 3, 1} { + _, reason, stop, err := preparationGate(c, n, "main", fmt.Sprintf("head%d", n)) + want := preparationQueued(1) + if n == 1 { + want = basePreparationReady + } + if err != nil || !stop || reason != want { + t.Fatalf("lane %s PR %d: reason=%q stop=%v err=%v", lane, n, reason, stop, err) + } + } + } + if api.updates[1] != 1 || api.updates[2] != 0 || api.updates[3] != 0 { + t.Fatalf("refresh stampede: %v", api.updates) + } +} + +func TestBasePreparationReleasesInvalidFront(t *testing.T) { + for _, why := range []string{"closed", "gone", "held", "draft", "conflicting", "retargeted", "expired", "ci-red", "review-rejected", "withdrawn"} { + t.Run(why, func(t *testing.T) { + api, srv := newPreparationAPI(t) + c := preparationEngine(srv.URL, NewBasePreparationStore("")) + now := time.Date(2026, 10, 30, 17, 0, 0, 0, time.UTC) + c.now = func() time.Time { return now } + if _, _, _, err := preparationGate(c, 1, "main", "head1"); err != nil { + t.Fatal(err) + } + switch why { + case "closed": + api.prs[1].State = gh.Ptr("closed") + case "gone": + delete(api.prs, 1) + case "held": + api.prs[1].Labels = []*gh.Label{{Name: gh.Ptr("hold")}} + case "draft": + api.prs[1].Draft = gh.Ptr(true) + case "conflicting": + api.prs[1].MergeableState = gh.Ptr("dirty") + case "retargeted": + api.prs[1].Base.Ref = gh.Ptr("other") + case "expired": + now = now.Add(c.preparationWindow()) + case "ci-red", "review-rejected", "withdrawn": + reason := "required-check-failing:build" + if why == "review-rejected" { + reason = autoMergeReasonReviewGatePrefix + ReviewGateOutcomeChangesRequested + } else if why == "withdrawn" { + reason = "untrusted-author-role" + } + c.recordStage(stageLaneSelf, "acme/widget", "acme", "widget", api.prs[1], reason, false, nil) + } + if ev, reason, _, err := preparationGate(c, 2, "main", "head2"); err != nil || !ev.BranchUpdated { + t.Fatalf("successor blocked after %s: %+v %q %v", why, ev, reason, err) + } + }) + } +} + +func TestBasePreparationCurrentReadySuccessorBypassesPendingOwnerWithoutBypassingGates(t *testing.T) { + for _, blocker := range []string{"", "ci-pending", "review-owed"} { + t.Run(blocker, func(t *testing.T) { + api, srv := newPreparationAPI(t) + c := preparationEngine(srv.URL, NewBasePreparationStore("")) + approved := true + c.reviewGate = &ReviewGate{ + RequireApproval: func() bool { return true }, + Verdict: func(_ string, n int, _ string) ReviewGateVerdict { + return ReviewGateVerdict{Loaded: true, Found: true, Approved: n == 1 || approved} + }, + } + if ev, _, err := c.trySweepSelfAuthoredPR(context.Background(), "acme/widget", "acme", "widget", 1, true); err != nil || !ev.BranchUpdated { + t.Fatalf("admit owner: %+v %v", ev, err) + } + // A refresh head is pending on runners, not red or missing DCO. + api.prs[1].Head.SHA = gh.Ptr("refreshed1") + api.pending["refreshed1"] = true + api.behind["head2"] = 0 // successor is already tested on current base + if blocker == "ci-pending" { + api.pending["head2"] = true + } + if blocker == "review-owed" { + approved = false + } + if _, _, err := c.trySweepSelfAuthoredPR(context.Background(), "acme/widget", "acme", "widget", 1, true); err != nil { + t.Fatal(err) + } + ev, reason, err := c.trySweepSelfAuthoredPR(context.Background(), "acme/widget", "acme", "widget", 2, true) + if err != nil { + t.Fatal(err) + } + if blocker == "" { + if ev.MergeSHA == "" || reason != "" || api.merges[2] != 1 { + t.Fatalf("ready successor blocked by pending preparation: %+v %q", ev, reason) + } + } else if ev.MergeSHA != "" || api.merges[2] != 0 || reason == "" { + t.Fatalf("bypassed %s: %+v %q", blocker, ev, reason) + } + if api.updates[1] != 1 || api.updates[2] != 0 { + t.Fatalf("unexpected refreshes: %v", api.updates) + } + }) + } +} + +func TestBasePreparationConcurrentDuplicateAndIndependentDestinations(t *testing.T) { + api, srv := newPreparationAPI(t) + dir := filepath.Join(t.TempDir(), "preparation") + var wg sync.WaitGroup + const sweeps = 8 + for range sweeps { + wg.Add(1) + go func() { + defer wg.Done() + c := preparationEngine(srv.URL, NewBasePreparationStore(dir)) + if _, _, stop, err := preparationGate(c, 1, "main", "head1"); err != nil || !stop { + t.Errorf("concurrent gate stop=%v err=%v", stop, err) + } + }() + } + wg.Wait() + if api.updates[1] != 1 { + t.Fatalf("duplicate refreshes: %v", api.updates) + } + api.prs[2].Base.Ref = gh.Ptr("release") + c := preparationEngine(srv.URL, NewBasePreparationStore(dir)) + if ev, _, _, err := preparationGate(c, 2, "release", "head2"); err != nil || !ev.BranchUpdated { + t.Fatalf("separate base blocked: %+v %v", ev, err) + } + // Repository case aliases share ownership; ref case remains distinct. + if reason, err := c.admitBasePreparation(context.Background(), "ACME", "WIDGET", "main", 3, "head3", stageLaneContributor, false); err != nil || reason != preparationQueued(1) { + t.Fatalf("canonical repo gate: %q %v", reason, err) + } + if reason, err := c.admitBasePreparation(context.Background(), "acme", "other", "main", 3, "head3", stageLaneTrusted, false); err != nil || reason != "" { + t.Fatalf("independent repo blocked: %q %v", reason, err) + } +} + +func TestBasePreparationHeadBaseChangesAndFairRelease(t *testing.T) { + api, srv := newPreparationAPI(t) + c := preparationEngine(srv.URL, NewBasePreparationStore("")) + for _, n := range []int{1, 2, 3} { + if _, _, _, err := preparationGate(c, n, "main", fmt.Sprintf("head%d", n)); err != nil { + t.Fatal(err) + } + } + // A changed head cannot be updated through a stale snapshot. + api.prs[1].Head.SHA = gh.Ptr("pushed1") + if err := c.executeUpdateBranchEffect(context.Background(), "acme", "widget", 1, "head1", "another-lane"); preparationWaitReason(err) != basePreparationPrefix+"snapshot-changed" { + t.Fatalf("stale head update: %v", err) + } + // A new external base tip may be coalesced once the owner qualifies again. + api.prs[1].Base.SHA = gh.Ptr("base2") + api.behind["pushed1"] = 1 + if ev, _, _, err := preparationGate(c, 1, "main", "pushed1"); err != nil || !ev.BranchUpdated { + t.Fatalf("new qualified head/base: %+v %v", ev, err) + } + c.recordStage(stageLaneSelf, "acme/widget", "acme", "widget", api.prs[1], "", true, nil) + if _, reason, _, err := preparationGate(c, 3, "main", "head3"); err != nil || reason != preparationQueued(2) { + t.Fatalf("arrival order lost after merge: %q %v", reason, err) + } + if ev, _, _, err := preparationGate(c, 2, "main", "head2"); err != nil || !ev.BranchUpdated { + t.Fatalf("oldest waiting candidate did not advance: %+v %v", ev, err) + } +} + +func TestBasePreparationIgnoresOtherLanesNonAdmission(t *testing.T) { + api, srv := newPreparationAPI(t) + c := preparationEngine(srv.URL, NewBasePreparationStore("")) + for _, n := range []int{1, 2} { + if _, _, _, err := preparationGate(c, n, "main", fmt.Sprintf("head%d", n)); err != nil { + t.Fatal(err) + } + } + for _, otherLane := range []string{stageLaneTrusted, stageLaneContributor} { + for _, reason := range []string{"trusted-bot-lane", "untrusted-author-role", "not-app-authored", "contributor-lane:app-authored"} { + for _, n := range []int{1, 2} { + c.recordStage(otherLane, "acme/widget", "acme", "widget", api.prs[n], reason, false, nil) + } + } + } + if _, reason, _, err := preparationGate(c, 3, "main", "head3"); err != nil || reason != preparationQueued(1) { + t.Fatalf("another lane revoked the pending owner: %q %v", reason, err) + } + c.recordStage(stageLaneSelf, "acme/widget", "acme", "widget", api.prs[1], "", true, nil) + if _, reason, _, err := preparationGate(c, 3, "main", "head3"); err != nil || reason != preparationQueued(2) { + t.Fatalf("another lane erased FIFO tickets: %q %v", reason, err) + } +} + +func TestBasePreparationNeverPromotesChangedOrUnqualifiedWaitingHead(t *testing.T) { + for _, changed := range []bool{false, true} { + t.Run(fmt.Sprint(changed), func(t *testing.T) { + api, srv := newPreparationAPI(t) + c := preparationEngine(srv.URL, NewBasePreparationStore("")) + for _, n := range []int{1, 2} { + if _, _, _, err := preparationGate(c, n, "main", fmt.Sprintf("head%d", n)); err != nil { + t.Fatal(err) + } + } + api.prs[1].State = gh.Ptr("closed") + if changed { + api.prs[2].Head.SHA = gh.Ptr("pushed2") + } + ev, reason, _, err := preparationGate(c, 3, "main", "head3") + if err != nil { + t.Fatal(err) + } + if changed { + if !ev.BranchUpdated { + t.Fatalf("changed waiting head blocked successor: %q", reason) + } + } else { + if reason != preparationQueued(2) { + t.Fatalf("lost FIFO: %q", reason) + } + var owner int + _ = c.preparationStore().change(preparationKey("acme", "widget", "main"), func(st *basePreparation) error { + owner = st.Owner.Number + return nil + }) + if owner != 0 { + t.Fatalf("old ticket became an unqualified owner: %d", owner) + } + c.recordStage(stageLaneSelf, "acme/widget", "acme", "widget", api.prs[2], "required-check-pending:build", false, nil) + if ev, _, _, err := preparationGate(c, 3, "main", "head3"); err != nil || !ev.BranchUpdated { + t.Fatalf("pending waiting ticket did not yield: %+v %v", ev, err) + } + } + }) + } +} + +func TestBasePreparationPersistenceFailsClosedAndNoFalseReadyStage(t *testing.T) { + api, srv := newPreparationAPI(t) + dir := t.TempDir() + // Use a file as the store directory, deterministically refusing persistence. + path := filepath.Join(dir, "not-a-directory") + if err := os.WriteFile(path, []byte("occupied"), 0o600); err != nil { + t.Fatal(err) + } + c := preparationEngine(srv.URL, NewBasePreparationStore(path)) + if _, reason, stop, err := preparationGate(c, 1, "main", "head1"); err == nil || !stop || reason != basePreparationPrefix+"state-unavailable" { + t.Fatalf("persistence failure did not fail closed: %q %v %v", reason, stop, err) + } + for _, reason := range []string{preparationQueued(1), basePreparationReady} { + v := classifyStage(stageObservation{Lane: stageLaneSelf, Reason: reason}) + if v.Stage != stagelabels.ReviewPending || v.Stage.Number() >= stagelabels.AwaitingMergeAdmission.Number() { + t.Fatalf("preparation advertised ready: %+v", v) + } + } + if api.updates[1] != 0 { + t.Fatalf("unexpected update %v", api.updates) + } +} diff --git a/src/pkg/github/automerge/automerge_contributor_lane.go b/src/pkg/github/automerge/automerge_contributor_lane.go index 254e894bb3..7d7c7b43f8 100644 --- a/src/pkg/github/automerge/automerge_contributor_lane.go +++ b/src/pkg/github/automerge/automerge_contributor_lane.go @@ -1382,7 +1382,7 @@ func (c *Engine) evaluateContributorLanePR(ctx context.Context, displayRepo, own // See trySweepSelfAuthoredPR: the queue bypass is an explicit opt-in. A // relay evaluation never enqueues, so it keeps the current-base gates. queueSkipsBase := sync.OnceValue(func() bool { return !relay && c.nativeQueueSkipsCurrentBase(ctx, owner+"/"+repo, pr) }) - if mergeableState == "behind" && !queueSkipsBase() { + if mergeableState == "behind" && !queueSkipsBase() && !c.requireCurrentBaseFor(owner+"/"+repo) { if relay { return eval, "behind", nil } @@ -1458,7 +1458,7 @@ func (c *Engine) evaluateContributorLanePR(ctx context.Context, displayRepo, own // approved head is worth an update-branch commit; a relay evaluation // never updates the branch. if c.requireCurrentBaseFor(owner+"/"+repo) && !queueSkipsBase() { - if _, reason, stop, err := c.currentBaseGate(ctx, displayRepo, owner, repo, number, baseBranch, headSHA, !relay, "behind", c.isForkPR(pr)); stop { + if _, reason, stop, err := c.currentBaseGate(ctx, displayRepo, owner, repo, number, baseBranch, headSHA, !relay, "behind", c.isForkPR(pr), stageLaneContributor); stop { return eval, reason, err } } diff --git a/src/pkg/github/automerge/automerge_contributor_lane_test.go b/src/pkg/github/automerge/automerge_contributor_lane_test.go index 7dbf742581..04fe2c00f6 100644 --- a/src/pkg/github/automerge/automerge_contributor_lane_test.go +++ b/src/pkg/github/automerge/automerge_contributor_lane_test.go @@ -190,6 +190,7 @@ func newContributorLaneAPI(t *testing.T, fx *contributorLaneFixture) *httptest.S } if fx.baseRef != "" { p["base"].(map[string]any)["ref"] = fx.baseRef + p["base"].(map[string]any)["sha"] = "base1" } if !fx.omitAssociation { p["author_association"] = association diff --git a/src/pkg/github/automerge/automerge_current_base.go b/src/pkg/github/automerge/automerge_current_base.go index d9cdf6de63..2885fa1aac 100644 --- a/src/pkg/github/automerge/automerge_current_base.go +++ b/src/pkg/github/automerge/automerge_current_base.go @@ -89,7 +89,7 @@ func (c *Engine) warnCurrentBaseFallbackOnce(displayRepo string, number int, rea // // stop is false when the merge may proceed; otherwise the caller returns the // event, reason and error as-is. -func (c *Engine) currentBaseGate(ctx context.Context, displayRepo, owner, repo string, number int, baseBranch, headSHA string, branchUpdateAllowed bool, holdReason string, fork bool) (AutoMergeSweepEvent, string, bool, error) { +func (c *Engine) currentBaseGate(ctx context.Context, displayRepo, owner, repo string, number int, baseBranch, headSHA string, branchUpdateAllowed bool, holdReason string, fork bool, admissionLane ...string) (AutoMergeSweepEvent, string, bool, error) { if !c.requireCurrentBaseFor(owner+"/"+repo) || c.gh == nil { return AutoMergeSweepEvent{}, "", false, nil } @@ -102,13 +102,24 @@ func (c *Engine) currentBaseGate(ctx context.Context, displayRepo, owner, repo s return AutoMergeSweepEvent{}, autoMergeReasonCurrentBaseCheck, true, err } if cmp.GetBehindBy() <= 0 { + // Ownership serializes REFRESHES, never already-current ready work. + // A fully gated successor may land while another head prepares. return AutoMergeSweepEvent{}, "", false, nil } if !branchUpdateAllowed { return AutoMergeSweepEvent{}, holdReason, true, nil } - c.info("automerge head is behind the current base; updating before merge", "repo", displayRepo, "pr", number, "base", baseBranch, "behind_by", cmp.GetBehindBy()) + lane := stageLaneSelf + if len(admissionLane) > 0 { + lane = admissionLane[0] + } + if reason, err := c.admitBasePreparation(ctx, owner, repo, baseBranch, number, headSHA, lane, false); reason != "" || err != nil { + return AutoMergeSweepEvent{}, nonEmpty(reason, basePreparationPrefix+"state-unavailable"), true, err + } event, reason, err := c.updateBranchAndSkip(ctx, displayRepo, owner, repo, number, headSHA, currentBaseUpdateReason, fork) + if event.BranchUpdated { + c.info("automerge head is behind the current base; updating before merge", "repo", displayRepo, "pr", number, "base", baseBranch, "behind_by", cmp.GetBehindBy()) + } if currentBaseUpdateUnavailable(reason, err) && !c.requireCurrentBaseStrictFor(owner+"/"+repo) { // Gate inherited from the hive-wide default (#15335) on a head the App // may not write: parking it would turn every non-editable fork PR diff --git a/src/pkg/github/automerge/automerge_current_base_test.go b/src/pkg/github/automerge/automerge_current_base_test.go index 30e27de9b5..59838049e1 100644 --- a/src/pkg/github/automerge/automerge_current_base_test.go +++ b/src/pkg/github/automerge/automerge_current_base_test.go @@ -29,7 +29,7 @@ func TestCurrentBaseGate(t *testing.T) { }{ {name: "off by default", optIn: false, behindBy: 3, updateAllowed: true}, {name: "up to date merges", optIn: true, behindBy: 0, updateAllowed: true, wantCompares: 1}, - {name: "behind updates branch", optIn: true, behindBy: 2, updateAllowed: true, wantStop: true, wantReason: "check-pending", wantCompares: 1, wantUpdates: 1, wantUpdated: true}, + {name: "behind updates branch", optIn: true, behindBy: 2, updateAllowed: true, wantStop: true, wantReason: basePreparationReady, wantCompares: 1, wantUpdates: 1, wantUpdated: true}, {name: "behind without update budget holds", optIn: true, behindBy: 2, updateAllowed: false, wantStop: true, wantReason: "behind", wantCompares: 1}, {name: "compare failure fails closed", optIn: true, compareStatus: http.StatusInternalServerError, updateAllowed: true, wantStop: true, wantReason: autoMergeReasonCurrentBaseCheck, wantErr: true, wantCompares: 1}, } @@ -41,6 +41,8 @@ func TestCurrentBaseGate(t *testing.T) { mu.Lock() defer mu.Unlock() switch { + case r.Method == http.MethodGet && r.URL.Path == "/repos/acme/widget/pulls/7": + _ = json.NewEncoder(w).Encode(map[string]any{"number": 7, "state": "open", "head": map[string]string{"sha": head}, "base": map[string]string{"ref": "main", "sha": "base1"}}) case r.Method == http.MethodGet && r.URL.Path == "/repos/acme/widget/compare/main..."+head: compares++ if tc.compareStatus != 0 { @@ -115,6 +117,8 @@ func TestCurrentBaseGateUpdateUnavailableFallback(t *testing.T) { t.Run(tc.name, func(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch { + case r.Method == http.MethodGet && r.URL.Path == "/repos/acme/widget/pulls/7": + _ = json.NewEncoder(w).Encode(map[string]any{"number": 7, "state": "open", "head": map[string]string{"sha": head}, "base": map[string]string{"ref": "main", "sha": "base1"}}) case r.Method == http.MethodGet && r.URL.Path == "/repos/acme/widget/compare/main..."+head: _ = json.NewEncoder(w).Encode(map[string]any{"behind_by": 2, "ahead_by": 1}) case r.Method == http.MethodPut && r.URL.Path == "/repos/acme/widget/pulls/7/update-branch": @@ -190,7 +194,7 @@ func newCurrentBaseLaneServer(t *testing.T, a *currentBaseLaneAPI) *httptest.Ser "mergeable": true, "user": map[string]string{"login": testHiveAppBotLogin}, "head": map[string]any{"sha": head, "repo": map[string]string{"full_name": "acme/widget"}}, - "base": map[string]any{"ref": "main", "repo": map[string]string{"full_name": "acme/widget"}}, + "base": map[string]any{"ref": "main", "sha": "base1", "repo": map[string]string{"full_name": "acme/widget"}}, "labels": []map[string]string{}, }) case r.Method == http.MethodGet && r.URL.Path == "/repos/acme/widget/pulls/7/reviews": @@ -264,7 +268,7 @@ func TestTrySweepSelfAuthoredPRCurrentBaseGateAfterGreen(t *testing.T) { wantMerges int }{ {name: "green current head merges", optIn: true, statusState: "success", behindBy: 0, updateAllowed: true, wantMergeSHA: "merge7", wantCompares: 1, wantMerges: 1}, - {name: "green behind head is updated and counted against the budget", optIn: true, statusState: "success", behindBy: 2, updateAllowed: true, wantReason: "check-pending", wantUpdated: true, wantCompares: 1, wantUpdates: 1}, + {name: "green behind head is updated and counted against the budget", optIn: true, statusState: "success", behindBy: 2, updateAllowed: true, wantReason: basePreparationReady, wantUpdated: true, wantCompares: 1, wantUpdates: 1}, {name: "green behind head holds when the budget is spent", optIn: true, statusState: "success", behindBy: 2, updateAllowed: false, wantReason: autoMergeReasonBehindBudgetExhausted, wantCompares: 1}, {name: "pending behind head is left alone", optIn: true, statusState: "pending", behindBy: 2, updateAllowed: true, wantCIKind: ciReasonPending}, {name: "red behind head is left alone", optIn: true, statusState: "failure", behindBy: 2, updateAllowed: true, wantCIKind: ciReasonRed}, @@ -321,7 +325,7 @@ func TestContributorLaneCurrentBaseGateAfterGreen(t *testing.T) { wantMerges int }{ {name: "green soaked current head merges", optIn: true, behindBy: 0, soaked: true, wantCompares: 1, wantMerges: 1}, - {name: "green soaked behind head is updated instead of merged", optIn: true, behindBy: 2, soaked: true, wantReason: "check-pending", wantCompares: 1, wantUpdates: 1}, + {name: "green soaked behind head is updated instead of merged", optIn: true, behindBy: 2, soaked: true, wantReason: basePreparationReady, wantCompares: 1, wantUpdates: 1}, {name: "relay evaluation of a behind head holds without updating", optIn: true, behindBy: 2, soaked: true, relay: true, wantReason: "behind", wantCompares: 1}, {name: "red behind head is left alone", optIn: true, headStatus: "failure", behindBy: 2, soaked: true, wantCIKind: ciReasonRed}, {name: "soaking behind head is left alone until soaked", optIn: true, behindBy: 2, soaked: false, wantSoaking: true}, @@ -337,7 +341,7 @@ func TestContributorLaneCurrentBaseGateAfterGreen(t *testing.T) { defer api.Close() policy := defaultContributorLanePolicy() clock := &contributorLaneClock{at: time.Date(2026, 10, 22, 13, 0, 0, 0, time.UTC)} - opts := Options{RequireCurrentBase: func(repo string) bool { return tc.optIn && strings.EqualFold(repo, "acme/widget") }} + opts := Options{BasePreparation: NewBasePreparationStore(""), RequireCurrentBase: func(repo string) bool { return tc.optIn && strings.EqualFold(repo, "acme/widget") }} if tc.soaked { opts.ContributorSoak = soakedTracker(clock) } diff --git a/src/pkg/github/automerge/automerge_merge_forward.go b/src/pkg/github/automerge/automerge_merge_forward.go index 575b27f916..3cdc7293df 100644 --- a/src/pkg/github/automerge/automerge_merge_forward.go +++ b/src/pkg/github/automerge/automerge_merge_forward.go @@ -74,6 +74,22 @@ func (c *Engine) mergeForwardOrConflict(ctx context.Context, displayRepo, owner, if headRef == "" || baseRef == "" || headSHA == "" || appLogin == "" { return AutoMergeSweepEvent{}, "conflicting", nil } + if c.requireCurrentBaseFor(owner + "/" + repo) { + if reason, err := c.admitBasePreparation(ctx, owner, repo, baseRef, number, headSHA, stageLaneTrusted, true); reason != "" || err != nil { + if err != nil { + return AutoMergeSweepEvent{}, "merge-forward", err + } + return AutoMergeSweepEvent{}, reason, nil + } + if err := c.claimBasePreparationUpdate(ctx, owner, repo, number, headSHA); err != nil { + // A conflicting outsider belongs to the existing repair path, + // not to final preparation or a ready merge-train seat. + if wait := preparationWaitReason(err); wait != "" { + return AutoMergeSweepEvent{}, wait, nil + } + return AutoMergeSweepEvent{}, "merge-forward", err + } + } if !c.markMergeForwardAttempted(owner, repo, number, headSHA) { return AutoMergeSweepEvent{}, "conflicting", nil } diff --git a/src/pkg/github/automerge/automerge_merge_forward_test.go b/src/pkg/github/automerge/automerge_merge_forward_test.go index 0ba0c27333..abc0167723 100644 --- a/src/pkg/github/automerge/automerge_merge_forward_test.go +++ b/src/pkg/github/automerge/automerge_merge_forward_test.go @@ -48,6 +48,12 @@ func newMergeForwardServer(t *testing.T, a *mergeForwardAPI) *httptest.Server { a.mu.Lock() defer a.mu.Unlock() switch { + case r.Method == http.MethodGet && r.URL.Path == "/repos/acme/widget/pulls/7": + pr := mergeForwardPR(false, "author/human") + pr.State = gh.Ptr("open") + pr.MergeableState = gh.Ptr("dirty") + pr.Base.SHA = gh.Ptr("base1") + _ = json.NewEncoder(w).Encode(pr) case r.Method == http.MethodPost && r.URL.Path == "/repos/acme/widget/merges": a.merges++ _ = json.NewDecoder(r.Body).Decode(&a.mergeBody) @@ -167,6 +173,7 @@ func TestMergeForwardOrConflictOncePerHead(t *testing.T) { if _, reason, err := c.mergeForwardOrConflict(context.Background(), "acme/widget", "acme", "widget", pr, 7); reason != "conflicting" || err != nil { t.Fatalf("tick %d: reason, err = %q, %v", i, reason, err) } + } api.mu.Lock() defer api.mu.Unlock() @@ -175,6 +182,30 @@ func TestMergeForwardOrConflictOncePerHead(t *testing.T) { } } +func TestMergeForwardRepairSharesPreparationWithoutDisablingRepair(t *testing.T) { + api := &mergeForwardAPI{mergeStatus: http.StatusCreated} + srv := newMergeForwardServer(t, api) + defer srv.Close() + c := newAutoMergeSweepClient(srv.URL) + c.requireCurrentBase = func(string) bool { return true } + c.mergeForwardTrustedHuman = func(string) bool { return true } + for tick := range 2 { + ev, reason, err := c.mergeForwardOrConflict(context.Background(), "acme/widget", "acme", "widget", mergeForwardPR(false, "author/human"), 7) + if err != nil { + t.Fatal(err) + } + if tick == 0 && !ev.BranchUpdated { + t.Fatalf("explicit repair was disabled: %+v %q", ev, reason) + } + if tick == 1 && (ev.BranchUpdated || reason != basePreparationReady) { + t.Fatalf("repair refresh repeated: %+v %q", ev, reason) + } + } + if api.merges != 1 { + t.Fatalf("repair mutations=%d, want 1", api.merges) + } +} + func TestMergeForwardOrConflictNeedsRefsAndAppLogin(t *testing.T) { srv := newMergeForwardServer(t, &mergeForwardAPI{}) defer srv.Close() diff --git a/src/pkg/github/automerge/automerge_merge_train.go b/src/pkg/github/automerge/automerge_merge_train.go index 20eaf711ea..c759dc32fe 100644 --- a/src/pkg/github/automerge/automerge_merge_train.go +++ b/src/pkg/github/automerge/automerge_merge_train.go @@ -423,6 +423,9 @@ func (c *Engine) planMergeTrains(ctx context.Context, lane, displayRepo, owner, if !mergeTrainCandidate(prefilter(pr)) { continue } + if c.preparingBase(owner, repo, pr.GetBase().GetRef(), pr.GetNumber()) { + continue // preparation is not a current-head-ready train seat + } if why := c.mergeTrainUnderRepair(owner, repo, pr); why != "" { c.mergeTrainSkipLogged(owner, repo, displayRepo, pr.GetNumber(), pr.GetHead().GetSHA(), why) continue @@ -778,6 +781,9 @@ func (c *Engine) mergeTrainAfterFrontMerge(ctx context.Context, plan *mergeTrain head := live.GetHead().GetSHA() switch { case state == "behind": + if c.requireCurrentBaseFor(plan.owner + "/" + plan.repo) { + continue // the next admitted preparation front owns its refresh + } if err := c.executeUpdateBranchEffect(ctx, plan.owner, plan.repo, n, head, "merge-train:front-merged#"+strconv.Itoa(front)); err != nil { switch classifyUpdateBranchError(err, c.isForkPR(live)) { case updateBranchUpToDate: diff --git a/src/pkg/github/automerge/automerge_native_queue_test.go b/src/pkg/github/automerge/automerge_native_queue_test.go index f24524a2d7..819ba405c4 100644 --- a/src/pkg/github/automerge/automerge_native_queue_test.go +++ b/src/pkg/github/automerge/automerge_native_queue_test.go @@ -467,14 +467,27 @@ func TestNativeQueueCurrentBaseGateIsExplicit(t *testing.T) { author = "alice" } updates := 0 - api := newAutoMergeSweepAPI(t, "", []sweepPR{{number: 7, nodeID: "PR_7", author: author, mergeableState: state, statusState: "success", checkStatus: "completed", checkConclusion: "success"}}, &merged, func(w http.ResponseWriter, r *http.Request) { + api := newAutoMergeSweepAPI(t, "", []sweepPR{{number: 7, nodeID: "PR_7", author: author, baseRef: "main", baseSHA: "base1", mergeableState: state, statusState: "success", checkStatus: "completed", checkConclusion: "success"}}, &merged, func(w http.ResponseWriter, r *http.Request) { switch { + case r.Method == http.MethodGet && r.URL.Path == "/repos/acme/widget/branches/main": + _ = json.NewEncoder(w).Encode(map[string]any{"name": "main", "commit": map[string]string{"sha": "base1"}}) + case r.Method == http.MethodGet && r.URL.Path == "/repos/acme/widget/commits/base1/status": + _ = json.NewEncoder(w).Encode(map[string]any{"state": "success", "total_count": 0, "statuses": []any{}}) + case r.Method == http.MethodGet && r.URL.Path == "/repos/acme/widget/branches/main/protection/required_status_checks": + w.WriteHeader(http.StatusNotFound) + _ = json.NewEncoder(w).Encode(map[string]string{"message": "Branch not protected"}) + case r.Method == http.MethodGet && r.URL.Path == "/repos/acme/widget/rules/branches/main": + _ = json.NewEncoder(w).Encode([]any{}) case r.Method == http.MethodPut && strings.HasSuffix(r.URL.Path, "/update-branch"): updates++ w.WriteHeader(http.StatusAccepted) _, _ = w.Write([]byte(`{"message":"Updating pull request branch."}`)) case strings.Contains(r.URL.Path, "/compare/"): - _ = json.NewEncoder(w).Encode(map[string]any{"behind_by": 2, "ahead_by": 1}) + behind := 0 + if state == "behind" { + behind = 2 + } + _ = json.NewEncoder(w).Encode(map[string]any{"behind_by": behind, "ahead_by": 1}) default: t.Errorf("unexpected request %s %s", r.Method, r.URL.Path) w.WriteHeader(http.StatusNotFound) @@ -496,14 +509,11 @@ func TestNativeQueueCurrentBaseGateIsExplicit(t *testing.T) { c.trustedAuthorPolicy = func() TrustedAuthorPolicy { return TrustedAuthorPolicy{Enabled: true, RequireRole: "merger"} } _, err = c.SweepTrustedAuthorAutoMerges(context.Background(), AutoMergeSweepOptions{}) } - // The fixture has no base ref, so a clean head with the - // gate kept parks at current-base-check instead of - // updating; either way it is not enqueued. wantUpdates, wantEnqueues := 0, 0 if !skip && state == "behind" { wantUpdates = 1 } - if skip { + if skip || state == "clean" { wantEnqueues = 1 } if err != nil || updates != wantUpdates || f.enqueues != wantEnqueues || len(merged) != 0 { diff --git a/src/pkg/github/automerge/automerge_stage_labels.go b/src/pkg/github/automerge/automerge_stage_labels.go index 4c1a37abb3..d14cdee6e3 100644 --- a/src/pkg/github/automerge/automerge_stage_labels.go +++ b/src/pkg/github/automerge/automerge_stage_labels.go @@ -463,6 +463,7 @@ func (c *Engine) removeNonStageLabel(ctx context.Context, owner, repo string, nu // reason it settled on (""+merged for a merge). It is a no-op when the // policy is off so lanes never pay for it. func (c *Engine) recordStage(lane, displayRepo, owner, repo string, pr *gh.PullRequest, reason string, merged bool, err error) { + c.observeBasePreparation(lane, owner, repo, pr, reason, merged, err) policy := c.stagePolicy() if !policy.Enabled || pr == nil || pr.GetNumber() <= 0 { return diff --git a/src/pkg/github/automerge/automerge_sweep.go b/src/pkg/github/automerge/automerge_sweep.go index 17a2bd2f5a..39420d32f7 100644 --- a/src/pkg/github/automerge/automerge_sweep.go +++ b/src/pkg/github/automerge/automerge_sweep.go @@ -82,6 +82,9 @@ const autoMergeReasonLabelNeedsRebase = "label:needs-rebase" // Options carries policy dependencies owned by the caller. type Options struct { + // BasePreparation persists front-only final preparation across all lanes. + // nil uses the shared durable store. + BasePreparation *BasePreparationStore Logger *slog.Logger MergerAuthorizer MergerAuthorizer RequiredChecks map[string]bool @@ -239,9 +242,10 @@ type Options struct { // Engine owns the automerge sweep policy state. type Engine struct { - transport Transport - gh *gh.Client - logger *slog.Logger + basePreparation *BasePreparationStore + transport Transport + gh *gh.Client + logger *slog.Logger inactiveQueueMu sync.Mutex inactiveQueueLast map[string]time.Time @@ -386,6 +390,7 @@ func New(transport Transport, opts Options) *Engine { ghClient = transport.GoGitHub() } e := &Engine{ + basePreparation: opts.BasePreparation, transport: transport, gh: ghClient, logger: opts.Logger, @@ -1912,7 +1917,7 @@ func (c *Engine) trySweepSelfAuthoredPR(ctx context.Context, displayRepo, owner, // current-base gate stays in force unless the operator explicitly set // auto_merge.native_merge_queue.skip_current_base_gate. queueSkipsBase := sync.OnceValue(func() bool { return c.nativeQueueSkipsCurrentBase(ctx, owner+"/"+repo, pr) }) - if strings.EqualFold(pr.GetMergeableState(), "behind") && !queueSkipsBase() { + if strings.EqualFold(pr.GetMergeableState(), "behind") && !queueSkipsBase() && !c.requireCurrentBaseFor(owner+"/"+repo) { if !branchUpdateAllowed { // Keep the cause: still behind, retried next tick, not CI // state and not author work (hivecommons/hive#12607). @@ -1991,7 +1996,10 @@ func (c *Engine) trySweepSelfAuthoredPR(ctx context.Context, displayRepo, owner, // head policy refuses would be updated for nothing. Sits before the desk // so the desk is only asked about a head that would merge this tick. if c.requireCurrentBaseFor(owner+"/"+repo) && !queueSkipsBase() { - if ev, reason, stop, err := c.currentBaseGate(ctx, displayRepo, owner, repo, number, baseBranch, evaluatedHeadSHA, branchUpdateAllowed, autoMergeReasonBehindBudgetExhausted, c.isForkPR(pr)); stop { + if reviewReason := c.mergeReviewGate(ctx, displayRepo, owner, repo, lane, number, pr, evaluatedHeadSHA, priorApprover, priorChecked); reviewReason != "" { + return AutoMergeSweepEvent{}, reviewReason, nil + } + if ev, reason, stop, err := c.currentBaseGate(ctx, displayRepo, owner, repo, number, baseBranch, evaluatedHeadSHA, branchUpdateAllowed, autoMergeReasonBehindBudgetExhausted, c.isForkPR(pr), stageLaneSelf); stop { return ev, reason, err } } @@ -2415,7 +2423,7 @@ func (c *Engine) trySweepTrustedAuthorPR(ctx context.Context, displayRepo, owner mergeableState := strings.ToLower(strings.TrimSpace(pr.GetMergeableState())) // See trySweepSelfAuthoredPR: the queue bypass is an explicit opt-in. queueSkipsBase := sync.OnceValue(func() bool { return c.nativeQueueSkipsCurrentBase(ctx, owner+"/"+repo, pr) }) - if mergeableState == "behind" && !queueSkipsBase() { + if mergeableState == "behind" && !queueSkipsBase() && !c.requireCurrentBaseFor(owner+"/"+repo) { headSHA := "" if pr.GetHead() != nil { headSHA = pr.GetHead().GetSHA() @@ -2470,7 +2478,10 @@ func (c *Engine) trySweepTrustedAuthorPR(ctx context.Context, displayRepo, owner // Current-base gate (hivecommons/hive#13634): only a green head that // every gate above lets through is worth an update-branch commit. if c.requireCurrentBaseFor(owner+"/"+repo) && !queueSkipsBase() { - if ev, reason, stop, err := c.currentBaseGate(ctx, displayRepo, owner, repo, number, baseBranch, headSHA, true, "behind", c.isForkPR(pr)); stop { + if reviewReason := c.mergeReviewGate(ctx, displayRepo, owner, repo, stageLaneTrusted, number, pr, headSHA, priorApprover, priorChecked); reviewReason != "" { + return AutoMergeSweepEvent{}, reviewReason, nil + } + if ev, reason, stop, err := c.currentBaseGate(ctx, displayRepo, owner, repo, number, baseBranch, headSHA, true, "behind", c.isForkPR(pr), stageLaneTrusted); stop { return ev, reason, err } } @@ -2580,6 +2591,12 @@ func (c *Engine) isForkPR(pr *gh.PullRequest) bool { func (c *Engine) updateBranchAndSkip(ctx context.Context, displayRepo, owner, repo string, number int, headSHA, reason string, fork bool) (AutoMergeSweepEvent, string, error) { if err := c.executeUpdateBranchEffect(ctx, owner, repo, number, headSHA, reason); err != nil { + if wait := preparationWaitReason(err); wait != "" { + return AutoMergeSweepEvent{}, wait, nil + } + if c.requireCurrentBaseFor(owner+"/"+repo) && hgithub.IsUpdateBranchStaleHead(err) { + return AutoMergeSweepEvent{}, basePreparationPrefix + "snapshot-changed", nil + } switch classifyUpdateBranchError(err, fork) { case updateBranchUpToDate: return AutoMergeSweepEvent{}, "check-pending", nil @@ -2592,6 +2609,9 @@ func (c *Engine) updateBranchAndSkip(ctx context.Context, displayRepo, owner, re return AutoMergeSweepEvent{}, "update-branch", err } c.info("automerge update-branch", "repo", displayRepo, "pr", number, "reason", reason) + if c.requireCurrentBaseFor(owner + "/" + repo) { + return AutoMergeSweepEvent{Repo: displayRepo, Number: number, HeadSHA: headSHA, BranchUpdated: true}, basePreparationReady, nil + } return AutoMergeSweepEvent{Repo: displayRepo, Number: number, HeadSHA: headSHA, BranchUpdated: true}, "check-pending", nil } @@ -2642,6 +2662,11 @@ func classifyUpdateBranchError(err error, fork bool) updateBranchFailure { } func (c *Engine) executeUpdateBranchEffect(ctx context.Context, owner, repo string, number int, headSHA, reason string) error { + if c.requireCurrentBaseFor(owner + "/" + repo) { + if err := c.claimBasePreparationUpdate(ctx, owner, repo, number, headSHA); err != nil { + return err + } + } inputs := map[string]string{"expected_head_sha": headSHA, "reason": reason} observe := func(ctx context.Context) (effects.ExternalState, error) { pr, _, getErr := c.gh.PullRequests.Get(hgithub.WithRESTCaller(ctx, "hive:automerge_reconcile_branch_update"), owner, repo, number) @@ -2671,6 +2696,11 @@ func (c *Engine) executeUpdateBranchEffect(ctx context.Context, owner, repo stri } return effects.Result{Provenance: owner + "/" + repo + "#" + strconv.Itoa(number)}, nil }) + var rejected *gh.ErrorResponse + if c.requireCurrentBaseFor(owner+"/"+repo) && errors.As(err, &rejected) && rejected.Response != nil && + (rejected.Response.StatusCode == http.StatusForbidden || rejected.Response.StatusCode == http.StatusUnprocessableEntity) { + c.forgetRejectedPreparationUpdate(owner, repo, number, headSHA) + } return err } diff --git a/src/pkg/github/automerge/automerge_sweep_test.go b/src/pkg/github/automerge/automerge_sweep_test.go index d59bfe07b0..bce088ee0d 100644 --- a/src/pkg/github/automerge/automerge_sweep_test.go +++ b/src/pkg/github/automerge/automerge_sweep_test.go @@ -28,6 +28,8 @@ type sweepPR struct { reviewAuthor string reviewState string headSHA string + baseRef string + baseSHA string reviewCommitID *string label bool extraLabels []string @@ -2890,7 +2892,7 @@ func newAutoMergeSweepClient(apiURL string) *Engine { // Audit F3: the sweep fails CLOSED without an authorizer, so every test // exercising the merge path must install one. Tests that assert the // fail-closed behaviour itself build their client without this helper. - return New(client, Options{HeadFirstSeen: NewHeadFirstSeenTracker(), MergerAuthorizer: func(login string) bool { + return New(client, Options{BasePreparation: NewBasePreparationStore(""), HeadFirstSeen: NewHeadFirstSeenTracker(), MergerAuthorizer: func(login string) bool { return testTrustedMergers[strings.ToLower(strings.TrimSpace(login))] }}) } @@ -2941,6 +2943,8 @@ func newAutoMergeSweepAPI(t *testing.T, expectedLabel string, prs []sweepPR, mer "repo": map[string]string{"full_name": headRepo}, }, "base": map[string]any{ + "ref": pr.baseRef, + "sha": pr.baseSHA, "repo": map[string]string{"full_name": "acme/widget"}, }, "labels": issueLabels(expectedLabel, pr.extraLabels), @@ -2997,7 +3001,7 @@ func newAutoMergeSweepAPI(t *testing.T, expectedLabel string, prs []sweepPR, mer "mergeable": pr.mergeableState == "clean", "user": map[string]string{"login": pr.author}, "head": map[string]any{"sha": headSHA, "repo": map[string]string{"full_name": headRepo}}, - "base": map[string]any{"repo": map[string]string{"full_name": "acme/widget"}}, + "base": map[string]any{"ref": pr.baseRef, "sha": pr.baseSHA, "repo": map[string]string{"full_name": "acme/widget"}}, "labels": prLabels, }) case r.Method == http.MethodGet && strings.HasPrefix(r.URL.Path, "/repos/acme/widget/commits/") && strings.HasSuffix(r.URL.Path, "/status") && hasShaNumber(r.URL.Path): diff --git a/src/pkg/github/automerge/base_preparation_store.go b/src/pkg/github/automerge/base_preparation_store.go new file mode 100644 index 0000000000..6d1c7e3880 --- /dev/null +++ b/src/pkg/github/automerge/base_preparation_store.go @@ -0,0 +1,136 @@ +package automerge + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "os" + "path/filepath" + "strings" + "sync" + "syscall" + "time" +) + +// BasePreparationStore serializes final preparation independently of merge +// train membership. A nonempty directory must be on the hive's durable volume. +// Each destination has its own flock and atomic snapshot, including the +// write-ahead update identity, so overlapping sweeps and restarting pods agree. +type BasePreparationStore struct { + dir string + mu sync.Mutex + locks map[string]*sync.Mutex + items map[string]basePreparation +} + +type preparationCandidate struct { + Number int `json:"number"` + Head string `json:"head"` + Lane string `json:"lane"` + Repair bool `json:"repair,omitempty"` + Since time.Time `json:"since"` +} + +type preparationAttempt struct { + Head string `json:"head"` + Base string `json:"base"` + At time.Time `json:"at"` +} + +type basePreparation struct { + Owner preparationCandidate `json:"owner"` + Waiting []preparationCandidate `json:"waiting,omitempty"` + Attempts map[int]preparationAttempt `json:"attempts,omitempty"` +} + +// NewBasePreparationStore accepts an empty directory for isolated, in-memory +// tests. Production uses a directory beside the durable stage store. +func NewBasePreparationStore(dir string) *BasePreparationStore { + return &BasePreparationStore{dir: dir, items: map[string]basePreparation{}, locks: map[string]*sync.Mutex{}} +} + +func preparationKey(owner, repo, base string) string { + // GitHub repository names are case insensitive; refs are not. + return strings.ToLower(strings.TrimSpace(owner)+"/"+strings.TrimSpace(repo)) + ":" + strings.TrimSpace(base) +} + +func (s *BasePreparationStore) change(key string, fn func(*basePreparation) error) error { + s.mu.Lock() + lockMu := s.locks[key] + if lockMu == nil { + lockMu = &sync.Mutex{} + s.locks[key] = lockMu + } + s.mu.Unlock() + lockMu.Lock() + defer lockMu.Unlock() + if s.dir == "" { + s.mu.Lock() + st := s.items[key] + s.mu.Unlock() + if err := fn(&st); err != nil { + return err + } + s.mu.Lock() + s.items[key] = st + s.mu.Unlock() + return nil + } + const directoryMode = 0o775 + const fileMode = 0o664 + if err := os.MkdirAll(s.dir, directoryMode); err != nil { + return err + } + sum := sha256.Sum256([]byte(key)) + path := filepath.Join(s.dir, hex.EncodeToString(sum[:])) + lock, err := os.OpenFile(path+".lock", os.O_CREATE|os.O_RDWR, fileMode) + if err != nil { + return err + } + defer lock.Close() + if err := syscall.Flock(int(lock.Fd()), syscall.LOCK_EX); err != nil { + return err + } + defer syscall.Flock(int(lock.Fd()), syscall.LOCK_UN) // process death also releases the lock + var st basePreparation + data, err := os.ReadFile(path + ".json") + if err == nil { + if err := json.Unmarshal(data, &st); err != nil { + return err // corrupt or unreadable ownership must fail closed + } + } else if !errors.Is(err, os.ErrNotExist) { + return err + } + if err := fn(&st); err != nil { + return err + } + data, err = json.Marshal(st) + if err != nil { + return err + } + // The separate lock inode remains stable while the snapshot is replaced. + f, err := os.OpenFile(path+".next", os.O_CREATE|os.O_TRUNC|os.O_WRONLY, fileMode) + if err != nil { + return err + } + if _, err = f.Write(data); err == nil { + err = f.Sync() + } + closeErr := f.Close() + if err != nil { + return err + } + if closeErr != nil { + return closeErr + } + if err := os.Rename(path+".next", path+".json"); err != nil { + return err + } + dir, err := os.Open(s.dir) + if err != nil { + return err + } + defer dir.Close() + return dir.Sync() +} diff --git a/src/pkg/github/base_refresh_policy.go b/src/pkg/github/base_refresh_policy.go new file mode 100644 index 0000000000..f5fa218bf3 --- /dev/null +++ b/src/pkg/github/base_refresh_policy.go @@ -0,0 +1,18 @@ +package github + +// SetScheduledBaseRefresh installs the live current-base scheduling policy. +// Automatic client-side updates (follow-up, merge relay, supersession) defer +// to the automerge engine's shared preparation owner on these repositories. +// The engine uses its own head-pinned REST mutation after admission. +func (c *Client) SetScheduledBaseRefresh(policy func(repo string) bool) { + c.reposMu.Lock() + defer c.reposMu.Unlock() + c.scheduledBaseRefresh = policy +} + +func (c *Client) baseRefreshScheduled(repo string) bool { + c.reposMu.RLock() + policy := c.scheduledBaseRefresh + c.reposMu.RUnlock() + return policy != nil && policy(repo) +} diff --git a/src/pkg/github/base_refresh_policy_test.go b/src/pkg/github/base_refresh_policy_test.go new file mode 100644 index 0000000000..1b7bfbf31c --- /dev/null +++ b/src/pkg/github/base_refresh_policy_test.go @@ -0,0 +1,40 @@ +package github + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +func TestScheduledBaseRefreshDefersAllClientUpdatePaths(t *testing.T) { + updates := 0 + api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + updates++ + w.WriteHeader(http.StatusAccepted) + _, _ = w.Write([]byte(`{"message":"Updating pull request branch"}`)) + })) + defer api.Close() + c := NewClient("token", "acme", []string{"widget"}, nil, api.URL) + scheduled := true + c.SetScheduledBaseRefresh(func(repo string) bool { return scheduled && repo == "acme/widget" }) + for _, update := range []func() error{ + func() error { return c.UpdateBranch(context.Background(), "acme/widget", 1) }, + func() error { return c.UpdateBranchExpectedHead(context.Background(), "widget", 1, "head", "base") }, + } { + if err := update(); err == nil || !strings.Contains(err.Error(), "base-refresh:awaiting-admission") { + t.Fatalf("client update bypassed preparation: %v", err) + } + } + if updates != 0 { + t.Fatalf("queued client paths mutated branches: %d", updates) + } + scheduled = false // live policy changes preserve opted-out behaviour + if err := c.UpdateBranchExpectedHead(context.Background(), "widget", 1, "head", "base"); err != nil { + t.Fatal(err) + } + if updates != 1 { + t.Fatalf("non-policy repo updates=%d, want 1", updates) + } +} diff --git a/src/pkg/github/client.go b/src/pkg/github/client.go index 79e2f15c9b..eaf3d7c93a 100644 --- a/src/pkg/github/client.go +++ b/src/pkg/github/client.go @@ -75,7 +75,8 @@ type Client struct { // enumeration goroutine is deciding what work exists. Nil (the zero value, // and what every test constructs) means nothing is paused, so the client // behaves exactly as it did before. - repoPaused func(repo string) bool + repoPaused func(repo string) bool + scheduledBaseRefresh func(repo string) bool // writeback is the unsolicited write-back gate (writeback_gate.go); // nil allows everything. writeback *writeback.Gate diff --git a/src/pkg/github/pullrequest.go b/src/pkg/github/pullrequest.go index a695161f99..1baafe8768 100644 --- a/src/pkg/github/pullrequest.go +++ b/src/pkg/github/pullrequest.go @@ -442,7 +442,7 @@ func (c *Client) MergePRVia(ctx context.Context, repo string, number int, mergeM // Issue-author credit (hivecommons/hive#14581) rides the PR already // fetched above; a merge-commit request fetches it only when the credit // is on for the repo. - if creditor := c.MergeCreditor(); creditor.Enabled(owner+"/"+repo) { + if creditor := c.MergeCreditor(); creditor.Enabled(owner + "/" + repo) { if fetched == nil && !fetchedTried { if pr, _, err := c.client.PullRequests.Get(WithRESTCaller(ctx, mergeCreditCaller), owner, repo, number); err == nil { fetched = pr @@ -598,6 +598,9 @@ func (c *Client) updateBranch(ctx context.Context, repo string, number int, expe owner, repo = parts[0], parts[1] } var inputs map[string]string + if c.baseRefreshScheduled(owner + "/" + repo) { + return fmt.Errorf("base-refresh:awaiting-admission: %s/%s#%d is refreshed only by the automerge preparation front", owner, repo, number) + } if expectedHeadSHA != "" || observedBaseSHA != "" { inputs = map[string]string{"expected_head_sha": expectedHeadSHA, "observed_base_sha": observedBaseSHA} } diff --git a/src/pkg/stageengine/derive_pr_classify.go b/src/pkg/stageengine/derive_pr_classify.go index abca3bc989..5a9ca1cb0b 100644 --- a/src/pkg/stageengine/derive_pr_classify.go +++ b/src/pkg/stageengine/derive_pr_classify.go @@ -372,6 +372,10 @@ func ClassifyStage(o stageObservation) stageVerdict { if stageHandoffReasons[reason] { return stageVerdict{} } + if strings.HasPrefix(reason, "base-refresh:") { + // Preparation/queued is not ready-to-merge or a merge train seat. + return stageVerdict{Stage: stagelabels.ReviewPending, Rank: stageRankOwned, Reason: reason} + } if train, ok := prstage.ParseMergeTrain(reason); ok { // The merge train parked this PR behind an overlapping front // (hivecommons/hive#15117): a hive-side wait at the merge gate,