From 9f03835eed2d7695d1b4cd93c5a76c0602552d95 Mon Sep 17 00:00:00 2001 From: Andrew Anderson Date: Mon, 2 Nov 2026 13:58:01 -0500 Subject: [PATCH] =?UTF-8?q?=F0=9F=90=9B=20fix(automerge):=20fence=20prepar?= =?UTF-8?q?ation=20settlement=20by=20live=20identity?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Andrew Anderson --- .../fixed-preparation-observation-identity.md | 1 + src/docs/hive-merge.md | 13 ++ .../automerge/automerge_base_preparation.go | 60 ++++- .../automerge_base_preparation_races_test.go | 1 + .../automerge_base_preparation_test.go | 5 + .../automerge/base_preparation_identity.go | 31 +++ .../base_preparation_identity_test.go | 219 ++++++++++++++++++ 7 files changed, 321 insertions(+), 9 deletions(-) create mode 100644 changelog.d/fixed-preparation-observation-identity.md create mode 100644 src/pkg/github/automerge/base_preparation_identity.go create mode 100644 src/pkg/github/automerge/base_preparation_identity_test.go diff --git a/changelog.d/fixed-preparation-observation-identity.md b/changelog.d/fixed-preparation-observation-identity.md new file mode 100644 index 0000000000..20a5b3f6e6 --- /dev/null +++ b/changelog.d/fixed-preparation-observation-identity.md @@ -0,0 +1 @@ +- Fence preparation settlement against superseded head/base/review inputs and concurrent owner or attempt changes, preserving unresolved refresh ownership and fresh-review requirements without treating required-green as release authority. diff --git a/src/docs/hive-merge.md b/src/docs/hive-merge.md index fa451fc8ba..40c1867642 100644 --- a/src/docs/hive-merge.md +++ b/src/docs/hive-merge.md @@ -431,6 +431,19 @@ Admission provenance prevents a different lane's non-admission from revoking the owning lane's reservation or FIFO ticket. An old waiting ticket reserves order only: its head must qualify afresh in its lane before becoming owner. +The persisted owner head is the **admitted source head**, so a different live +head is not by itself a stale reservation: an App-authored refresh preserves +that source and its write-ahead attempt while the new head earns CI and a fresh +review. Required-green alone does not release ownership or confer approval. +State-changing lane observations re-read the head/base, author, policy labels +and review inputs outside the preparation lock, then compare the persisted +owner/attempt or exact waiting ticket before settling it. Superseded callbacks +cannot release a newer reservation or rewind a yielded head. An accepted refresh +whose expected head is still live remains exclusive through the existing +mutation-reconciliation window, even if an older evaluation reports failure. +Unavailable or changed evidence retains ownership for normal reevaluation; +neither this validation nor a dashboard snapshot forcibly clears the queue. + Audit logs use `automerge base preparation selected`, `refresh claimed`, `waiting`, `released` and `completed owner yielded`, with repository, base, PR, head/base SHA where relevant, and release/skip reason. `automerge update-branch` and the sweep's diff --git a/src/pkg/github/automerge/automerge_base_preparation.go b/src/pkg/github/automerge/automerge_base_preparation.go index e7faa362ca..2ca2726ca9 100644 --- a/src/pkg/github/automerge/automerge_base_preparation.go +++ b/src/pkg/github/automerge/automerge_base_preparation.go @@ -312,27 +312,69 @@ func (c *Engine) observeBasePreparation(lane, owner, repo string, pr *gh.PullReq if base == "" { return } - if err := c.preparationStore().change(preparationKey(owner, repo, base), func(st *basePreparation) error { - keep := !merged && preparationKeepsOwner(reason, evalErr) - if reason == basePreparationPrefix+"snapshot-changed" && st.Attempts[pr.GetNumber()].Head == "" { - // A push won the race before we issued any refresh. Pending - // work on that unreviewed head is not our preparation. - keep = false + key := preparationKey(owner, repo, base) + snapshot, err := c.preparationStore().ownerSnapshot(key) + if err != nil { + c.warn("automerge base preparation persistence failed", "repo", owner+"/"+repo, "pr", pr.GetNumber(), "error", err) + return + } + keep := !merged && preparationKeepsOwner(reason, evalErr) + if reason == basePreparationPrefix+"snapshot-changed" && snapshot.Owner.Number == pr.GetNumber() && snapshot.Attempt.Head == "" { + // A push won the race before we issued any refresh. Pending + // work on that unreviewed head is not our preparation. + keep = false + } + owns := snapshot.Owner.Lane == lane && snapshot.Owner.Number == pr.GetNumber() + changes := owns && !keep + for _, w := range snapshot.Waiters { + if w.Lane == lane && w.Number == pr.GetNumber() { + changes = changes || w.Head != pr.GetHead().GetSHA() || + (!(w.Yielded && keep && !merged) && !strings.HasPrefix(reason, basePreparationPrefix)) + } + } + if !changes || c.gh == nil { + return + } + ctx, cancel := context.WithTimeout(context.Background(), preparationObservationTimeout) + defer cancel() + current, _, err := c.gh.PullRequests.Get(hgithub.WithRESTCaller(ctx, "hive:base_preparation_settlement"), owner, repo, pr.GetNumber()) + if err != nil || !preparationObservationIdentity(pr, current) { + c.info("automerge base preparation observation skipped", "repo", owner+"/"+repo, "pr", pr.GetNumber(), "reason", "unverified-current-identity", "error", err) + return + } + if owns && !keep && !merged && snapshot.Attempt.Head != "" && + snapshot.Attempt.Head == current.GetHead().GetSHA() && + c.now().Sub(snapshot.Attempt.At) < effects.DefaultUnresolvedTTL && + c.preparationInvalidPR(current, base, snapshot.Owner.Repair) == "" { + // The accepted write has not become observable. An older evaluation + // cannot release its exclusive slot while that mutation is unresolved. + c.info("automerge base preparation observation skipped", "repo", owner+"/"+repo, "pr", pr.GetNumber(), "reason", "refresh-mutation-unobserved") + return + } + if err := c.preparationStore().change(key, func(st *basePreparation) error { + if owns && (st.Owner != snapshot.Owner || st.Attempts[pr.GetNumber()] != snapshot.Attempt) { + return nil + } + matchesWaiter := func(w preparationCandidate) bool { + return w.Lane == lane && w.Number == pr.GetNumber() && slices.Contains(snapshot.Waiters, w) } // Waiting is not preparation yet: a changed or now-pending waiting // head loses its admission ticket and must qualify again. for i := range st.Waiting { - if st.Waiting[i].Yielded && st.Waiting[i].Number == pr.GetNumber() && keep { + if matchesWaiter(st.Waiting[i]) && st.Waiting[i].Yielded && keep { st.Waiting[i].Head = pr.GetHead().GetSHA() } } st.Waiting = slices.DeleteFunc(st.Waiting, func(w preparationCandidate) bool { + if !matchesWaiter(w) { + return false + } if w.Yielded && keep && !merged { return false } - return w.Lane == lane && w.Number == pr.GetNumber() && (w.Head != pr.GetHead().GetSHA() || !strings.HasPrefix(reason, basePreparationPrefix)) + return w.Head != pr.GetHead().GetSHA() || !strings.HasPrefix(reason, basePreparationPrefix) }) - if !keep && st.Owner.Lane == lane && st.Owner.Number == pr.GetNumber() { + if owns && !keep && st.Owner.Lane == lane && st.Owner.Number == pr.GetNumber() { if merged { reason = "merged" delete(st.Attempts, pr.GetNumber()) diff --git a/src/pkg/github/automerge/automerge_base_preparation_races_test.go b/src/pkg/github/automerge/automerge_base_preparation_races_test.go index 0036797715..9a0ea05180 100644 --- a/src/pkg/github/automerge/automerge_base_preparation_races_test.go +++ b/src/pkg/github/automerge/automerge_base_preparation_races_test.go @@ -23,6 +23,7 @@ func TestBasePreparationProvenanceForEveryAdmittingLane(t *testing.T) { if _, reason, _, err := preparationGate(c, 2, "main", "head2"); err != nil || reason != preparationQueued(1) { t.Fatalf("other lane revoked %s owner: %q %v", lane, reason, err) } + api.prs[1].Head.SHA = gh.Ptr("observed-refresh") c.recordStage(lane, "acme/widget", "acme", "widget", api.prs[1], "required-check-failing:build", false, nil) if ev, _, _, err := preparationGate(c, 2, "main", "head2"); err != nil || !ev.BranchUpdated { t.Fatalf("owning lane did not release: %+v %v", ev, err) diff --git a/src/pkg/github/automerge/automerge_base_preparation_test.go b/src/pkg/github/automerge/automerge_base_preparation_test.go index 065d169932..fc30dd3538 100644 --- a/src/pkg/github/automerge/automerge_base_preparation_test.go +++ b/src/pkg/github/automerge/automerge_base_preparation_test.go @@ -38,6 +38,7 @@ type preparationAPI struct { dependencyBlocked bool truncated bool onFiles func(*preparationAPI, int) + onGet func(*preparationAPI, int) } func preparationPR(number int, base, head string) *gh.PullRequest { @@ -81,6 +82,9 @@ func newPreparationAPI(t *testing.T) (*preparationAPI, *httptest.Server) { case "pulls": n, _ := strconv.Atoi(path[4]) if len(path) == 5 { + if api.onGet != nil { + api.onGet(api, n) + } if api.prs[n] == nil { w.WriteHeader(http.StatusNotFound) } else { @@ -270,6 +274,7 @@ func TestBasePreparationReleasesInvalidFront(t *testing.T) { case "expired": now = now.Add(c.preparationWindow()) case "ci-red", "review-rejected", "withdrawn": + api.prs[1].Head.SHA = gh.Ptr("observed-refresh") reason := "required-check-failing:build" if why == "review-rejected" { reason = autoMergeReasonReviewGatePrefix + ReviewGateOutcomeChangesRequested diff --git a/src/pkg/github/automerge/base_preparation_identity.go b/src/pkg/github/automerge/base_preparation_identity.go new file mode 100644 index 0000000000..3e73264acc --- /dev/null +++ b/src/pkg/github/automerge/base_preparation_identity.go @@ -0,0 +1,31 @@ +package automerge + +import ( + "slices" + "strings" + "time" + + gh "github.com/google/go-github/v72/github" +) + +const preparationObservationTimeout = 10 * time.Second + +// A lane result belongs to the evaluated admission identity, not merely to a +// PR number. State/mergeability may change as our mutation completes; mutable +// authorship, head/base, policy labels and review inputs must still agree. +func preparationObservationIdentity(evaluated, current *gh.PullRequest) bool { + if evaluated == nil || current == nil || evaluated.GetHead().GetSHA() == "" { + return false + } + beforeLabels, afterLabels := labelNames(evaluated.Labels), labelNames(current.Labels) + slices.Sort(beforeLabels) + slices.Sort(afterLabels) + return evaluated.GetNumber() == current.GetNumber() && + evaluated.GetHead().GetSHA() == current.GetHead().GetSHA() && + evaluated.GetHead().GetRepo().GetFullName() == current.GetHead().GetRepo().GetFullName() && + evaluated.GetBase().GetRef() == current.GetBase().GetRef() && + evaluated.GetBase().GetSHA() == current.GetBase().GetSHA() && + strings.EqualFold(evaluated.GetUser().GetLogin(), current.GetUser().GetLogin()) && + evaluated.GetTitle() == current.GetTitle() && evaluated.GetBody() == current.GetBody() && + evaluated.GetDraft() == current.GetDraft() && slices.Equal(beforeLabels, afterLabels) +} diff --git a/src/pkg/github/automerge/base_preparation_identity_test.go b/src/pkg/github/automerge/base_preparation_identity_test.go new file mode 100644 index 0000000000..7dc5beb16a --- /dev/null +++ b/src/pkg/github/automerge/base_preparation_identity_test.go @@ -0,0 +1,219 @@ +package automerge + +import ( + "context" + "encoding/json" + "testing" + "time" + + gh "github.com/google/go-github/v72/github" + "github.com/hivecommons/hive/pkg/effects" +) + +func copyPreparationPR(t *testing.T, pr *gh.PullRequest) *gh.PullRequest { + t.Helper() + data, err := json.Marshal(pr) + if err != nil { + t.Fatal(err) + } + var out gh.PullRequest + if err := json.Unmarshal(data, &out); err != nil { + t.Fatal(err) + } + return &out +} + +func preparationOwnerForTest(t *testing.T, c *Engine) preparationOwnerSnapshot { + t.Helper() + snapshot, err := c.preparationStore().ownerSnapshot(preparationKey("acme", "widget", "main")) + if err != nil { + t.Fatal(err) + } + return snapshot +} + +func TestPreparationSettlementRejectsSupersededAdmissionIdentity(t *testing.T) { + for _, changed := range []string{"head", "base", "base-sha", "head-repo", "author", "body", "title", "labels", "draft", "unavailable"} { + t.Run(changed, func(t *testing.T) { + api, srv := newPreparationAPI(t) + c := preparationEngine(srv.URL, NewBasePreparationStore("")) + if reason, err := c.admitBasePreparation(context.Background(), "acme", "widget", "main", 1, "head1", stageLaneSelf, false); err != nil || reason != "" { + t.Fatalf("admission: %q %v", reason, err) + } + evaluated := copyPreparationPR(t, api.prs[1]) + before := preparationOwnerForTest(t, c) + switch changed { + case "head": + api.prs[1].Head.SHA = gh.Ptr("new-head") + case "base": + api.prs[1].Base.Ref = gh.Ptr("other") + case "base-sha": + api.prs[1].Base.SHA = gh.Ptr("base2") + case "head-repo": + api.prs[1].Head.Repo.FullName = gh.Ptr("other/widget") + case "author": + api.prs[1].User.Login = gh.Ptr("other-author") + case "body": + api.prs[1].Body = gh.Ptr("Closes #200") + case "title": + api.prs[1].Title = gh.Ptr("new review input") + case "labels": + api.prs[1].Labels = []*gh.Label{{Name: gh.Ptr("role/trusted")}} + case "draft": + api.prs[1].Draft = gh.Ptr(true) + case "unavailable": + delete(api.prs, 1) + } + c.observeBasePreparation(stageLaneSelf, "acme", "widget", evaluated, "required-check-failing:build", false, nil) + if after := preparationOwnerForTest(t, c); after.Owner != before.Owner || after.Attempt != before.Attempt { + t.Fatalf("stale %s callback changed ownership: before=%+v after=%+v", changed, before, after) + } + }) + } +} + +func TestPreparationUnobservedRefreshCannotBeReleasedByOldEvaluation(t *testing.T) { + for _, reason := range []string{"required-check-failing:build", autoMergeReasonReviewGatePrefix + ReviewGateOutcomeChangesRequested, "untrusted-author-role"} { + t.Run(reason, func(t *testing.T) { + api, srv := newPreparationAPI(t) + c := preparationEngine(srv.URL, NewBasePreparationStore("")) + now := time.Date(2026, 11, 2, 18, 0, 0, 0, time.UTC) + c.now = func() time.Time { return now } + if _, _, _, err := preparationGate(c, 1, "main", "head1"); err != nil { + t.Fatal(err) + } + before := preparationOwnerForTest(t, c) + c.observeBasePreparation(stageLaneSelf, "acme", "widget", api.prs[1], reason, false, nil) + if after := preparationOwnerForTest(t, c); after.Owner != before.Owner || after.Attempt != before.Attempt { + t.Fatalf("unobserved mutation lost its exclusive owner: %+v", after) + } + now = now.Add(effects.DefaultUnresolvedTTL) + c.observeBasePreparation(stageLaneSelf, "acme", "widget", api.prs[1], reason, false, nil) + if after := preparationOwnerForTest(t, c); after.Owner.Number != 0 { + t.Fatalf("validated unchanged head could not settle after normal reconciliation window: %+v", after) + } + }) + } +} + +func TestPreparationObservedRefreshRetainsFreshReviewAndSettlesValidatedFailure(t *testing.T) { + api, srv := newPreparationAPI(t) + c := preparationEngine(srv.URL, NewBasePreparationStore("")) + if _, _, _, err := preparationGate(c, 1, "main", "head1"); err != nil { + t.Fatal(err) + } + api.prs[1].Head.SHA = gh.Ptr("observed-refresh") + for _, reason := range []string{"required-check-pending:build", autoMergeReasonReviewGatePrefix + ReviewGateOutcomeNoBundle, autoMergeReasonReviewGatePrefix + ReviewGateOutcomeAwaiting} { + c.observeBasePreparation(stageLaneSelf, "acme", "widget", api.prs[1], reason, false, nil) + if after := preparationOwnerForTest(t, c); after.Owner.Number != 1 || after.Owner.Head != "head1" || after.Attempt.Head != "head1" { + t.Fatalf("expected old source identity lost while %s: %+v", reason, after) + } + } + c.observeBasePreparation(stageLaneSelf, "acme", "widget", api.prs[1], "required-check-failing:build", false, nil) + if after := preparationOwnerForTest(t, c); after.Owner.Number != 0 { + t.Fatalf("fresh observed-head failure did not settle old source reservation: %+v", after) + } + if ev, _, _, err := preparationGate(c, 2, "main", "head2"); err != nil || !ev.BranchUpdated { + t.Fatalf("normally admitted successor could not refresh: %+v %v", ev, err) + } +} + +func TestPreparationSettlementFencesOwnerAndAttemptChangesDuringRead(t *testing.T) { + for _, change := range []string{"owner", "attempt", "promoted-waiter"} { + t.Run(change, func(t *testing.T) { + api, srv := newPreparationAPI(t) + c := preparationEngine(srv.URL, NewBasePreparationStore("")) + key := preparationKey("acme", "widget", "main") + if reason, err := c.admitBasePreparation(context.Background(), "acme", "widget", "main", 1, "head1", stageLaneSelf, false); err != nil || reason != "" { + t.Fatalf("admission: %q %v", reason, err) + } + if change == "promoted-waiter" { + if err := c.preparationStore().change(key, func(st *basePreparation) error { + st.Waiting = []preparationCandidate{st.Owner} + st.Owner = preparationCandidate{Number: 2, Head: "head2", Lane: stageLaneSelf, Since: c.now()} + return nil + }); err != nil { + t.Fatal(err) + } + } + var concurrent preparationOwnerSnapshot + api.onGet = func(a *preparationAPI, _ int) { + a.onGet = nil + if err := c.preparationStore().change(key, func(st *basePreparation) error { + switch change { + case "owner": + st.Owner.Since = st.Owner.Since.Add(time.Second) + case "attempt": + st.Attempts = map[int]preparationAttempt{1: {Head: "head1", Base: "base2", At: c.now(), AuthoritativeBase: true}} + case "promoted-waiter": + st.Owner = st.Waiting[0] + st.Waiting = nil + } + return nil + }); err != nil { + t.Error(err) + } + concurrent = preparationOwnerForTest(t, c) + } + c.observeBasePreparation(stageLaneSelf, "acme", "widget", api.prs[1], "required-check-failing:build", false, nil) + if after := preparationOwnerForTest(t, c); after.Owner != concurrent.Owner || after.Attempt != concurrent.Attempt { + t.Fatalf("callback overwrote concurrent %s: concurrent=%+v after=%+v", change, concurrent, after) + } + }) + } +} + +func TestPreparationStaleCallbackCannotRewriteYieldedHead(t *testing.T) { + api, srv := newPreparationAPI(t) + c := preparationEngine(srv.URL, NewBasePreparationStore("")) + evaluated := copyPreparationPR(t, api.prs[1]) + api.prs[1].Head.SHA = gh.Ptr("new-head") + key := preparationKey("acme", "widget", "main") + if err := c.preparationStore().change(key, func(st *basePreparation) error { + st.Owner = preparationCandidate{Number: 2, Head: "head2", Lane: stageLaneSelf, Since: c.now()} + st.Waiting = []preparationCandidate{{Number: 1, Head: "new-head", Lane: stageLaneSelf, Yielded: true, Since: c.now()}} + return nil + }); err != nil { + t.Fatal(err) + } + c.observeBasePreparation(stageLaneSelf, "acme", "widget", evaluated, "required-check-pending:build", false, nil) + if after := preparationOwnerForTest(t, c); after.Owner.Number != 2 || len(after.Waiters) != 1 || after.Waiters[0].Head != "new-head" { + t.Fatalf("stale callback rewound yielded head: %+v", after) + } +} + +func TestPreparationValidatedYieldedRenewalDoesNotOverwriteNewerTicket(t *testing.T) { + for _, concurrent := range []bool{false, true} { + t.Run(map[bool]string{false: "validated", true: "superseded-during-read"}[concurrent], func(t *testing.T) { + api, srv := newPreparationAPI(t) + c := preparationEngine(srv.URL, NewBasePreparationStore("")) + key := preparationKey("acme", "widget", "main") + if err := c.preparationStore().change(key, func(st *basePreparation) error { + st.Owner = preparationCandidate{Number: 2, Head: "head2", Lane: stageLaneSelf, Since: c.now()} + st.Waiting = []preparationCandidate{{Number: 1, Head: "old-head", Lane: stageLaneSelf, Yielded: true, Since: c.now()}} + return nil + }); err != nil { + t.Fatal(err) + } + if concurrent { + api.onGet = func(a *preparationAPI, _ int) { + a.onGet = nil + if err := c.preparationStore().change(key, func(st *basePreparation) error { + st.Waiting[0].Head = "newer-ticket" + return nil + }); err != nil { + t.Error(err) + } + } + } + c.observeBasePreparation(stageLaneSelf, "acme", "widget", api.prs[1], "required-check-pending:build", false, nil) + want := "head1" + if concurrent { + want = "newer-ticket" + } + if after := preparationOwnerForTest(t, c); after.Owner.Number != 2 || len(after.Waiters) != 1 || after.Waiters[0].Head != want { + t.Fatalf("invalid yielded renewal: %+v, want head %s", after, want) + } + }) + } +}