Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/fixed-preparation-observation-identity.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- Fence preparation settlement against superseded head/base/review inputs and concurrent owner or attempt changes, preserving unresolved refresh ownership and fresh-review requirements without treating required-green as release authority.
13 changes: 13 additions & 0 deletions src/docs/hive-merge.md
Original file line number Diff line number Diff line change
Expand Up @@ -431,6 +431,19 @@ Admission provenance prevents a different lane's non-admission from revoking
the owning lane's reservation or FIFO ticket. An old waiting ticket reserves
order only: its head must qualify afresh in its lane before becoming owner.

The persisted owner head is the **admitted source head**, so a different live
head is not by itself a stale reservation: an App-authored refresh preserves
that source and its write-ahead attempt while the new head earns CI and a fresh
review. Required-green alone does not release ownership or confer approval.
State-changing lane observations re-read the head/base, author, policy labels
and review inputs outside the preparation lock, then compare the persisted
owner/attempt or exact waiting ticket before settling it. Superseded callbacks
cannot release a newer reservation or rewind a yielded head. An accepted refresh
whose expected head is still live remains exclusive through the existing
mutation-reconciliation window, even if an older evaluation reports failure.
Unavailable or changed evidence retains ownership for normal reevaluation;
neither this validation nor a dashboard snapshot forcibly clears the queue.

Audit logs use `automerge base preparation selected`, `refresh claimed`,
`waiting`, `released` and `completed owner yielded`, with repository, base, PR,
head/base SHA where relevant, and release/skip reason. `automerge update-branch` and the sweep's
Expand Down
60 changes: 51 additions & 9 deletions src/pkg/github/automerge/automerge_base_preparation.go
Original file line number Diff line number Diff line change
Expand Up @@ -312,27 +312,69 @@ func (c *Engine) observeBasePreparation(lane, owner, repo string, pr *gh.PullReq
if base == "" {
return
}
if err := c.preparationStore().change(preparationKey(owner, repo, base), func(st *basePreparation) error {
keep := !merged && preparationKeepsOwner(reason, evalErr)
if reason == basePreparationPrefix+"snapshot-changed" && st.Attempts[pr.GetNumber()].Head == "" {
// A push won the race before we issued any refresh. Pending
// work on that unreviewed head is not our preparation.
keep = false
key := preparationKey(owner, repo, base)
snapshot, err := c.preparationStore().ownerSnapshot(key)
if err != nil {
c.warn("automerge base preparation persistence failed", "repo", owner+"/"+repo, "pr", pr.GetNumber(), "error", err)
return
}
keep := !merged && preparationKeepsOwner(reason, evalErr)
if reason == basePreparationPrefix+"snapshot-changed" && snapshot.Owner.Number == pr.GetNumber() && snapshot.Attempt.Head == "" {
// A push won the race before we issued any refresh. Pending
// work on that unreviewed head is not our preparation.
keep = false
}
owns := snapshot.Owner.Lane == lane && snapshot.Owner.Number == pr.GetNumber()
changes := owns && !keep
for _, w := range snapshot.Waiters {
if w.Lane == lane && w.Number == pr.GetNumber() {
changes = changes || w.Head != pr.GetHead().GetSHA() ||
(!(w.Yielded && keep && !merged) && !strings.HasPrefix(reason, basePreparationPrefix))
}
}
if !changes || c.gh == nil {
return
}
ctx, cancel := context.WithTimeout(context.Background(), preparationObservationTimeout)
defer cancel()
current, _, err := c.gh.PullRequests.Get(hgithub.WithRESTCaller(ctx, "hive:base_preparation_settlement"), owner, repo, pr.GetNumber())
if err != nil || !preparationObservationIdentity(pr, current) {
c.info("automerge base preparation observation skipped", "repo", owner+"/"+repo, "pr", pr.GetNumber(), "reason", "unverified-current-identity", "error", err)
return
}
if owns && !keep && !merged && snapshot.Attempt.Head != "" &&
snapshot.Attempt.Head == current.GetHead().GetSHA() &&
c.now().Sub(snapshot.Attempt.At) < effects.DefaultUnresolvedTTL &&
c.preparationInvalidPR(current, base, snapshot.Owner.Repair) == "" {
// The accepted write has not become observable. An older evaluation
// cannot release its exclusive slot while that mutation is unresolved.
c.info("automerge base preparation observation skipped", "repo", owner+"/"+repo, "pr", pr.GetNumber(), "reason", "refresh-mutation-unobserved")
return
}
if err := c.preparationStore().change(key, func(st *basePreparation) error {
if owns && (st.Owner != snapshot.Owner || st.Attempts[pr.GetNumber()] != snapshot.Attempt) {
return nil
}
matchesWaiter := func(w preparationCandidate) bool {
return w.Lane == lane && w.Number == pr.GetNumber() && slices.Contains(snapshot.Waiters, w)
}
// Waiting is not preparation yet: a changed or now-pending waiting
// head loses its admission ticket and must qualify again.
for i := range st.Waiting {
if st.Waiting[i].Yielded && st.Waiting[i].Number == pr.GetNumber() && keep {
if matchesWaiter(st.Waiting[i]) && st.Waiting[i].Yielded && keep {
st.Waiting[i].Head = pr.GetHead().GetSHA()
}
}
st.Waiting = slices.DeleteFunc(st.Waiting, func(w preparationCandidate) bool {
if !matchesWaiter(w) {
return false
}
if w.Yielded && keep && !merged {
return false
}
return w.Lane == lane && w.Number == pr.GetNumber() && (w.Head != pr.GetHead().GetSHA() || !strings.HasPrefix(reason, basePreparationPrefix))
return w.Head != pr.GetHead().GetSHA() || !strings.HasPrefix(reason, basePreparationPrefix)
})
if !keep && st.Owner.Lane == lane && st.Owner.Number == pr.GetNumber() {
if owns && !keep && st.Owner.Lane == lane && st.Owner.Number == pr.GetNumber() {
if merged {
reason = "merged"
delete(st.Attempts, pr.GetNumber())
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ func TestBasePreparationProvenanceForEveryAdmittingLane(t *testing.T) {
if _, reason, _, err := preparationGate(c, 2, "main", "head2"); err != nil || reason != preparationQueued(1) {
t.Fatalf("other lane revoked %s owner: %q %v", lane, reason, err)
}
api.prs[1].Head.SHA = gh.Ptr("observed-refresh")
c.recordStage(lane, "acme/widget", "acme", "widget", api.prs[1], "required-check-failing:build", false, nil)
if ev, _, _, err := preparationGate(c, 2, "main", "head2"); err != nil || !ev.BranchUpdated {
t.Fatalf("owning lane did not release: %+v %v", ev, err)
Expand Down
5 changes: 5 additions & 0 deletions src/pkg/github/automerge/automerge_base_preparation_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ type preparationAPI struct {
dependencyBlocked bool
truncated bool
onFiles func(*preparationAPI, int)
onGet func(*preparationAPI, int)
}

func preparationPR(number int, base, head string) *gh.PullRequest {
Expand Down Expand Up @@ -81,6 +82,9 @@ func newPreparationAPI(t *testing.T) (*preparationAPI, *httptest.Server) {
case "pulls":
n, _ := strconv.Atoi(path[4])
if len(path) == 5 {
if api.onGet != nil {
api.onGet(api, n)
}
if api.prs[n] == nil {
w.WriteHeader(http.StatusNotFound)
} else {
Expand Down Expand Up @@ -270,6 +274,7 @@ func TestBasePreparationReleasesInvalidFront(t *testing.T) {
case "expired":
now = now.Add(c.preparationWindow())
case "ci-red", "review-rejected", "withdrawn":
api.prs[1].Head.SHA = gh.Ptr("observed-refresh")
reason := "required-check-failing:build"
if why == "review-rejected" {
reason = autoMergeReasonReviewGatePrefix + ReviewGateOutcomeChangesRequested
Expand Down
31 changes: 31 additions & 0 deletions src/pkg/github/automerge/base_preparation_identity.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
package automerge

import (
"slices"
"strings"
"time"

gh "github.com/google/go-github/v72/github"
)

const preparationObservationTimeout = 10 * time.Second

// A lane result belongs to the evaluated admission identity, not merely to a
// PR number. State/mergeability may change as our mutation completes; mutable
// authorship, head/base, policy labels and review inputs must still agree.
func preparationObservationIdentity(evaluated, current *gh.PullRequest) bool {
if evaluated == nil || current == nil || evaluated.GetHead().GetSHA() == "" {
return false
}
beforeLabels, afterLabels := labelNames(evaluated.Labels), labelNames(current.Labels)
slices.Sort(beforeLabels)
slices.Sort(afterLabels)
return evaluated.GetNumber() == current.GetNumber() &&
evaluated.GetHead().GetSHA() == current.GetHead().GetSHA() &&
evaluated.GetHead().GetRepo().GetFullName() == current.GetHead().GetRepo().GetFullName() &&
evaluated.GetBase().GetRef() == current.GetBase().GetRef() &&
evaluated.GetBase().GetSHA() == current.GetBase().GetSHA() &&
strings.EqualFold(evaluated.GetUser().GetLogin(), current.GetUser().GetLogin()) &&
evaluated.GetTitle() == current.GetTitle() && evaluated.GetBody() == current.GetBody() &&
evaluated.GetDraft() == current.GetDraft() && slices.Equal(beforeLabels, afterLabels)
}
219 changes: 219 additions & 0 deletions src/pkg/github/automerge/base_preparation_identity_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,219 @@
package automerge

import (
"context"
"encoding/json"
"testing"
"time"

gh "github.com/google/go-github/v72/github"
"github.com/hivecommons/hive/pkg/effects"
)

func copyPreparationPR(t *testing.T, pr *gh.PullRequest) *gh.PullRequest {
t.Helper()
data, err := json.Marshal(pr)
if err != nil {
t.Fatal(err)
}
var out gh.PullRequest
if err := json.Unmarshal(data, &out); err != nil {
t.Fatal(err)
}
return &out
}

func preparationOwnerForTest(t *testing.T, c *Engine) preparationOwnerSnapshot {
t.Helper()
snapshot, err := c.preparationStore().ownerSnapshot(preparationKey("acme", "widget", "main"))
if err != nil {
t.Fatal(err)
}
return snapshot
}

func TestPreparationSettlementRejectsSupersededAdmissionIdentity(t *testing.T) {
for _, changed := range []string{"head", "base", "base-sha", "head-repo", "author", "body", "title", "labels", "draft", "unavailable"} {
t.Run(changed, func(t *testing.T) {
api, srv := newPreparationAPI(t)
c := preparationEngine(srv.URL, NewBasePreparationStore(""))
if reason, err := c.admitBasePreparation(context.Background(), "acme", "widget", "main", 1, "head1", stageLaneSelf, false); err != nil || reason != "" {
t.Fatalf("admission: %q %v", reason, err)
}
evaluated := copyPreparationPR(t, api.prs[1])
before := preparationOwnerForTest(t, c)
switch changed {
case "head":
api.prs[1].Head.SHA = gh.Ptr("new-head")
case "base":
api.prs[1].Base.Ref = gh.Ptr("other")
case "base-sha":
api.prs[1].Base.SHA = gh.Ptr("base2")
case "head-repo":
api.prs[1].Head.Repo.FullName = gh.Ptr("other/widget")
case "author":
api.prs[1].User.Login = gh.Ptr("other-author")
case "body":
api.prs[1].Body = gh.Ptr("Closes #200")
case "title":
api.prs[1].Title = gh.Ptr("new review input")
case "labels":
api.prs[1].Labels = []*gh.Label{{Name: gh.Ptr("role/trusted")}}
case "draft":
api.prs[1].Draft = gh.Ptr(true)
case "unavailable":
delete(api.prs, 1)
}
c.observeBasePreparation(stageLaneSelf, "acme", "widget", evaluated, "required-check-failing:build", false, nil)
if after := preparationOwnerForTest(t, c); after.Owner != before.Owner || after.Attempt != before.Attempt {
t.Fatalf("stale %s callback changed ownership: before=%+v after=%+v", changed, before, after)
}
})
}
}

func TestPreparationUnobservedRefreshCannotBeReleasedByOldEvaluation(t *testing.T) {
for _, reason := range []string{"required-check-failing:build", autoMergeReasonReviewGatePrefix + ReviewGateOutcomeChangesRequested, "untrusted-author-role"} {
t.Run(reason, func(t *testing.T) {
api, srv := newPreparationAPI(t)
c := preparationEngine(srv.URL, NewBasePreparationStore(""))
now := time.Date(2026, 11, 2, 18, 0, 0, 0, time.UTC)
c.now = func() time.Time { return now }
if _, _, _, err := preparationGate(c, 1, "main", "head1"); err != nil {
t.Fatal(err)
}
before := preparationOwnerForTest(t, c)
c.observeBasePreparation(stageLaneSelf, "acme", "widget", api.prs[1], reason, false, nil)
if after := preparationOwnerForTest(t, c); after.Owner != before.Owner || after.Attempt != before.Attempt {
t.Fatalf("unobserved mutation lost its exclusive owner: %+v", after)
}
now = now.Add(effects.DefaultUnresolvedTTL)
c.observeBasePreparation(stageLaneSelf, "acme", "widget", api.prs[1], reason, false, nil)
if after := preparationOwnerForTest(t, c); after.Owner.Number != 0 {
t.Fatalf("validated unchanged head could not settle after normal reconciliation window: %+v", after)
}
})
}
}

func TestPreparationObservedRefreshRetainsFreshReviewAndSettlesValidatedFailure(t *testing.T) {
api, srv := newPreparationAPI(t)
c := preparationEngine(srv.URL, NewBasePreparationStore(""))
if _, _, _, err := preparationGate(c, 1, "main", "head1"); err != nil {
t.Fatal(err)
}
api.prs[1].Head.SHA = gh.Ptr("observed-refresh")
for _, reason := range []string{"required-check-pending:build", autoMergeReasonReviewGatePrefix + ReviewGateOutcomeNoBundle, autoMergeReasonReviewGatePrefix + ReviewGateOutcomeAwaiting} {
c.observeBasePreparation(stageLaneSelf, "acme", "widget", api.prs[1], reason, false, nil)
if after := preparationOwnerForTest(t, c); after.Owner.Number != 1 || after.Owner.Head != "head1" || after.Attempt.Head != "head1" {
t.Fatalf("expected old source identity lost while %s: %+v", reason, after)
}
}
c.observeBasePreparation(stageLaneSelf, "acme", "widget", api.prs[1], "required-check-failing:build", false, nil)
if after := preparationOwnerForTest(t, c); after.Owner.Number != 0 {
t.Fatalf("fresh observed-head failure did not settle old source reservation: %+v", after)
}
if ev, _, _, err := preparationGate(c, 2, "main", "head2"); err != nil || !ev.BranchUpdated {
t.Fatalf("normally admitted successor could not refresh: %+v %v", ev, err)
}
}

func TestPreparationSettlementFencesOwnerAndAttemptChangesDuringRead(t *testing.T) {
for _, change := range []string{"owner", "attempt", "promoted-waiter"} {
t.Run(change, func(t *testing.T) {
api, srv := newPreparationAPI(t)
c := preparationEngine(srv.URL, NewBasePreparationStore(""))
key := preparationKey("acme", "widget", "main")
if reason, err := c.admitBasePreparation(context.Background(), "acme", "widget", "main", 1, "head1", stageLaneSelf, false); err != nil || reason != "" {
t.Fatalf("admission: %q %v", reason, err)
}
if change == "promoted-waiter" {
if err := c.preparationStore().change(key, func(st *basePreparation) error {
st.Waiting = []preparationCandidate{st.Owner}
st.Owner = preparationCandidate{Number: 2, Head: "head2", Lane: stageLaneSelf, Since: c.now()}
return nil
}); err != nil {
t.Fatal(err)
}
}
var concurrent preparationOwnerSnapshot
api.onGet = func(a *preparationAPI, _ int) {
a.onGet = nil
if err := c.preparationStore().change(key, func(st *basePreparation) error {
switch change {
case "owner":
st.Owner.Since = st.Owner.Since.Add(time.Second)
case "attempt":
st.Attempts = map[int]preparationAttempt{1: {Head: "head1", Base: "base2", At: c.now(), AuthoritativeBase: true}}
case "promoted-waiter":
st.Owner = st.Waiting[0]
st.Waiting = nil
}
return nil
}); err != nil {
t.Error(err)
}
concurrent = preparationOwnerForTest(t, c)
}
c.observeBasePreparation(stageLaneSelf, "acme", "widget", api.prs[1], "required-check-failing:build", false, nil)
if after := preparationOwnerForTest(t, c); after.Owner != concurrent.Owner || after.Attempt != concurrent.Attempt {
t.Fatalf("callback overwrote concurrent %s: concurrent=%+v after=%+v", change, concurrent, after)
}
})
}
}

func TestPreparationStaleCallbackCannotRewriteYieldedHead(t *testing.T) {
api, srv := newPreparationAPI(t)
c := preparationEngine(srv.URL, NewBasePreparationStore(""))
evaluated := copyPreparationPR(t, api.prs[1])
api.prs[1].Head.SHA = gh.Ptr("new-head")
key := preparationKey("acme", "widget", "main")
if err := c.preparationStore().change(key, func(st *basePreparation) error {
st.Owner = preparationCandidate{Number: 2, Head: "head2", Lane: stageLaneSelf, Since: c.now()}
st.Waiting = []preparationCandidate{{Number: 1, Head: "new-head", Lane: stageLaneSelf, Yielded: true, Since: c.now()}}
return nil
}); err != nil {
t.Fatal(err)
}
c.observeBasePreparation(stageLaneSelf, "acme", "widget", evaluated, "required-check-pending:build", false, nil)
if after := preparationOwnerForTest(t, c); after.Owner.Number != 2 || len(after.Waiters) != 1 || after.Waiters[0].Head != "new-head" {
t.Fatalf("stale callback rewound yielded head: %+v", after)
}
}

func TestPreparationValidatedYieldedRenewalDoesNotOverwriteNewerTicket(t *testing.T) {
for _, concurrent := range []bool{false, true} {
t.Run(map[bool]string{false: "validated", true: "superseded-during-read"}[concurrent], func(t *testing.T) {
api, srv := newPreparationAPI(t)
c := preparationEngine(srv.URL, NewBasePreparationStore(""))
key := preparationKey("acme", "widget", "main")
if err := c.preparationStore().change(key, func(st *basePreparation) error {
st.Owner = preparationCandidate{Number: 2, Head: "head2", Lane: stageLaneSelf, Since: c.now()}
st.Waiting = []preparationCandidate{{Number: 1, Head: "old-head", Lane: stageLaneSelf, Yielded: true, Since: c.now()}}
return nil
}); err != nil {
t.Fatal(err)
}
if concurrent {
api.onGet = func(a *preparationAPI, _ int) {
a.onGet = nil
if err := c.preparationStore().change(key, func(st *basePreparation) error {
st.Waiting[0].Head = "newer-ticket"
return nil
}); err != nil {
t.Error(err)
}
}
}
c.observeBasePreparation(stageLaneSelf, "acme", "widget", api.prs[1], "required-check-pending:build", false, nil)
want := "head1"
if concurrent {
want = "newer-ticket"
}
if after := preparationOwnerForTest(t, c); after.Owner.Number != 2 || len(after.Waiters) != 1 || after.Waiters[0].Head != want {
t.Fatalf("invalid yielded renewal: %+v, want head %s", after, want)
}
})
}
}
Loading