Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/fixed-fork-guard-update-branch-record.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- The hive no longer merges `main` into a fork PR parked at 085 on a maintainer-run fork-guarded check (such as a fork Canary) after a restart or a rate-limited pass. The branch-update guard read only an in-memory index that has no entry for the head in those windows, so on projectbluefin/utah-packages#917 the App merged `main` five times in 22 hours and each merge discarded the run the PR was waiting for. When the index has nothing, the guard now falls back to the PR's stage record: it holds the update while a recorded guard check is red on the exact head (or while the head's checks cannot be read), and lets it through once every guarded check is green.
95 changes: 86 additions & 9 deletions src/pkg/github/automerge/automerge_fork_guard_carry.go
Original file line number Diff line number Diff line change
@@ -1,8 +1,12 @@
package automerge

import (
"context"
"slices"
"strings"

gh "github.com/google/go-github/v72/github"
hgithub "github.com/hivecommons/hive/pkg/github"
"github.com/hivecommons/hive/pkg/stagelabels"
)

Expand Down Expand Up @@ -70,15 +74,88 @@ func carriedForkGuard(rec stagelabels.Record, ok bool, obs stageObservation) []s
return nil
}

// waitingForkGuard is the fork guard a fork PR head waits on, from the index
// for this exact head: the check is red on it and only a maintainer's
// exact-head run turns it green. A branch update would discard that run, so
// the caller leaves the branch alone while this is non-empty; once the check
// is green the index drops it and updates resume before merge. The stage
// record is not consulted here: a guard it remembers may already be green,
// and skipping on it would keep the PR parked on a check that passed.
func (c *Engine) waitingForkGuard(owner, repo string, number int, head string) []string {
return stageForkGuard(c.stagePolicy(), owner, repo, number, head)
// waitingForkGuard is the fork guard a fork PR head waits on: the check is
// red on it and only a maintainer's exact-head run turns it green. A branch
// update would discard that run, so the caller leaves the branch alone while
// this is non-empty; once the check is green updates resume before merge.
//
// The index for this exact head answers first. It is process memory rebuilt
// from each CI-enriched enumeration, so it is empty after a restart and has
// no entry after a pass that could not read the check's annotations; on
// projectbluefin/utah-packages#917 each such gap let a merge of main through
// (five in 22 hours). When it has nothing, the stage record's fork guard
// (ForkGuardChecks, or a `fork-guard:` reason) is consulted. A remembered
// guard may already be green, so it counts only for the checks whose latest
// run on this head is red (failure or action_required) or still running
// (queued or in_progress: that is the maintainer's run, and an update would
// discard it); with every one green the update goes ahead. When the head's
// checks cannot be read the guard stands for this tick: skipping costs a
// tick, an update costs the maintainer's run.
func (c *Engine) waitingForkGuard(ctx context.Context, owner, repo string, number int, head string) []string {
if checks := stageForkGuard(c.stagePolicy(), owner, repo, number, head); len(checks) > 0 {
return checks
}
rec, ok := c.stageStoreOrShared().Get(stageKey(owner, repo, number))
guard := rec.ForkGuardChecks
if ok && len(guard) == 0 {
guard = recordedForkGuard(rec, ok)
}
if !ok || len(guard) == 0 || strings.TrimSpace(head) == "" {
return nil
}
held, err := c.forkGuardHeldOnHead(ctx, owner, repo, head, guard)
if err != nil {
c.warn("automerge: fork guard check state unreadable; holding the branch update this tick", "repo", owner+"/"+repo, "pr", number, "head", head, "checks", guard, "error", err)
return guard
}
return held
}

// forkGuardHeldOnHead is the guard checks whose latest run on head has not
// completed, or completed failure or action_required, in guard order. A
// guard with no run on the head is not held.
func (c *Engine) forkGuardHeldOnHead(ctx context.Context, owner, repo, head string, guard []string) ([]string, error) {
if c == nil || c.gh == nil {
return nil, hgithub.ErrNoGitHubClient
}
ctx = hgithub.WithRESTCaller(ctx, "hive:automerge_sweep")
latest := map[string]*gh.CheckRun{}
opts := &gh.ListCheckRunsOptions{ListOptions: gh.ListOptions{PerPage: 100}}
for {
runs, resp, err := c.gh.Checks.ListCheckRunsForRef(ctx, owner, repo, head, opts)
if err != nil {
return nil, err
}
if runs != nil {
for _, cr := range runs.CheckRuns {
if cr == nil || !slices.Contains(guard, cr.GetName()) {
continue
}
if prev := latest[cr.GetName()]; prev == nil || cr.GetID() > prev.GetID() {
latest[cr.GetName()] = cr
}
}
}
if resp == nil || resp.NextPage == 0 {
break
}
opts.Page = resp.NextPage
}
var held []string
for _, name := range guard {
cr := latest[name]
if cr == nil {
continue
}
if cr.GetStatus() != "completed" {
held = append(held, name)
continue
}
if conclusion := cr.GetConclusion(); conclusion == "failure" || conclusion == "action_required" {
held = append(held, name)
}
}
return held, nil
}

// forkGuardEntryStale reports a PR parked at 085 on a fork guard whose 085
Expand Down
70 changes: 70 additions & 0 deletions src/pkg/github/automerge/automerge_fork_guard_carry_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -145,6 +145,76 @@ func TestUpdateBranchSkipsForkGuardedHead(t *testing.T) {
}
}

// projectbluefin/utah-packages#917: five merges of main in 22 hours while
// the PR sat at 085 on `fork-guard:Canary`, each in the first sweep after a
// restart or a rate-limited pass left the index without an entry for the
// head. With the index empty, the stage record's guard holds the update
// while the guarded check is red or still running on the head, lets it
// through once green, and holds it while the head's checks cannot be read.
func TestUpdateBranchHonoursRecordedForkGuard(t *testing.T) {
var updates []string
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
switch {
case r.Method == http.MethodPut && strings.HasSuffix(r.URL.Path, "/update-branch"):
updates = append(updates, r.URL.Path)
w.WriteHeader(http.StatusAccepted)
_, _ = w.Write([]byte(`{"message":"Updating pull request branch."}`))
case r.Method == http.MethodGet && strings.HasPrefix(r.URL.Path, "/repos/acme/widget/commits/") && strings.HasSuffix(r.URL.Path, "/check-runs"):
switch strings.TrimSuffix(strings.TrimPrefix(r.URL.Path, "/repos/acme/widget/commits/"), "/check-runs") {
case "red":
_, _ = w.Write([]byte(`{"total_count":2,"check_runs":[{"id":1,"name":"Canary","status":"completed","conclusion":"failure"},{"id":2,"name":"unit","status":"completed","conclusion":"success"}]}`))
case "running":
// The maintainer's re-run is newer than the failure; an update would discard it.
_, _ = w.Write([]byte(`{"total_count":2,"check_runs":[{"id":1,"name":"Canary","status":"completed","conclusion":"failure"},{"id":4,"name":"Canary","status":"in_progress","conclusion":null}]}`))
case "queued":
_, _ = w.Write([]byte(`{"total_count":1,"check_runs":[{"id":5,"name":"Canary","status":"queued","conclusion":null}]}`))
case "green":
_, _ = w.Write([]byte(`{"total_count":2,"check_runs":[{"id":1,"name":"Canary","status":"completed","conclusion":"failure"},{"id":3,"name":"Canary","status":"completed","conclusion":"success"}]}`))
case "forbidden":
w.WriteHeader(http.StatusForbidden)
_, _ = w.Write([]byte(`{"message":"Resource not accessible by integration"}`))
default:
w.WriteHeader(http.StatusInternalServerError)
_, _ = w.Write([]byte(`{"message":"boom"}`))
}
case r.Method == http.MethodGet && strings.HasPrefix(r.URL.Path, "/repos/acme/widget/pulls/"):
_, _ = w.Write([]byte(`{"number":917,"head":{"sha":"new"},"mergeable_state":"clean"}`))
default:
w.WriteHeader(http.StatusNotFound)
}
}))
defer srv.Close()
clock := &contributorLaneClock{at: time.Date(2026, 11, 8, 6, 0, 0, 0, time.UTC)}
policy := StageLabelPolicy{Enabled: true, ForkGuard: func(string, int, string) []string { return nil }}
e := newNudgeEngine(t, srv.URL+"/", clock, nil, policy)
e.stageStoreOrShared().Observe(stageKey("acme", "widget", 917), stagelabels.NeedsDecision, stageReasonForkGuard+"Canary")
ctx := context.Background()
fork := currentBaseTestPR(true, "task/475-libfdk-aac", "kylerankin", "behind")

for _, head := range []string{"red", "running", "queued", "forbidden", "broken"} {
_, reason, err := e.updateBranchAndSkip(ctx, "acme/widget", "acme", "widget", 917, head, "base1", "behind", fork)
if err != nil || reason != "required-check-failing:Canary" || len(updates) != 0 {
t.Fatalf("head %s: reason=%q err=%v updates=%v, want the update held on Canary", head, reason, err, updates)
}
}
if _, _, err := e.updateBranchAndSkip(ctx, "acme/widget", "acme", "widget", 917, "green", "base1", "behind", fork); err != nil || len(updates) != 1 {
t.Fatalf("Canary green on the head: err=%v updates=%v, want one update", err, updates)
}

// The record's ForkGuardChecks name the guard without a fork-guard reason.
e.stageStoreOrShared().Observe(stageKey("acme", "widget", 921), stagelabels.ReturnedToContributor, "required-check-failing:Canary")
e.stageStoreOrShared().NoteForkGuard(stageKey("acme", "widget", 921), []string{"Canary"})
if _, reason, err := e.updateBranchAndSkip(ctx, "acme/widget", "acme", "widget", 921, "red", "base1", "behind", fork); err != nil || reason != "required-check-failing:Canary" || len(updates) != 1 {
t.Fatalf("record ForkGuardChecks: reason=%q err=%v updates=%v, want the update held", reason, err, updates)
}

// A same-repo head is never held by the record.
if _, _, err := e.updateBranchAndSkip(ctx, "acme/widget", "acme", "widget", 917, "red", "base1", "behind", currentBaseTestPR(false, "feature", "alice", "behind")); err != nil || len(updates) != 2 {
t.Fatalf("same-repo head: err=%v updates=%v, want an update", err, updates)
}
}

// projectbluefin/utah-packages#917: the maintainer's fork Canary passed on
// one head, the author rebased the same patch onto a newer base, and the
// carried guard kept the PR at 085 without a stage change. The 085 comment
Expand Down
2 changes: 1 addition & 1 deletion src/pkg/github/automerge/automerge_sweep_trusted.go
Original file line number Diff line number Diff line change
Expand Up @@ -916,7 +916,7 @@ func (c *Engine) updateBranchAndSkip(ctx context.Context, displayRepo, owner, re
}
fork := c.isForkPR(pr)
if fork {
if checks := c.waitingForkGuard(owner, repo, number, headSHA); len(checks) > 0 {
if checks := c.waitingForkGuard(ctx, owner, repo, number, headSHA); len(checks) > 0 {
// projectbluefin/utah-packages#862: 13 merge-of-main heads while
// the PR waited on a maintainer's exact-head Canary run; each one
// discarded the run it waited for.
Expand Down
Loading