diff --git a/changelog.d/fixed-fork-guard-update-branch-record.md b/changelog.d/fixed-fork-guard-update-branch-record.md new file mode 100644 index 0000000000..75e0bdccf5 --- /dev/null +++ b/changelog.d/fixed-fork-guard-update-branch-record.md @@ -0,0 +1 @@ +- The hive no longer merges `main` into a fork PR parked at 085 on a maintainer-run fork-guarded check (such as a fork Canary) after a restart or a rate-limited pass. The branch-update guard read only an in-memory index that has no entry for the head in those windows, so on projectbluefin/utah-packages#917 the App merged `main` five times in 22 hours and each merge discarded the run the PR was waiting for. When the index has nothing, the guard now falls back to the PR's stage record: it holds the update while a recorded guard check is red on the exact head (or while the head's checks cannot be read), and lets it through once every guarded check is green. diff --git a/src/pkg/github/automerge/automerge_fork_guard_carry.go b/src/pkg/github/automerge/automerge_fork_guard_carry.go index 541b6020d1..d17c292440 100644 --- a/src/pkg/github/automerge/automerge_fork_guard_carry.go +++ b/src/pkg/github/automerge/automerge_fork_guard_carry.go @@ -1,8 +1,12 @@ package automerge import ( + "context" + "slices" "strings" + gh "github.com/google/go-github/v72/github" + hgithub "github.com/hivecommons/hive/pkg/github" "github.com/hivecommons/hive/pkg/stagelabels" ) @@ -70,15 +74,88 @@ func carriedForkGuard(rec stagelabels.Record, ok bool, obs stageObservation) []s return nil } -// waitingForkGuard is the fork guard a fork PR head waits on, from the index -// for this exact head: the check is red on it and only a maintainer's -// exact-head run turns it green. A branch update would discard that run, so -// the caller leaves the branch alone while this is non-empty; once the check -// is green the index drops it and updates resume before merge. The stage -// record is not consulted here: a guard it remembers may already be green, -// and skipping on it would keep the PR parked on a check that passed. -func (c *Engine) waitingForkGuard(owner, repo string, number int, head string) []string { - return stageForkGuard(c.stagePolicy(), owner, repo, number, head) +// waitingForkGuard is the fork guard a fork PR head waits on: the check is +// red on it and only a maintainer's exact-head run turns it green. A branch +// update would discard that run, so the caller leaves the branch alone while +// this is non-empty; once the check is green updates resume before merge. +// +// The index for this exact head answers first. It is process memory rebuilt +// from each CI-enriched enumeration, so it is empty after a restart and has +// no entry after a pass that could not read the check's annotations; on +// projectbluefin/utah-packages#917 each such gap let a merge of main through +// (five in 22 hours). When it has nothing, the stage record's fork guard +// (ForkGuardChecks, or a `fork-guard:` reason) is consulted. A remembered +// guard may already be green, so it counts only for the checks whose latest +// run on this head is red (failure or action_required) or still running +// (queued or in_progress: that is the maintainer's run, and an update would +// discard it); with every one green the update goes ahead. When the head's +// checks cannot be read the guard stands for this tick: skipping costs a +// tick, an update costs the maintainer's run. +func (c *Engine) waitingForkGuard(ctx context.Context, owner, repo string, number int, head string) []string { + if checks := stageForkGuard(c.stagePolicy(), owner, repo, number, head); len(checks) > 0 { + return checks + } + rec, ok := c.stageStoreOrShared().Get(stageKey(owner, repo, number)) + guard := rec.ForkGuardChecks + if ok && len(guard) == 0 { + guard = recordedForkGuard(rec, ok) + } + if !ok || len(guard) == 0 || strings.TrimSpace(head) == "" { + return nil + } + held, err := c.forkGuardHeldOnHead(ctx, owner, repo, head, guard) + if err != nil { + c.warn("automerge: fork guard check state unreadable; holding the branch update this tick", "repo", owner+"/"+repo, "pr", number, "head", head, "checks", guard, "error", err) + return guard + } + return held +} + +// forkGuardHeldOnHead is the guard checks whose latest run on head has not +// completed, or completed failure or action_required, in guard order. A +// guard with no run on the head is not held. +func (c *Engine) forkGuardHeldOnHead(ctx context.Context, owner, repo, head string, guard []string) ([]string, error) { + if c == nil || c.gh == nil { + return nil, hgithub.ErrNoGitHubClient + } + ctx = hgithub.WithRESTCaller(ctx, "hive:automerge_sweep") + latest := map[string]*gh.CheckRun{} + opts := &gh.ListCheckRunsOptions{ListOptions: gh.ListOptions{PerPage: 100}} + for { + runs, resp, err := c.gh.Checks.ListCheckRunsForRef(ctx, owner, repo, head, opts) + if err != nil { + return nil, err + } + if runs != nil { + for _, cr := range runs.CheckRuns { + if cr == nil || !slices.Contains(guard, cr.GetName()) { + continue + } + if prev := latest[cr.GetName()]; prev == nil || cr.GetID() > prev.GetID() { + latest[cr.GetName()] = cr + } + } + } + if resp == nil || resp.NextPage == 0 { + break + } + opts.Page = resp.NextPage + } + var held []string + for _, name := range guard { + cr := latest[name] + if cr == nil { + continue + } + if cr.GetStatus() != "completed" { + held = append(held, name) + continue + } + if conclusion := cr.GetConclusion(); conclusion == "failure" || conclusion == "action_required" { + held = append(held, name) + } + } + return held, nil } // forkGuardEntryStale reports a PR parked at 085 on a fork guard whose 085 diff --git a/src/pkg/github/automerge/automerge_fork_guard_carry_test.go b/src/pkg/github/automerge/automerge_fork_guard_carry_test.go index 99f0a0f885..e003c73370 100644 --- a/src/pkg/github/automerge/automerge_fork_guard_carry_test.go +++ b/src/pkg/github/automerge/automerge_fork_guard_carry_test.go @@ -145,6 +145,76 @@ func TestUpdateBranchSkipsForkGuardedHead(t *testing.T) { } } +// projectbluefin/utah-packages#917: five merges of main in 22 hours while +// the PR sat at 085 on `fork-guard:Canary`, each in the first sweep after a +// restart or a rate-limited pass left the index without an entry for the +// head. With the index empty, the stage record's guard holds the update +// while the guarded check is red or still running on the head, lets it +// through once green, and holds it while the head's checks cannot be read. +func TestUpdateBranchHonoursRecordedForkGuard(t *testing.T) { + var updates []string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + switch { + case r.Method == http.MethodPut && strings.HasSuffix(r.URL.Path, "/update-branch"): + updates = append(updates, r.URL.Path) + w.WriteHeader(http.StatusAccepted) + _, _ = w.Write([]byte(`{"message":"Updating pull request branch."}`)) + case r.Method == http.MethodGet && strings.HasPrefix(r.URL.Path, "/repos/acme/widget/commits/") && strings.HasSuffix(r.URL.Path, "/check-runs"): + switch strings.TrimSuffix(strings.TrimPrefix(r.URL.Path, "/repos/acme/widget/commits/"), "/check-runs") { + case "red": + _, _ = w.Write([]byte(`{"total_count":2,"check_runs":[{"id":1,"name":"Canary","status":"completed","conclusion":"failure"},{"id":2,"name":"unit","status":"completed","conclusion":"success"}]}`)) + case "running": + // The maintainer's re-run is newer than the failure; an update would discard it. + _, _ = w.Write([]byte(`{"total_count":2,"check_runs":[{"id":1,"name":"Canary","status":"completed","conclusion":"failure"},{"id":4,"name":"Canary","status":"in_progress","conclusion":null}]}`)) + case "queued": + _, _ = w.Write([]byte(`{"total_count":1,"check_runs":[{"id":5,"name":"Canary","status":"queued","conclusion":null}]}`)) + case "green": + _, _ = w.Write([]byte(`{"total_count":2,"check_runs":[{"id":1,"name":"Canary","status":"completed","conclusion":"failure"},{"id":3,"name":"Canary","status":"completed","conclusion":"success"}]}`)) + case "forbidden": + w.WriteHeader(http.StatusForbidden) + _, _ = w.Write([]byte(`{"message":"Resource not accessible by integration"}`)) + default: + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte(`{"message":"boom"}`)) + } + case r.Method == http.MethodGet && strings.HasPrefix(r.URL.Path, "/repos/acme/widget/pulls/"): + _, _ = w.Write([]byte(`{"number":917,"head":{"sha":"new"},"mergeable_state":"clean"}`)) + default: + w.WriteHeader(http.StatusNotFound) + } + })) + defer srv.Close() + clock := &contributorLaneClock{at: time.Date(2026, 11, 8, 6, 0, 0, 0, time.UTC)} + policy := StageLabelPolicy{Enabled: true, ForkGuard: func(string, int, string) []string { return nil }} + e := newNudgeEngine(t, srv.URL+"/", clock, nil, policy) + e.stageStoreOrShared().Observe(stageKey("acme", "widget", 917), stagelabels.NeedsDecision, stageReasonForkGuard+"Canary") + ctx := context.Background() + fork := currentBaseTestPR(true, "task/475-libfdk-aac", "kylerankin", "behind") + + for _, head := range []string{"red", "running", "queued", "forbidden", "broken"} { + _, reason, err := e.updateBranchAndSkip(ctx, "acme/widget", "acme", "widget", 917, head, "base1", "behind", fork) + if err != nil || reason != "required-check-failing:Canary" || len(updates) != 0 { + t.Fatalf("head %s: reason=%q err=%v updates=%v, want the update held on Canary", head, reason, err, updates) + } + } + if _, _, err := e.updateBranchAndSkip(ctx, "acme/widget", "acme", "widget", 917, "green", "base1", "behind", fork); err != nil || len(updates) != 1 { + t.Fatalf("Canary green on the head: err=%v updates=%v, want one update", err, updates) + } + + // The record's ForkGuardChecks name the guard without a fork-guard reason. + e.stageStoreOrShared().Observe(stageKey("acme", "widget", 921), stagelabels.ReturnedToContributor, "required-check-failing:Canary") + e.stageStoreOrShared().NoteForkGuard(stageKey("acme", "widget", 921), []string{"Canary"}) + if _, reason, err := e.updateBranchAndSkip(ctx, "acme/widget", "acme", "widget", 921, "red", "base1", "behind", fork); err != nil || reason != "required-check-failing:Canary" || len(updates) != 1 { + t.Fatalf("record ForkGuardChecks: reason=%q err=%v updates=%v, want the update held", reason, err, updates) + } + + // A same-repo head is never held by the record. + if _, _, err := e.updateBranchAndSkip(ctx, "acme/widget", "acme", "widget", 917, "red", "base1", "behind", currentBaseTestPR(false, "feature", "alice", "behind")); err != nil || len(updates) != 2 { + t.Fatalf("same-repo head: err=%v updates=%v, want an update", err, updates) + } +} + // projectbluefin/utah-packages#917: the maintainer's fork Canary passed on // one head, the author rebased the same patch onto a newer base, and the // carried guard kept the PR at 085 without a stage change. The 085 comment diff --git a/src/pkg/github/automerge/automerge_sweep_trusted.go b/src/pkg/github/automerge/automerge_sweep_trusted.go index 7b92f9fde0..c069b3d036 100644 --- a/src/pkg/github/automerge/automerge_sweep_trusted.go +++ b/src/pkg/github/automerge/automerge_sweep_trusted.go @@ -916,7 +916,7 @@ func (c *Engine) updateBranchAndSkip(ctx context.Context, displayRepo, owner, re } fork := c.isForkPR(pr) if fork { - if checks := c.waitingForkGuard(owner, repo, number, headSHA); len(checks) > 0 { + if checks := c.waitingForkGuard(ctx, owner, repo, number, headSHA); len(checks) > 0 { // projectbluefin/utah-packages#862: 13 merge-of-main heads while // the PR waited on a maintainer's exact-head Canary run; each one // discarded the run it waited for.