From c526d8b822d188bf6765408e82d1c79a44156385 Mon Sep 17 00:00:00 2001 From: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Mon, 9 Nov 2026 07:22:44 -0500 Subject: [PATCH 1/2] =?UTF-8?q?=F0=9F=90=9B=20fix(automerge):=20update-bra?= =?UTF-8?q?nch=20honours=20the=20stage=20record's=20fork=20guard=20when=20?= =?UTF-8?q?the=20index=20has=20no=20entry?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: Danathar <6772335+Danathar@users.noreply.github.com> --- .../fixed-fork-guard-update-branch-record.md | 1 + .../automerge/automerge_fork_guard_carry.go | 89 +++++++++++++++++-- .../automerge_fork_guard_carry_test.go | 65 ++++++++++++++ .../automerge/automerge_sweep_trusted.go | 2 +- 4 files changed, 147 insertions(+), 10 deletions(-) create mode 100644 changelog.d/fixed-fork-guard-update-branch-record.md diff --git a/changelog.d/fixed-fork-guard-update-branch-record.md b/changelog.d/fixed-fork-guard-update-branch-record.md new file mode 100644 index 0000000000..75e0bdccf5 --- /dev/null +++ b/changelog.d/fixed-fork-guard-update-branch-record.md @@ -0,0 +1 @@ +- The hive no longer merges `main` into a fork PR parked at 085 on a maintainer-run fork-guarded check (such as a fork Canary) after a restart or a rate-limited pass. The branch-update guard read only an in-memory index that has no entry for the head in those windows, so on projectbluefin/utah-packages#917 the App merged `main` five times in 22 hours and each merge discarded the run the PR was waiting for. When the index has nothing, the guard now falls back to the PR's stage record: it holds the update while a recorded guard check is red on the exact head (or while the head's checks cannot be read), and lets it through once every guarded check is green. diff --git a/src/pkg/github/automerge/automerge_fork_guard_carry.go b/src/pkg/github/automerge/automerge_fork_guard_carry.go index 541b6020d1..9ada5ff66c 100644 --- a/src/pkg/github/automerge/automerge_fork_guard_carry.go +++ b/src/pkg/github/automerge/automerge_fork_guard_carry.go @@ -1,8 +1,12 @@ package automerge import ( + "context" + "slices" "strings" + gh "github.com/google/go-github/v72/github" + hgithub "github.com/hivecommons/hive/pkg/github" "github.com/hivecommons/hive/pkg/stagelabels" ) @@ -70,15 +74,82 @@ func carriedForkGuard(rec stagelabels.Record, ok bool, obs stageObservation) []s return nil } -// waitingForkGuard is the fork guard a fork PR head waits on, from the index -// for this exact head: the check is red on it and only a maintainer's -// exact-head run turns it green. A branch update would discard that run, so -// the caller leaves the branch alone while this is non-empty; once the check -// is green the index drops it and updates resume before merge. The stage -// record is not consulted here: a guard it remembers may already be green, -// and skipping on it would keep the PR parked on a check that passed. -func (c *Engine) waitingForkGuard(owner, repo string, number int, head string) []string { - return stageForkGuard(c.stagePolicy(), owner, repo, number, head) +// waitingForkGuard is the fork guard a fork PR head waits on: the check is +// red on it and only a maintainer's exact-head run turns it green. A branch +// update would discard that run, so the caller leaves the branch alone while +// this is non-empty; once the check is green updates resume before merge. +// +// The index for this exact head answers first. It is process memory rebuilt +// from each CI-enriched enumeration, so it is empty after a restart and has +// no entry after a pass that could not read the check's annotations; on +// projectbluefin/utah-packages#917 each such gap let a merge of main through +// (five in 22 hours). When it has nothing, the stage record's fork guard +// (ForkGuardChecks, or a `fork-guard:` reason) is consulted. A remembered +// guard may already be green, so it counts only for the checks whose latest +// run on this head is red (failure or action_required); with every one green +// the update goes ahead. When the head's checks cannot be read the guard +// stands for this tick: skipping costs a tick, an update costs the +// maintainer's run. +func (c *Engine) waitingForkGuard(ctx context.Context, owner, repo string, number int, head string) []string { + if checks := stageForkGuard(c.stagePolicy(), owner, repo, number, head); len(checks) > 0 { + return checks + } + rec, ok := c.stageStoreOrShared().Get(stageKey(owner, repo, number)) + guard := rec.ForkGuardChecks + if ok && len(guard) == 0 { + guard = recordedForkGuard(rec, ok) + } + if !ok || len(guard) == 0 || strings.TrimSpace(head) == "" { + return nil + } + red, err := c.forkGuardRedOnHead(ctx, owner, repo, head, guard) + if err != nil { + c.warn("automerge: fork guard check state unreadable; holding the branch update this tick", "repo", owner+"/"+repo, "pr", number, "head", head, "checks", guard, "error", err) + return guard + } + return red +} + +// forkGuardRedOnHead is the guard checks whose latest run on head completed +// failure or action_required, in guard order. +func (c *Engine) forkGuardRedOnHead(ctx context.Context, owner, repo, head string, guard []string) ([]string, error) { + if c == nil || c.gh == nil { + return nil, hgithub.ErrNoGitHubClient + } + ctx = hgithub.WithRESTCaller(ctx, "hive:automerge_sweep") + latest := map[string]*gh.CheckRun{} + opts := &gh.ListCheckRunsOptions{ListOptions: gh.ListOptions{PerPage: 100}} + for { + runs, resp, err := c.gh.Checks.ListCheckRunsForRef(ctx, owner, repo, head, opts) + if err != nil { + return nil, err + } + if runs != nil { + for _, cr := range runs.CheckRuns { + if cr == nil || !slices.Contains(guard, cr.GetName()) { + continue + } + if prev := latest[cr.GetName()]; prev == nil || cr.GetID() > prev.GetID() { + latest[cr.GetName()] = cr + } + } + } + if resp == nil || resp.NextPage == 0 { + break + } + opts.Page = resp.NextPage + } + var red []string + for _, name := range guard { + cr := latest[name] + if cr == nil || cr.GetStatus() != "completed" { + continue + } + if conclusion := cr.GetConclusion(); conclusion == "failure" || conclusion == "action_required" { + red = append(red, name) + } + } + return red, nil } // forkGuardEntryStale reports a PR parked at 085 on a fork guard whose 085 diff --git a/src/pkg/github/automerge/automerge_fork_guard_carry_test.go b/src/pkg/github/automerge/automerge_fork_guard_carry_test.go index 99f0a0f885..31c8f200fe 100644 --- a/src/pkg/github/automerge/automerge_fork_guard_carry_test.go +++ b/src/pkg/github/automerge/automerge_fork_guard_carry_test.go @@ -145,6 +145,71 @@ func TestUpdateBranchSkipsForkGuardedHead(t *testing.T) { } } +// projectbluefin/utah-packages#917: five merges of main in 22 hours while +// the PR sat at 085 on `fork-guard:Canary`, each in the first sweep after a +// restart or a rate-limited pass left the index without an entry for the +// head. With the index empty, the stage record's guard holds the update +// while the guarded check is red on the head, lets it through once green, +// and holds it while the head's checks cannot be read. +func TestUpdateBranchHonoursRecordedForkGuard(t *testing.T) { + var updates []string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + switch { + case r.Method == http.MethodPut && strings.HasSuffix(r.URL.Path, "/update-branch"): + updates = append(updates, r.URL.Path) + w.WriteHeader(http.StatusAccepted) + _, _ = w.Write([]byte(`{"message":"Updating pull request branch."}`)) + case r.Method == http.MethodGet && strings.HasPrefix(r.URL.Path, "/repos/acme/widget/commits/") && strings.HasSuffix(r.URL.Path, "/check-runs"): + switch strings.TrimSuffix(strings.TrimPrefix(r.URL.Path, "/repos/acme/widget/commits/"), "/check-runs") { + case "red": + _, _ = w.Write([]byte(`{"total_count":2,"check_runs":[{"id":1,"name":"Canary","status":"completed","conclusion":"failure"},{"id":2,"name":"unit","status":"completed","conclusion":"success"}]}`)) + case "green": + _, _ = w.Write([]byte(`{"total_count":2,"check_runs":[{"id":1,"name":"Canary","status":"completed","conclusion":"failure"},{"id":3,"name":"Canary","status":"completed","conclusion":"success"}]}`)) + case "forbidden": + w.WriteHeader(http.StatusForbidden) + _, _ = w.Write([]byte(`{"message":"Resource not accessible by integration"}`)) + default: + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte(`{"message":"boom"}`)) + } + case r.Method == http.MethodGet && strings.HasPrefix(r.URL.Path, "/repos/acme/widget/pulls/"): + _, _ = w.Write([]byte(`{"number":917,"head":{"sha":"new"},"mergeable_state":"clean"}`)) + default: + w.WriteHeader(http.StatusNotFound) + } + })) + defer srv.Close() + clock := &contributorLaneClock{at: time.Date(2026, 11, 8, 6, 0, 0, 0, time.UTC)} + policy := StageLabelPolicy{Enabled: true, ForkGuard: func(string, int, string) []string { return nil }} + e := newNudgeEngine(t, srv.URL+"/", clock, nil, policy) + e.stageStoreOrShared().Observe(stageKey("acme", "widget", 917), stagelabels.NeedsDecision, stageReasonForkGuard+"Canary") + ctx := context.Background() + fork := currentBaseTestPR(true, "task/475-libfdk-aac", "kylerankin", "behind") + + for _, head := range []string{"red", "forbidden", "broken"} { + _, reason, err := e.updateBranchAndSkip(ctx, "acme/widget", "acme", "widget", 917, head, "base1", "behind", fork) + if err != nil || reason != "required-check-failing:Canary" || len(updates) != 0 { + t.Fatalf("head %s: reason=%q err=%v updates=%v, want the update held on Canary", head, reason, err, updates) + } + } + if _, _, err := e.updateBranchAndSkip(ctx, "acme/widget", "acme", "widget", 917, "green", "base1", "behind", fork); err != nil || len(updates) != 1 { + t.Fatalf("Canary green on the head: err=%v updates=%v, want one update", err, updates) + } + + // The record's ForkGuardChecks name the guard without a fork-guard reason. + e.stageStoreOrShared().Observe(stageKey("acme", "widget", 921), stagelabels.ReturnedToContributor, "required-check-failing:Canary") + e.stageStoreOrShared().NoteForkGuard(stageKey("acme", "widget", 921), []string{"Canary"}) + if _, reason, err := e.updateBranchAndSkip(ctx, "acme/widget", "acme", "widget", 921, "red", "base1", "behind", fork); err != nil || reason != "required-check-failing:Canary" || len(updates) != 1 { + t.Fatalf("record ForkGuardChecks: reason=%q err=%v updates=%v, want the update held", reason, err, updates) + } + + // A same-repo head is never held by the record. + if _, _, err := e.updateBranchAndSkip(ctx, "acme/widget", "acme", "widget", 917, "red", "base1", "behind", currentBaseTestPR(false, "feature", "alice", "behind")); err != nil || len(updates) != 2 { + t.Fatalf("same-repo head: err=%v updates=%v, want an update", err, updates) + } +} + // projectbluefin/utah-packages#917: the maintainer's fork Canary passed on // one head, the author rebased the same patch onto a newer base, and the // carried guard kept the PR at 085 without a stage change. The 085 comment diff --git a/src/pkg/github/automerge/automerge_sweep_trusted.go b/src/pkg/github/automerge/automerge_sweep_trusted.go index 7b92f9fde0..c069b3d036 100644 --- a/src/pkg/github/automerge/automerge_sweep_trusted.go +++ b/src/pkg/github/automerge/automerge_sweep_trusted.go @@ -916,7 +916,7 @@ func (c *Engine) updateBranchAndSkip(ctx context.Context, displayRepo, owner, re } fork := c.isForkPR(pr) if fork { - if checks := c.waitingForkGuard(owner, repo, number, headSHA); len(checks) > 0 { + if checks := c.waitingForkGuard(ctx, owner, repo, number, headSHA); len(checks) > 0 { // projectbluefin/utah-packages#862: 13 merge-of-main heads while // the PR waited on a maintainer's exact-head Canary run; each one // discarded the run it waited for. From 65c838778b790257f5b6f0a1c2b573a85a758fc4 Mon Sep 17 00:00:00 2001 From: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Mon, 9 Nov 2026 07:42:16 -0500 Subject: [PATCH 2/2] =?UTF-8?q?=F0=9F=90=9B=20fix(automerge):=20a=20runnin?= =?UTF-8?q?g=20fork=20guard=20run=20on=20the=20head=20holds=20the=20branch?= =?UTF-8?q?=20update?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit forkGuardRedOnHead only reported completed failure/action_required runs. A newer queued or in_progress run for the guard name won latest[] and was skipped, so the update merged main over the maintainer's live run. A non-completed latest guarded run is now held. Hive-Fix-Attempt: 1/6 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- .../automerge/automerge_fork_guard_carry.go | 32 +++++++++++-------- .../automerge_fork_guard_carry_test.go | 11 +++++-- 2 files changed, 27 insertions(+), 16 deletions(-) diff --git a/src/pkg/github/automerge/automerge_fork_guard_carry.go b/src/pkg/github/automerge/automerge_fork_guard_carry.go index 9ada5ff66c..d17c292440 100644 --- a/src/pkg/github/automerge/automerge_fork_guard_carry.go +++ b/src/pkg/github/automerge/automerge_fork_guard_carry.go @@ -86,10 +86,11 @@ func carriedForkGuard(rec stagelabels.Record, ok bool, obs stageObservation) []s // (five in 22 hours). When it has nothing, the stage record's fork guard // (ForkGuardChecks, or a `fork-guard:` reason) is consulted. A remembered // guard may already be green, so it counts only for the checks whose latest -// run on this head is red (failure or action_required); with every one green -// the update goes ahead. When the head's checks cannot be read the guard -// stands for this tick: skipping costs a tick, an update costs the -// maintainer's run. +// run on this head is red (failure or action_required) or still running +// (queued or in_progress: that is the maintainer's run, and an update would +// discard it); with every one green the update goes ahead. When the head's +// checks cannot be read the guard stands for this tick: skipping costs a +// tick, an update costs the maintainer's run. func (c *Engine) waitingForkGuard(ctx context.Context, owner, repo string, number int, head string) []string { if checks := stageForkGuard(c.stagePolicy(), owner, repo, number, head); len(checks) > 0 { return checks @@ -102,17 +103,18 @@ func (c *Engine) waitingForkGuard(ctx context.Context, owner, repo string, numbe if !ok || len(guard) == 0 || strings.TrimSpace(head) == "" { return nil } - red, err := c.forkGuardRedOnHead(ctx, owner, repo, head, guard) + held, err := c.forkGuardHeldOnHead(ctx, owner, repo, head, guard) if err != nil { c.warn("automerge: fork guard check state unreadable; holding the branch update this tick", "repo", owner+"/"+repo, "pr", number, "head", head, "checks", guard, "error", err) return guard } - return red + return held } -// forkGuardRedOnHead is the guard checks whose latest run on head completed -// failure or action_required, in guard order. -func (c *Engine) forkGuardRedOnHead(ctx context.Context, owner, repo, head string, guard []string) ([]string, error) { +// forkGuardHeldOnHead is the guard checks whose latest run on head has not +// completed, or completed failure or action_required, in guard order. A +// guard with no run on the head is not held. +func (c *Engine) forkGuardHeldOnHead(ctx context.Context, owner, repo, head string, guard []string) ([]string, error) { if c == nil || c.gh == nil { return nil, hgithub.ErrNoGitHubClient } @@ -139,17 +141,21 @@ func (c *Engine) forkGuardRedOnHead(ctx context.Context, owner, repo, head strin } opts.Page = resp.NextPage } - var red []string + var held []string for _, name := range guard { cr := latest[name] - if cr == nil || cr.GetStatus() != "completed" { + if cr == nil { + continue + } + if cr.GetStatus() != "completed" { + held = append(held, name) continue } if conclusion := cr.GetConclusion(); conclusion == "failure" || conclusion == "action_required" { - red = append(red, name) + held = append(held, name) } } - return red, nil + return held, nil } // forkGuardEntryStale reports a PR parked at 085 on a fork guard whose 085 diff --git a/src/pkg/github/automerge/automerge_fork_guard_carry_test.go b/src/pkg/github/automerge/automerge_fork_guard_carry_test.go index 31c8f200fe..e003c73370 100644 --- a/src/pkg/github/automerge/automerge_fork_guard_carry_test.go +++ b/src/pkg/github/automerge/automerge_fork_guard_carry_test.go @@ -149,8 +149,8 @@ func TestUpdateBranchSkipsForkGuardedHead(t *testing.T) { // the PR sat at 085 on `fork-guard:Canary`, each in the first sweep after a // restart or a rate-limited pass left the index without an entry for the // head. With the index empty, the stage record's guard holds the update -// while the guarded check is red on the head, lets it through once green, -// and holds it while the head's checks cannot be read. +// while the guarded check is red or still running on the head, lets it +// through once green, and holds it while the head's checks cannot be read. func TestUpdateBranchHonoursRecordedForkGuard(t *testing.T) { var updates []string srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -164,6 +164,11 @@ func TestUpdateBranchHonoursRecordedForkGuard(t *testing.T) { switch strings.TrimSuffix(strings.TrimPrefix(r.URL.Path, "/repos/acme/widget/commits/"), "/check-runs") { case "red": _, _ = w.Write([]byte(`{"total_count":2,"check_runs":[{"id":1,"name":"Canary","status":"completed","conclusion":"failure"},{"id":2,"name":"unit","status":"completed","conclusion":"success"}]}`)) + case "running": + // The maintainer's re-run is newer than the failure; an update would discard it. + _, _ = w.Write([]byte(`{"total_count":2,"check_runs":[{"id":1,"name":"Canary","status":"completed","conclusion":"failure"},{"id":4,"name":"Canary","status":"in_progress","conclusion":null}]}`)) + case "queued": + _, _ = w.Write([]byte(`{"total_count":1,"check_runs":[{"id":5,"name":"Canary","status":"queued","conclusion":null}]}`)) case "green": _, _ = w.Write([]byte(`{"total_count":2,"check_runs":[{"id":1,"name":"Canary","status":"completed","conclusion":"failure"},{"id":3,"name":"Canary","status":"completed","conclusion":"success"}]}`)) case "forbidden": @@ -187,7 +192,7 @@ func TestUpdateBranchHonoursRecordedForkGuard(t *testing.T) { ctx := context.Background() fork := currentBaseTestPR(true, "task/475-libfdk-aac", "kylerankin", "behind") - for _, head := range []string{"red", "forbidden", "broken"} { + for _, head := range []string{"red", "running", "queued", "forbidden", "broken"} { _, reason, err := e.updateBranchAndSkip(ctx, "acme/widget", "acme", "widget", 917, head, "base1", "behind", fork) if err != nil || reason != "required-check-failing:Canary" || len(updates) != 0 { t.Fatalf("head %s: reason=%q err=%v updates=%v, want the update held on Canary", head, reason, err, updates)