Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/fixed-17606-merge-watcher-ci-gate-reeval.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- Merge requests: a re-request for a PR head whose earlier merge request was exhausted on a CI-gate verdict now re-evaluates the CI gate once instead of being refused for 24 hours; it merges if the gate now passes (for example after a gate false positive was fixed or a check was re-run green) and is still refused if the gate blocks. Exhaustions for conflicts, permissions or merge-queue-required are refused as before, so fork PRs no longer need a new commit or a manual merge to recover from a stale CI verdict (hivecommons/hive#17606).
10 changes: 10 additions & 0 deletions src/docs/hive-merge.md
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,16 @@ on the same head SHA is refused within 24 hours: the request is moved to
`.exhausted` and its result file repeats the recorded reason instead of starting
another poll cycle. Pushing a new commit (a new head SHA) starts fresh.

One carve-out: when the recorded reason is a CI-gate verdict (`ci gate: …`,
including the never-created quarantine above), a re-request re-evaluates the
gate once on the same head before refusing. If the gate is now green — a
cancelled or flaky check re-run green, or a gate false positive fixed since —
the exhausted record is cleared and the request proceeds; otherwise it is
refused with the recorded reason plus `re-evaluated: <fresh verdict>`. Re-running
the failing check green and re-requesting is therefore a valid recovery, and
the only one available for a fork PR whose head the hive cannot move (#17606).
Non-CI reasons (branch protection, merge queue, scope) still refuse outright.

**Fork PR workflow approval remains manual.** When a cross-repo pull request's
workflow runs conclude `action_required`, the watcher refuses the merge with a
reason that says the fork PR workflow runs are awaiting maintainer approval.
Expand Down
41 changes: 25 additions & 16 deletions src/pkg/github/merge_ci_gate.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,15 @@ import (
gh "github.com/google/go-github/v72/github"
)

// mergeRequestCIGateReasonPrefix starts every reason the watcher's pre-merge
// CI gate (verifyMergeRequestCI) records; the exhausted-head re-evaluation
// keys off it.
const mergeRequestCIGateReasonPrefix = "ci gate: "

func ciGateReason(format string, args ...any) string {
return mergeRequestCIGateReasonPrefix + fmt.Sprintf(format, args...)
}

// mergeCIVerdict is the merge-request watcher's pre-merge CI verdict (#6173).
//
// Before this gate existed the watcher issued MergePR unconditionally and could
Expand Down Expand Up @@ -150,7 +159,7 @@ func (c *Client) verifyMergeRequestCI(ctx context.Context, repo string, number i
}
pr, _, err := c.client.PullRequests.Get(WithRESTCaller(ctx, "hive:merge_request_ci_gate"), owner, name, number)
if err != nil {
return mergeCIUnverified, "ci gate: fetching PR", fmt.Errorf("ci gate: fetching PR %s/%s#%d: %w", owner, name, number, err)
return mergeCIUnverified, ciGateReason("fetching PR"), fmt.Errorf(mergeRequestCIGateReasonPrefix+"fetching PR %s/%s#%d: %w", owner, name, number, err)
}
headSHA := pr.GetHead().GetSHA()
baseBranch := pr.GetBase().GetRef()
Expand All @@ -159,13 +168,13 @@ func (c *Client) verifyMergeRequestCI(ctx context.Context, repo string, number i
sha = headSHA
}
if sha == "" {
return mergeCIUnverified, "ci gate: PR has no head SHA", fmt.Errorf("ci gate: PR %s/%s#%d has no head SHA", owner, name, number)
return mergeCIUnverified, ciGateReason("PR has no head SHA"), fmt.Errorf(mergeRequestCIGateReasonPrefix+"PR %s/%s#%d has no head SHA", owner, name, number)
}
if headSHA != "" && !strings.EqualFold(sha, headSHA) {
// The commit the governor judged eligible is no longer the PR head.
// MergePR's pinned-SHA merge would 409 anyway; refusing here means the
// CI of the WRONG commit is never what authorizes a merge.
return mergeCIRed, fmt.Sprintf("ci gate: head moved: request pinned %s but PR head is %s", shortSHA(sha), shortSHA(headSHA)), nil
return mergeCIRed, ciGateReason("head moved: request pinned %s but PR head is %s", shortSHA(sha), shortSHA(headSHA)), nil
}

cfgSet, cfgKnown := c.configRequiredChecksForRepo(owner + "/" + name)
Expand Down Expand Up @@ -217,7 +226,7 @@ func (c *Client) verifyMergeRequestCI(ctx context.Context, repo string, number i
HeadFirstSeenAt: firstSeen,
})
if err != nil {
return mergeCIUnverified, "ci gate: " + st.Reason, fmt.Errorf("ci gate: %s for %s/%s@%s: %w", st.Reason, owner, name, shortSHA(sha), err)
return mergeCIUnverified, mergeRequestCIGateReasonPrefix + st.Reason, fmt.Errorf(mergeRequestCIGateReasonPrefix+"%s for %s/%s@%s: %w", st.Reason, owner, name, shortSHA(sha), err)
}
if cfgKnown && actualKnown {
c.warnRequiredChecksMismatch(owner, name, cfgSet, actualRequired, st.Observed)
Expand All @@ -242,25 +251,25 @@ func (c *Client) verifyMergeRequestCI(ctx context.Context, repo string, number i
// skipped) carry no verdict of their own here.
opaque, err := c.opaqueWorkflowRuns(ctx, owner, name, sha)
if err != nil {
return mergeCIUnverified, "ci gate: workflow-runs", fmt.Errorf("ci gate: listing workflow runs for %s/%s@%s: %w", owner, name, shortSHA(sha), err)
return mergeCIUnverified, ciGateReason("workflow-runs"), fmt.Errorf(mergeRequestCIGateReasonPrefix+"listing workflow runs for %s/%s@%s: %w", owner, name, shortSHA(sha), err)
}

switch {
case len(opaque.failed) > 0:
return mergeCIRed, fmt.Sprintf("ci gate: required status check has not succeeded: workflow run(s) %s concluded failure without producing a job (zero check runs)", strings.Join(opaque.failed, ", ")), nil
return mergeCIRed, ciGateReason("required status check has not succeeded: workflow run(s) %s concluded failure without producing a job (zero check runs)", strings.Join(opaque.failed, ", ")), nil
case !st.Green && !commitCIReasonIsPending(st.Reason):
return mergeCIRed, fmt.Sprintf("ci gate: required status check has not succeeded (%s)", st.Reason), nil
return mergeCIRed, ciGateReason("required status check has not succeeded (%s)", st.Reason), nil
case requiredCheckNeverCreated(st, opaque, headPushedAt, now):
// The reason deliberately avoids GitHub's protection vocabulary so
// isRequiredCheckMergeBlocker never routes it to the fix loop: no
// code push can create a check whose workflow cannot be triggered.
return mergeCINeverCreated, fmt.Sprintf("ci gate: required check %s was never created on %s and nothing on the head is queued or in progress after %s (the PR author's token likely cannot trigger workflows); a human must re-trigger CI or merge it", quotedNames(st.MissingRequired), shortSHA(sha), mergeCIMissingRequiredGrace), nil
return mergeCINeverCreated, ciGateReason("required check %s was never created on %s and nothing on the head is queued or in progress after %s (the PR author's token likely cannot trigger workflows); a human must re-trigger CI or merge it", quotedNames(st.MissingRequired), shortSHA(sha), mergeCIMissingRequiredGrace), nil
case !st.Green:
return mergeCIPending, fmt.Sprintf("ci gate: CI still running (%s)", st.Reason), nil
return mergeCIPending, ciGateReason("CI still running (%s)", st.Reason), nil
case len(st.MissingRequired) > 0:
return mergeCIPending, fmt.Sprintf("ci gate: required check(s) not yet reported on %s: %s", shortSHA(sha), strings.Join(st.MissingRequired, ", ")), nil
return mergeCIPending, ciGateReason("required check(s) not yet reported on %s: %s", shortSHA(sha), strings.Join(st.MissingRequired, ", ")), nil
case len(opaque.pending) > 0:
return mergeCIPending, fmt.Sprintf("ci gate: workflow run(s) %s still in flight without a job yet", strings.Join(opaque.pending, ", ")), nil
return mergeCIPending, ciGateReason("workflow run(s) %s still in flight without a job yet", strings.Join(opaque.pending, ", ")), nil
case len(opaque.actionRequired) > 0:
return mergeCIRed, forkRunApprovalReason(owner, name, opaque.actionRequired, pullRequestIsFromFork(pr)), nil
case st.Evidence == 0:
Expand All @@ -270,11 +279,11 @@ func (c *Client) verifyMergeRequestCI(ctx context.Context, repo string, number i
// opt-in downgrades this — and ONLY this — verdict; red and pending
// are never downgraded, and the default stays refuse.
if c.repoNoCIOK(owner, name) {
return mergeCIGreen, fmt.Sprintf("ci gate: no CI evidence on %s, permitted by explicit auto_merge.no_ci_ok opt-in for %s/%s", shortSHA(sha), owner, name), nil
return mergeCIGreen, ciGateReason("no CI evidence on %s, permitted by explicit auto_merge.no_ci_ok opt-in for %s/%s", shortSHA(sha), owner, name), nil
}
return mergeCIUnverified, fmt.Sprintf("ci gate: no commit statuses, check runs, or workflow runs found on %s - absent CI is not passing", shortSHA(sha)), nil
return mergeCIUnverified, ciGateReason("no commit statuses, check runs, or workflow runs found on %s - absent CI is not passing", shortSHA(sha)), nil
}
green := fmt.Sprintf("ci gate: %d status/check run(s) on %s, all gating checks succeeded", st.Evidence, shortSHA(sha))
green := ciGateReason("%d status/check run(s) on %s, all gating checks succeeded", st.Evidence, shortSHA(sha))
if len(opaque.heldButVerified) > 0 {
green += fmt.Sprintf("; held run(s) %s ignored: the same workflow(s) succeeded on this head under another event", strings.Join(opaque.heldButVerified, ", "))
}
Expand Down Expand Up @@ -507,9 +516,9 @@ func pullRequestIsFromFork(pr *gh.PullRequest) bool {

func forkRunApprovalReason(owner, repo string, labels []string, fromFork bool) string {
if fromFork {
return fmt.Sprintf("ci gate: fork PR workflow runs are awaiting maintainer approval (action_required): %s. Approve the runs or relax the repo setting at https://github.com/%s/%s/settings/actions (Approval for running fork pull request workflows)", strings.Join(labels, ", "), owner, repo)
return ciGateReason("fork PR workflow runs are awaiting maintainer approval (action_required): %s. Approve the runs or relax the repo setting at https://github.com/%s/%s/settings/actions (Approval for running fork pull request workflows)", strings.Join(labels, ", "), owner, repo)
}
return fmt.Sprintf("ci gate: same-repo PR workflow runs are awaiting maintainer approval (action_required): %s. The PR is not from a fork; GitHub holds runs for PRs opened by a bot token (GITHUB_TOKEN) under the fork-PR approval policy. Approve the runs, have the opener dispatch the required workflows on the branch, or relax the repo setting at https://github.com/%s/%s/settings/actions (Approval for running fork pull request workflows)", strings.Join(labels, ", "), owner, repo)
return ciGateReason("same-repo PR workflow runs are awaiting maintainer approval (action_required): %s. The PR is not from a fork; GitHub holds runs for PRs opened by a bot token (GITHUB_TOKEN) under the fork-PR approval policy. Approve the runs, have the opener dispatch the required workflows on the branch, or relax the repo setting at https://github.com/%s/%s/settings/actions (Approval for running fork pull request workflows)", strings.Join(labels, ", "), owner, repo)
}

// logCIVerdict records the gate's decision for the operator with the same
Expand Down
49 changes: 47 additions & 2 deletions src/pkg/github/merge_ci_gate_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -1162,8 +1162,9 @@ func TestMergeCIGate_RequiredCheckNeverCreated(t *testing.T) {
mustExist(t, reqPath+".exhausted", "never-created request quarantined on the first verdict")
mustNotExist(t, reqPath, "live request after quarantine")

// A re-request for the same head is refused with the recorded
// reason instead of being evaluated again.
// A re-request for the same head re-evaluates the gate once
// (#17606); the check is still never created, so it is refused
// with the recorded reason.
again, _ := WriteMergeRequest(t.TempDir(), MergeRequest{Repo: "o/r", Number: 42, ExpectSHA: "abc", Agent: "scanner"})
c.handleOneMergeRequest(context.Background(), again, fixedNow)
if resp := readMergeResult(t, again); !strings.Contains(resp.Error, "exhausted on the same head") || !strings.Contains(resp.Error, "was never created") {
Expand Down Expand Up @@ -1214,3 +1215,47 @@ func TestMergeCIGate_SkewedHeadWithoutExpectSHAHoldsFromFirstObservation(t *test
t.Fatalf("verifyMergeRequestCI must never merge, got %d", got)
}
}

func TestCIGateReason(t *testing.T) {
cases := []struct {
name, got, want string
}{
{"plain", ciGateReason("fetching PR"), "ci gate: fetching PR"},
{"formatted", ciGateReason("CI still running (%s)", "build"), "ci gate: CI still running (build)"},
}
for _, tc := range cases {
if tc.got != tc.want {
t.Errorf("%s: got %q, want %q", tc.name, tc.got, tc.want)
}
}
}

// Every reason verifyMergeRequestCI returns must carry the prefix the
// exhausted-head re-evaluation keys off.
func TestMergeCIGate_ReasonsCarryPrefix(t *testing.T) {
cases := []struct {
name string
f *ciFixture
sha string
}{
{"green", greenFixture(), "abc"},
{"no CI evidence", &ciFixture{defaultHead: "abc"}, "abc"},
{"head moved", greenFixture(), "def"},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
tc.f.defaultHead = "abc"
var merges atomic.Int32
srv := ciGateServer(t, tc.f, &merges)
defer srv.Close()
c := testMergeClient(t, srv.URL)
_, why, err := c.verifyMergeRequestCI(context.Background(), "o/r", 42, tc.sha, fixedNow(), time.Time{})
if err != nil {
why = err.Error()
}
if !strings.HasPrefix(why, mergeRequestCIGateReasonPrefix) {
t.Fatalf("reason %q must start with %q", why, mergeRequestCIGateReasonPrefix)
}
})
}
}
4 changes: 3 additions & 1 deletion src/pkg/github/merge_request_state.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,9 @@ const mergeRequestMaxCIWait = time.Hour
// makes the watcher refuse new requests for the same PR+SHA. Long enough that
// a scanner re-dropping the request every kick cannot restart the poll cycle,
// short enough that a check re-run that later turns the same SHA green is
// eventually picked up again without an operator.
// eventually picked up again without an operator. A head exhausted on a
// CI-gate reason (mergeRequestCIGateReasonPrefix) is not refused outright: a
// re-request re-evaluates the gate once and refuses only if it still blocks.
const mergeRequestExhaustedRefusal = 24 * time.Hour

// mergeRequestStateRetention drops ledger entries untouched for this long so
Expand Down
89 changes: 87 additions & 2 deletions src/pkg/github/merge_request_state_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -147,14 +147,17 @@ func TestMergeRequestWatcher_StateSurvivesRestart(t *testing.T) {
before := prGets.Load()
reqPath, _ := WriteMergeRequest(t.TempDir(), MergeRequest{Repo: "o/r", Number: 42, ExpectSHA: "abc", Agent: "scanner"})
c.handleOneMergeRequest(context.Background(), reqPath, fixedNow)
if prGets.Load() != before {
t.Fatalf("refusing an exhausted head must not call GitHub")
if got := prGets.Load() - before; got != 1 {
t.Fatalf("an exhausted CI-gate head must be re-evaluated exactly once, got %d PR GETs", got)
}
resp := readMergeResult(t, reqPath)
if resp.OK || !strings.Contains(resp.Error, "exhausted on the same head") || !strings.Contains(resp.Error, "absent CI is not passing") {
t.Fatalf("expected a refusal carrying the recorded reason, got %+v", resp)
}
mustExist(t, reqPath+".exhausted", "refused re-request")
if st, ok := c.mergeRequestState("o/r", 42, "abc"); !ok || !st.Exhausted {
t.Fatalf("a gate that still blocks must keep the exhausted record, got %+v", st)
}

// After the refusal window the same head is evaluated again.
later := func() time.Time { return fixedNow().Add(mergeRequestExhaustedRefusal + time.Minute) }
Expand Down Expand Up @@ -198,6 +201,69 @@ func TestMergeRequestWatcher_StateSurvivesRestart(t *testing.T) {
}
}

// A head exhausted on a CI-gate verdict is re-evaluated on re-request: a gate
// that now passes merges (the stale check-cancelled false positive of #17606
// must not outlive its fix), one that still blocks refuses and keeps the
// record, and a non-CI reason still refuses without calling GitHub.
func TestMergeRequestWatcher_ExhaustedCIGateReevaluated(t *testing.T) {
cases := []struct {
name string
reason string
red bool
wantMerge bool
}{
{name: "ci gate now green merges", reason: "ci gate: required status check has not succeeded (check-cancelled)", wantMerge: true},
{name: "ci gate still red refuses", reason: "ci gate: required status check has not succeeded (check-cancelled)", red: true},
{name: "non-CI reason refuses", reason: "409 Merge conflict"},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
f := greenFixture()
f.defaultHead = "abc"
if tc.red {
f.checks = []ciCheck{{name: "build", status: "completed", conclusion: "failure"}}
}
var merges, prGets atomic.Int32
srv := ciGateServer(t, f, &merges)
defer srv.Close()
counted := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/pulls/42") {
prGets.Add(1)
}
srv.Config.Handler.ServeHTTP(w, r)
}))
defer counted.Close()
c := testMergeClient(t, counted.URL)
c.SetMergeRequestStatePath(filepath.Join(t.TempDir(), "merge-request-state.json"))
c.putMergeRequestState("o/r", 42, mergeRequestPRState{SHA: "abc", Attempts: mergeRequestMaxAttempts, Exhausted: true, Reason: tc.reason, At: fixedNow()})

reqPath, _ := WriteMergeRequest(t.TempDir(), MergeRequest{Repo: "o/r", Number: 42, ExpectSHA: "abc", Agent: "scanner"})
c.handleOneMergeRequest(context.Background(), reqPath, fixedNow)
resp := readMergeResult(t, reqPath)
st, tracked := c.mergeRequestState("o/r", 42, "abc")
if tc.wantMerge {
if merges.Load() != 1 || !resp.OK {
t.Fatalf("a CI-gate exhaustion whose gate now passes must merge, got %d merges (%+v)", merges.Load(), resp)
}
if tracked {
t.Fatalf("merged PR must not keep the exhausted record, got %+v", st)
}
return
}
if merges.Load() != 0 || resp.OK || !strings.Contains(resp.Error, "exhausted on the same head") || !strings.Contains(resp.Error, tc.reason) {
t.Fatalf("expected a refusal carrying the recorded reason, got %d merges (%+v)", merges.Load(), resp)
}
mustExist(t, reqPath+".exhausted", "refused re-request")
if !tracked || !st.Exhausted || st.Reason != tc.reason {
t.Fatalf("a refusal must keep the exhausted record, got %+v", st)
}
if !tc.red && prGets.Load() != 0 {
t.Fatalf("a non-CI exhaustion must be refused without calling GitHub, got %d PR GETs", prGets.Load())
}
})
}
}

// A merged PR drops its ledger entry so the file does not accumulate.
func TestMergeRequestWatcher_MergeClearsState(t *testing.T) {
f := greenFixture()
Expand All @@ -218,3 +284,22 @@ func TestMergeRequestWatcher_MergeClearsState(t *testing.T) {
t.Fatalf("merged PR must not keep wait state")
}
}

func TestRefuseExhaustedMergeRequest_HintMatchesReason(t *testing.T) {
cases := []struct {
reason, want, notWant string
}{
{"ci gate: required status check has not succeeded", "re-run the failing check", ""},
{"409 Merge conflict", "push a new commit or merge it manually", "re-run"},
}
for _, tc := range cases {
c := testMergeClient(t, "http://unused")
reqPath, _ := WriteMergeRequest(t.TempDir(), MergeRequest{Repo: "o/r", Number: 42, ExpectSHA: "abc", Agent: "scanner"})
c.refuseExhaustedMergeRequest(reqPath, MergeRequest{Repo: "o/r", Number: 42, ExpectSHA: "abc", Agent: "scanner"},
mergeRequestPRState{SHA: "abc", Exhausted: true, Reason: tc.reason, At: fixedNow()}, fixedNow)
resp := readMergeResult(t, reqPath)
if !strings.Contains(resp.Error, tc.want) || (tc.notWant != "" && strings.Contains(resp.Error, tc.notWant)) {
t.Errorf("reason %q: unexpected hint in %q", tc.reason, resp.Error)
}
}
}
Loading
Loading