Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/fixed-16229-merge-queue-405.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- The merge-request watcher now treats GitHub's "Changes must be made through the merge queue" 405 as terminal: it stops retrying the direct merge and logs one classified message per PR head instead of repeated warnings. (#16229)
13 changes: 13 additions & 0 deletions src/docs/hive-merge.md
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,19 @@ verdict:

An API error while gathering that evidence is a failed attempt, never a pass.

**A merge-queue-required refusal is terminal on the first attempt.** When the
merge call itself (`PUT /pulls/{n}/merge`) answers `405 Changes must be made
through the merge queue`, the base branch's ruleset forbids direct merges and no
retry, code push or re-engaged fix loop can change that. The request is
quarantined (`.exhausted`) for that head immediately, without consuming the
retry budget, raising an operator alert or re-engaging the fix loop; the result
file records `repo requires merge queue; direct merge not possible — enqueue via
merge queue or merge by hand` followed by GitHub's message, and the watcher logs
it once per head at INFO. Clear it by enqueuing the PR through the merge queue
(see `auto_merge.native_merge_queue.enabled` in the
[operator reference](operator-reference/README.md)) or merging by hand; a new
head SHA starts fresh like any other exhausted request.

**Wait state survives restarts and re-requests.** The attempt count, `ci_waits`,
the pending-poll backoff and an "exhausted" record are kept per PR and head SHA
in `/data/merge-request-state.json` (the PVC), not only in the tmpfs result
Expand Down
40 changes: 40 additions & 0 deletions src/pkg/github/merge_request_watcher.go
Original file line number Diff line number Diff line change
Expand Up @@ -225,6 +225,20 @@ func isRequiredCheckMergeBlocker(errMsg string) bool {
return false
}

// mergeQueueRequiredReason is the classified reason recorded when the base
// branch's ruleset only accepts merges through GitHub's merge queue.
const mergeQueueRequiredReason = "repo requires merge queue; direct merge not possible — enqueue via merge queue or merge by hand"

// isMergeQueueRequiredMergeBlocker reports the 405 GitHub returns for a direct
// REST merge on a base branch whose ruleset requires a merge queue
// ("Repository rule violations found ... Changes must be made through the
// merge queue"). No retry of PUT /merge can succeed there
// (hivecommons/hive#16229); the automerge sweep classifies the same error as
// a skip reason (hivecommons/hive#14018).
func isMergeQueueRequiredMergeBlocker(errMsg string) bool {
return strings.Contains(strings.ToLower(errMsg), "must be made through the merge queue")
}

func isConflictMergeBlocker(errMsg string) bool {
m := strings.ToLower(errMsg)
return strings.Contains(m, "merge conflict") ||
Expand Down Expand Up @@ -516,6 +530,10 @@ func (c *Client) handleOneMergeRequest(ctx context.Context, path string, nowFn f

res, err := c.MergePR(ctx, req.Repo, req.Number, req.Method, req.ExpectSHA)
if err != nil {
if isMergeQueueRequiredMergeBlocker(err.Error()) {
c.quarantineMergeQueueRequired(path, req, attempts, err.Error(), nowFn)
return
}
c.recordMergeFailure(path, req, attempts, err.Error(), nowFn)
return
}
Expand Down Expand Up @@ -596,6 +614,28 @@ func (c *Client) quarantineMergeRequest(path string, req MergeRequest, attempts
slog.String("sha", req.ExpectSHA), slog.String("reason", reason))
}

// quarantineMergeQueueRequired sets a request aside on the first
// merge-queue-required refusal, without spending the retry budget. Like
// quarantineMergeRequest it records the exhaustion against the PR+SHA, so a
// re-request for the same head is refused quietly and the classified message
// is logged once per head. It logs at INFO: the blocker is the target repo's
// configuration, not a hive defect, and a WARN per head would surface as an
// unclassified_error_repeated selfreport. It deliberately does not call
// raiseMergeFailureAlert: classifyMergeFailureForOperator has no merge-queue
// case, so no alert was ever raised for this error; add one there before
// routing this path through the alert if that changes.
func (c *Client) quarantineMergeQueueRequired(path string, req MergeRequest, attempts int, errMsg string, nowFn func() time.Time) {
now := nowFn()
reason := mergeQueueRequiredReason + ": " + errMsg
c.writeMergeResult(path, MergeResponse{Number: req.Number, Attempts: attempts, OK: false, Error: reason, At: now.UTC().Format(time.RFC3339)})
c.putMergeRequestState(req.Repo, req.Number, mergeRequestPRState{SHA: req.ExpectSHA, Attempts: attempts, Exhausted: true, Reason: reason, At: now})
_ = os.Rename(path, path+".exhausted")
c.logger.Info("merge-request watcher: "+mergeQueueRequiredReason,
slog.String("repo", req.Repo), slog.Int("number", req.Number),
slog.String("sha", req.ExpectSHA), slog.Int("attempts", attempts),
slog.String("error", errMsg))
}

// refuseExhaustedMergeRequest answers a new request for a PR+SHA whose earlier
// request was already exhausted with the recorded reason, instead of starting
// another wait/retry cycle on a head that has not changed.
Expand Down
78 changes: 78 additions & 0 deletions src/pkg/github/merge_request_watcher_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -483,3 +483,81 @@ func TestMergeRequestWatcher_RoutesNonGitHubRepoToForge(t *testing.T) {
t.Errorf("human_merge_paths repo should be denied: %v", err)
}
}

// A 405 "Changes must be made through the merge queue" can never be cleared by
// retrying PUT /merge (hivecommons/hive#16229): the request is set aside on the
// first refusal with the classified reason, a re-request for the same head is
// refused without another PUT, and nothing is logged at WARN.
func TestMergeRequestWatcher_MergeQueueRequiredIsTerminal(t *testing.T) {
var puts atomic.Int32
green := greenFixture()
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if green.serveCI(w, r) {
return
}
if r.Method == "PUT" && strings.HasSuffix(r.URL.Path, "/merge") {
puts.Add(1)
w.WriteHeader(http.StatusMethodNotAllowed)
_, _ = io.WriteString(w, `{"message":"Repository rule violations found\n\nChanges must be made through the merge queue\n\n"}`)
return
}
w.WriteHeader(http.StatusNotFound)
}))
defer srv.Close()
c := testMergeClient(t, srv.URL)
var logs strings.Builder
c.logger = slog.New(slog.NewTextHandler(&logs, &slog.HandlerOptions{Level: slog.LevelDebug}))
var reEngaged int
c.SetMergeReEngageHook(func(string, int) bool { reEngaged++; return true })

dir := t.TempDir()
mergeRequestDirForTest = dir
defer func() { mergeRequestDirForTest = "" }()

reqPath, _ := WriteMergeRequest(dir, MergeRequest{Repo: "o/r", Number: 74, ExpectSHA: "abc", Agent: "scanner"})
c.ProcessMergeRequestsOnce(context.Background())

if got := puts.Load(); got != 1 {
t.Fatalf("PUT /merge calls = %d, want 1", got)
}
if _, err := os.Stat(reqPath + ".exhausted"); err != nil {
t.Fatalf("merge-queue-required refusal should quarantine on the first attempt: %v", err)
}
resp := readMergeResult(t, reqPath)
if resp.OK || resp.Attempts != 1 || !strings.Contains(resp.Error, mergeQueueRequiredReason) {
t.Fatalf("result = %+v, want a failed attempt 1 carrying %q", resp, mergeQueueRequiredReason)
}

again, _ := WriteMergeRequest(dir, MergeRequest{Repo: "o/r", Number: 74, ExpectSHA: "abc", Agent: "scanner"})
c.ProcessMergeRequestsOnce(context.Background())
if got := puts.Load(); got != 1 {
t.Fatalf("re-request for the same head must not PUT again, got %d calls", got)
}
if _, err := os.Stat(again + ".exhausted"); err != nil {
t.Fatalf("re-request for the same head should be refused: %v", err)
}

out := logs.String()
for _, line := range strings.Split(out, "\n") {
if strings.Contains(line, "level=WARN") && (strings.Contains(line, "merge queue") || strings.Contains(line, "giving up")) {
t.Fatalf("merge-queue-required refusal must not log at WARN:\n%s", out)
}
}
if n := strings.Count(out, mergeQueueRequiredReason+"\""); n != 1 {
t.Fatalf("classified message logged %d times, want once:\n%s", n, out)
}
if reEngaged != 0 {
t.Fatalf("merge-queue-required is not fixable by code; re-engage hook called %d times", reEngaged)
}
}

func TestIsMergeQueueRequiredMergeBlocker(t *testing.T) {
if !isMergeQueueRequiredMergeBlocker("merging PR o/r#74 (squash): PUT https://api.github.com/repos/o/r/pulls/74/merge: 405 Repository rule violations found\n\nChanges must be made through the merge queue") {
t.Error("merge-queue-required 405 not recognised")
}
for _, msg := range []string{"Changes must be made through a pull request", "405 Pull Request is not mergeable"} {
if isMergeQueueRequiredMergeBlocker(msg) {
t.Errorf("%q misclassified as merge-queue-required", msg)
}
}
}
Loading