From 41df01421abe8a116d749d872859bdfa495e8276 Mon Sep 17 00:00:00 2001 From: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Mon, 2 Nov 2026 11:05:54 -0500 Subject: [PATCH 1/2] fix(merge-watcher): treat merge-queue-required 405 as terminal and log it once Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- changelog.d/fixed-16229-merge-queue-405.md | 1 + src/pkg/github/merge_request_watcher.go | 37 ++++++++++ src/pkg/github/merge_request_watcher_test.go | 78 ++++++++++++++++++++ 3 files changed, 116 insertions(+) create mode 100644 changelog.d/fixed-16229-merge-queue-405.md diff --git a/changelog.d/fixed-16229-merge-queue-405.md b/changelog.d/fixed-16229-merge-queue-405.md new file mode 100644 index 0000000000..5b55d53139 --- /dev/null +++ b/changelog.d/fixed-16229-merge-queue-405.md @@ -0,0 +1 @@ +- The merge-request watcher now treats GitHub's "Changes must be made through the merge queue" 405 as terminal: it stops retrying the direct merge and logs one classified message per PR head instead of repeated warnings. (#16229) diff --git a/src/pkg/github/merge_request_watcher.go b/src/pkg/github/merge_request_watcher.go index b48c534577..46a42b6d57 100644 --- a/src/pkg/github/merge_request_watcher.go +++ b/src/pkg/github/merge_request_watcher.go @@ -225,6 +225,20 @@ func isRequiredCheckMergeBlocker(errMsg string) bool { return false } +// mergeQueueRequiredReason is the classified reason recorded when the base +// branch's ruleset only accepts merges through GitHub's merge queue. +const mergeQueueRequiredReason = "repo requires merge queue; direct merge not possible — enqueue via merge queue or merge by hand" + +// isMergeQueueRequiredMergeBlocker reports the 405 GitHub returns for a direct +// REST merge on a base branch whose ruleset requires a merge queue +// ("Repository rule violations found ... Changes must be made through the +// merge queue"). No retry of PUT /merge can succeed there +// (hivecommons/hive#16229); the automerge sweep classifies the same error as +// a skip reason (hivecommons/hive#14018). +func isMergeQueueRequiredMergeBlocker(errMsg string) bool { + return strings.Contains(strings.ToLower(errMsg), "must be made through the merge queue") +} + func isConflictMergeBlocker(errMsg string) bool { m := strings.ToLower(errMsg) return strings.Contains(m, "merge conflict") || @@ -502,6 +516,10 @@ func (c *Client) handleOneMergeRequest(ctx context.Context, path string, nowFn f res, err := c.MergePR(ctx, req.Repo, req.Number, req.Method, req.ExpectSHA) if err != nil { + if isMergeQueueRequiredMergeBlocker(err.Error()) { + c.quarantineMergeQueueRequired(path, req, attempts, err.Error(), nowFn) + return + } c.recordMergeFailure(path, req, attempts, err.Error(), nowFn) return } @@ -582,6 +600,25 @@ func (c *Client) quarantineMergeRequest(path string, req MergeRequest, attempts slog.String("sha", req.ExpectSHA), slog.String("reason", reason)) } +// quarantineMergeQueueRequired sets a request aside on the first +// merge-queue-required refusal, without spending the retry budget. Like +// quarantineMergeRequest it records the exhaustion against the PR+SHA, so a +// re-request for the same head is refused quietly and the classified message +// is logged once per head. It logs at INFO: the blocker is the target repo's +// configuration, not a hive defect, and a WARN per head would surface as an +// unclassified_error_repeated selfreport. +func (c *Client) quarantineMergeQueueRequired(path string, req MergeRequest, attempts int, errMsg string, nowFn func() time.Time) { + now := nowFn() + reason := mergeQueueRequiredReason + ": " + errMsg + c.writeMergeResult(path, MergeResponse{Number: req.Number, Attempts: attempts, OK: false, Error: reason, At: now.UTC().Format(time.RFC3339)}) + c.putMergeRequestState(req.Repo, req.Number, mergeRequestPRState{SHA: req.ExpectSHA, Attempts: attempts, Exhausted: true, Reason: reason, At: now}) + _ = os.Rename(path, path+".exhausted") + c.logger.Info("merge-request watcher: "+mergeQueueRequiredReason, + slog.String("repo", req.Repo), slog.Int("number", req.Number), + slog.String("sha", req.ExpectSHA), slog.Int("attempts", attempts), + slog.String("error", errMsg)) +} + // refuseExhaustedMergeRequest answers a new request for a PR+SHA whose earlier // request was already exhausted with the recorded reason, instead of starting // another wait/retry cycle on a head that has not changed. diff --git a/src/pkg/github/merge_request_watcher_test.go b/src/pkg/github/merge_request_watcher_test.go index d03683ef5b..0498f1fb69 100644 --- a/src/pkg/github/merge_request_watcher_test.go +++ b/src/pkg/github/merge_request_watcher_test.go @@ -483,3 +483,81 @@ func TestMergeRequestWatcher_RoutesNonGitHubRepoToForge(t *testing.T) { t.Errorf("human_merge_paths repo should be denied: %v", err) } } + +// A 405 "Changes must be made through the merge queue" can never be cleared by +// retrying PUT /merge (hivecommons/hive#16229): the request is set aside on the +// first refusal with the classified reason, a re-request for the same head is +// refused without another PUT, and nothing is logged at WARN. +func TestMergeRequestWatcher_MergeQueueRequiredIsTerminal(t *testing.T) { + var puts atomic.Int32 + green := greenFixture() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if green.serveCI(w, r) { + return + } + if r.Method == "PUT" && strings.HasSuffix(r.URL.Path, "/merge") { + puts.Add(1) + w.WriteHeader(http.StatusMethodNotAllowed) + _, _ = io.WriteString(w, `{"message":"Repository rule violations found\n\nChanges must be made through the merge queue\n\n"}`) + return + } + w.WriteHeader(http.StatusNotFound) + })) + defer srv.Close() + c := testMergeClient(t, srv.URL) + var logs strings.Builder + c.logger = slog.New(slog.NewTextHandler(&logs, &slog.HandlerOptions{Level: slog.LevelDebug})) + var reEngaged int + c.SetMergeReEngageHook(func(string, int) bool { reEngaged++; return true }) + + dir := t.TempDir() + mergeRequestDirForTest = dir + defer func() { mergeRequestDirForTest = "" }() + + reqPath, _ := WriteMergeRequest(dir, MergeRequest{Repo: "o/r", Number: 74, ExpectSHA: "abc", Agent: "scanner"}) + c.ProcessMergeRequestsOnce(context.Background()) + + if got := puts.Load(); got != 1 { + t.Fatalf("PUT /merge calls = %d, want 1", got) + } + if _, err := os.Stat(reqPath + ".exhausted"); err != nil { + t.Fatalf("merge-queue-required refusal should quarantine on the first attempt: %v", err) + } + resp := readMergeResult(t, reqPath) + if resp.OK || resp.Attempts != 1 || !strings.Contains(resp.Error, mergeQueueRequiredReason) { + t.Fatalf("result = %+v, want a failed attempt 1 carrying %q", resp, mergeQueueRequiredReason) + } + + again, _ := WriteMergeRequest(dir, MergeRequest{Repo: "o/r", Number: 74, ExpectSHA: "abc", Agent: "scanner"}) + c.ProcessMergeRequestsOnce(context.Background()) + if got := puts.Load(); got != 1 { + t.Fatalf("re-request for the same head must not PUT again, got %d calls", got) + } + if _, err := os.Stat(again + ".exhausted"); err != nil { + t.Fatalf("re-request for the same head should be refused: %v", err) + } + + out := logs.String() + for _, line := range strings.Split(out, "\n") { + if strings.Contains(line, "level=WARN") && (strings.Contains(line, "merge queue") || strings.Contains(line, "giving up")) { + t.Fatalf("merge-queue-required refusal must not log at WARN:\n%s", out) + } + } + if n := strings.Count(out, mergeQueueRequiredReason+"\""); n != 1 { + t.Fatalf("classified message logged %d times, want once:\n%s", n, out) + } + if reEngaged != 0 { + t.Fatalf("merge-queue-required is not fixable by code; re-engage hook called %d times", reEngaged) + } +} + +func TestIsMergeQueueRequiredMergeBlocker(t *testing.T) { + if !isMergeQueueRequiredMergeBlocker("merging PR o/r#74 (squash): PUT https://api.github.com/repos/o/r/pulls/74/merge: 405 Repository rule violations found\n\nChanges must be made through the merge queue") { + t.Error("merge-queue-required 405 not recognised") + } + for _, msg := range []string{"Changes must be made through a pull request", "405 Pull Request is not mergeable"} { + if isMergeQueueRequiredMergeBlocker(msg) { + t.Errorf("%q misclassified as merge-queue-required", msg) + } + } +} From f6df2d57eca276692d00065ed7e7410bb6bf0291 Mon Sep 17 00:00:00 2001 From: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Mon, 2 Nov 2026 18:08:38 -0500 Subject: [PATCH 2/2] docs(hive-merge): describe the merge-queue-required terminal path The watcher doc enumerates every verdict that quarantines a request, but the new first-attempt quarantine on a 405 merge-queue refusal fires on the merge call rather than the CI verdict and raises no operator alert, so a reader matching an .exhausted result file had nothing to match it against. Also note on the helper why it skips raiseMergeFailureAlert so a future merge-queue classifier case is not silently bypassed. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- src/docs/hive-merge.md | 13 +++++++++++++ src/pkg/github/merge_request_watcher.go | 5 ++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/src/docs/hive-merge.md b/src/docs/hive-merge.md index fa451fc8ba..1af3c24b52 100644 --- a/src/docs/hive-merge.md +++ b/src/docs/hive-merge.md @@ -100,6 +100,19 @@ verdict: An API error while gathering that evidence is a failed attempt, never a pass. +**A merge-queue-required refusal is terminal on the first attempt.** When the +merge call itself (`PUT /pulls/{n}/merge`) answers `405 Changes must be made +through the merge queue`, the base branch's ruleset forbids direct merges and no +retry, code push or re-engaged fix loop can change that. The request is +quarantined (`.exhausted`) for that head immediately, without consuming the +retry budget, raising an operator alert or re-engaging the fix loop; the result +file records `repo requires merge queue; direct merge not possible — enqueue via +merge queue or merge by hand` followed by GitHub's message, and the watcher logs +it once per head at INFO. Clear it by enqueuing the PR through the merge queue +(see `auto_merge.native_merge_queue.enabled` in the +[operator reference](operator-reference/README.md)) or merging by hand; a new +head SHA starts fresh like any other exhausted request. + **Wait state survives restarts and re-requests.** The attempt count, `ci_waits`, the pending-poll backoff and an "exhausted" record are kept per PR and head SHA in `/data/merge-request-state.json` (the PVC), not only in the tmpfs result diff --git a/src/pkg/github/merge_request_watcher.go b/src/pkg/github/merge_request_watcher.go index bbb5484c36..754eda9ca2 100644 --- a/src/pkg/github/merge_request_watcher.go +++ b/src/pkg/github/merge_request_watcher.go @@ -620,7 +620,10 @@ func (c *Client) quarantineMergeRequest(path string, req MergeRequest, attempts // re-request for the same head is refused quietly and the classified message // is logged once per head. It logs at INFO: the blocker is the target repo's // configuration, not a hive defect, and a WARN per head would surface as an -// unclassified_error_repeated selfreport. +// unclassified_error_repeated selfreport. It deliberately does not call +// raiseMergeFailureAlert: classifyMergeFailureForOperator has no merge-queue +// case, so no alert was ever raised for this error; add one there before +// routing this path through the alert if that changes. func (c *Client) quarantineMergeQueueRequired(path string, req MergeRequest, attempts int, errMsg string, nowFn func() time.Time) { now := nowFn() reason := mergeQueueRequiredReason + ": " + errMsg