diff --git a/changelog.d/fixed-missing-check-alert-revalidation.md b/changelog.d/fixed-missing-check-alert-revalidation.md new file mode 100644 index 0000000000..3c9c92acaf --- /dev/null +++ b/changelog.d/fixed-missing-check-alert-revalidation.md @@ -0,0 +1 @@ +- Retire stale missing-required-check merge banners only after successful exact-current-head evidence, preserving required App identity and other PRs' unresolved blockers. diff --git a/src/docs/hive-merge.md b/src/docs/hive-merge.md index ff453008f7..4582acd903 100644 --- a/src/docs/hive-merge.md +++ b/src/docs/hive-merge.md @@ -174,8 +174,27 @@ not one per PR or retry tick. The alert text names the fix: install/grant the App Contents and Pull requests write permissions, adjust review/ruleset bypass or approve the PR, make a missing required check report, or enable/use an allowed merge method. Conflicts re-engage the fix loop instead of alerting, and -rate limits are treated as transient. A later successful merge in that repo -clears its merge-failure alerts automatically. +rate limits are treated as transient. Each aggregate banner retains every +originating PR; a successful merge or confirmed PR closure resolves only that +PR's blocker, not another PR's head-specific alert with the same reason. +Successful App merges still retire repository write-permission alerts and +disallowed-method alerts when the successful merge used that same method. +Method alerts also retire when a repository read positively confirms the +originally refused method is now enabled. A different method's success cannot +prove squash/rebase/merge settings, and unknown methods or failed reads retain +the banner. Reviews, absent CI and approval-held runs remain PR/head-specific; +another PR's successful merge does not prove those blockers resolved. + +The existing five-minute alert revalidation also retires a missing-required-check +banner when every named check succeeds on the PR's exact current head. Check-run +success, neutral and skipped conclusions follow the existing CI policy; commit +statuses must succeed. Readable protection metadata binds checks to the required +App when specified. The observer completes bounded pagination, rejects older-head +evidence and re-reads the PR head/base before retiring the originating blocker. +API errors, incomplete evidence, unknown protection metadata and pending/failed +checks retain the alert. Concurrent refusals replace the observation token and +cannot be cleared by an older observation. This changes only alert freshness: +all CI, review, protection and merge gates remain authoritative and unchanged. ```yaml auto_merge: diff --git a/src/pkg/github/client.go b/src/pkg/github/client.go index 5e3cae1380..2cdd22d528 100644 --- a/src/pkg/github/client.go +++ b/src/pkg/github/client.go @@ -154,7 +154,7 @@ type Client struct { autoMergeMinHeadAge time.Duration mergeAlertMu sync.Mutex mergeAlertSink MergeFailureAlertSink - mergeAlertIDsByRepo map[string]map[string]mergeAlertEntry + mergeAlertIDsByRepo map[string]map[string]*mergeAlertEntry mergeAlertLastRevalidate time.Time logger *slog.Logger stageFacts stageFactsState // out-of-cycle stage writes; see StageFacts @@ -4696,9 +4696,9 @@ func (c *Client) RecordPRMergedAudit(repo string, number int, method, sha, path // Every merge path ends here, so this is where the merged head's evidence // bundle gains its merge event and the PR gets its evidence pointer. c.recordReviewEvidenceMerge(repo, number, method, sha, path, credit.Logins()) - // A successful App merge on any path proves the repo is mergeable, so - // retire its "merge blocked" alerts (#11924). - c.clearMergeFailureAlertsForRepo(repo) + // A merge resolves this PR's blockers and proven repository-level causes, + // not another PR's required checks or review/head-specific conditions. + c.clearMergeFailureAlertsAfterMerge(repo, number, method) } // RepoWorkBreakdown explains the raw open issue and PR totals for one diff --git a/src/pkg/github/merge_failure_alert.go b/src/pkg/github/merge_failure_alert.go index 62d8061aa8..42f877a53a 100644 --- a/src/pkg/github/merge_failure_alert.go +++ b/src/pkg/github/merge_failure_alert.go @@ -28,6 +28,8 @@ type mergeFailureAlert struct { type mergeAlertEntry struct { key string number int + head string + method string } // mergeAlertRevalidateInterval bounds how often the merge-request watcher @@ -43,56 +45,62 @@ func (c *Client) SetMergeFailureAlertSink(sink MergeFailureAlertSink) { defer c.mergeAlertMu.Unlock() c.mergeAlertSink = sink if c.mergeAlertIDsByRepo == nil { - c.mergeAlertIDsByRepo = make(map[string]map[string]mergeAlertEntry) + c.mergeAlertIDsByRepo = make(map[string]map[string]*mergeAlertEntry) } } -func (c *Client) raiseMergeFailureAlert(repo string, number int, errMsg string) { +func (c *Client) raiseMergeFailureAlert(repo string, number int, errMsg string, provenance ...string) { alert, ok := classifyMergeFailureForOperator(repo, errMsg) if !ok || c == nil { return } c.mergeAlertMu.Lock() + defer c.mergeAlertMu.Unlock() sink := c.mergeAlertSink if sink == nil { - c.mergeAlertMu.Unlock() return } id := mergeFailureAlertID(repo, alert.key) if c.mergeAlertIDsByRepo == nil { - c.mergeAlertIDsByRepo = make(map[string]map[string]mergeAlertEntry) + c.mergeAlertIDsByRepo = make(map[string]map[string]*mergeAlertEntry) } repoKey := strings.ToLower(strings.TrimSpace(repo)) if c.mergeAlertIDsByRepo[repoKey] == nil { - c.mergeAlertIDsByRepo[repoKey] = make(map[string]mergeAlertEntry) + c.mergeAlertIDsByRepo[repoKey] = make(map[string]*mergeAlertEntry) } - c.mergeAlertIDsByRepo[repoKey][id] = mergeAlertEntry{key: alert.key, number: number} - c.mergeAlertMu.Unlock() + entry := &mergeAlertEntry{key: alert.key, number: number} + if len(provenance) > 0 { + entry.head = provenance[0] + } + if alert.key == "merge-method" { + var requested string + if len(provenance) > 1 { + requested = provenance[1] + } + entry.method = mergeFailureMethod(errMsg, requested) + } + c.mergeAlertIDsByRepo[repoKey][fmt.Sprintf("%s/%d", id, number)] = entry + // Serialize sink updates with bookkeeping so an older observation cannot + // clear a newly raised alert between deleting its entry and updating the sink. sink.AddSystemAlert(id, "error", alert.message) } -func (c *Client) clearMergeFailureAlertsForRepo(repo string) { +func (c *Client) clearMergeFailureAlertsAfterMerge(repo string, number int, method string) { if c == nil { return } repoKey := strings.ToLower(strings.TrimSpace(repo)) c.mergeAlertMu.Lock() - sink := c.mergeAlertSink - ids := c.mergeAlertIDsByRepo[repoKey] - if len(ids) == 0 || sink == nil { - c.mergeAlertMu.Unlock() - return - } - delete(c.mergeAlertIDsByRepo, repoKey) - clearIDs := make([]string, 0, len(ids)) - for id := range ids { - clearIDs = append(clearIDs, id) - } - c.mergeAlertMu.Unlock() - - for _, id := range clearIDs { - sink.ClearSystemAlert(id) + defer c.mergeAlertMu.Unlock() + for member, entry := range c.mergeAlertIDsByRepo[repoKey] { + // An App merge proves repository write access, and a merge using the + // same method resolves that method's setting. Reviews, absent CI and + // approval-held runs on another PR/head are not proven by this merge. + if entry.number == number || entry.key == "app-permission" || + (entry.key == "merge-method" && entry.method != "" && entry.method == method) { + c.clearMergeFailureAlertEntryLocked(repoKey, member, entry) + } } } @@ -100,7 +108,9 @@ func (c *Client) clearMergeFailureAlertsForRepo(repo string) { // gone away without an App merge in that repo: the blocked PR is no longer // open, (fork-run-approval) its head no longer has workflow runs awaiting // maintainer approval — the operator approved them or relaxed the repo/org -// setting — or (app-permission) a check-runs read on its head succeeds again. Errors leave the alert in place; only positive evidence clears it. +// setting — or (app-permission) a check-runs read on its head succeeds again. +// Missing-check alerts require successful exact-current-head evidence. +// Errors leave the alert in place; only positive evidence clears it. func (c *Client) revalidateMergeFailureAlerts(ctx context.Context, nowFn func() time.Time) { if c == nil || c.client == nil { return @@ -118,7 +128,7 @@ func (c *Client) revalidateMergeFailureAlerts(ctx context.Context, nowFn func() c.mergeAlertLastRevalidate = nowFn() type pending struct { repo, id string - entry mergeAlertEntry + entry *mergeAlertEntry } var work []pending for repo, ids := range c.mergeAlertIDsByRepo { @@ -140,7 +150,27 @@ func (c *Client) revalidateMergeFailureAlerts(ctx context.Context, nowFn func() if err != nil || pr == nil { continue } - resolved := pr.GetState() != "open" + resolved := pr.GetState() == "closed" + if pr.GetState() != "open" && !resolved { + continue + } + if !resolved && strings.HasPrefix(w.entry.key, "missing-checks:") { + resolved = c.missingMergeChecksResolved(ctx, owner, name, pr, + splitCheckNames(strings.TrimPrefix(w.entry.key, "missing-checks:"))) + } + if !resolved && w.entry.key == "merge-method" && w.entry.method != "" { + repository, _, rerr := c.client.Repositories.Get(ctx, owner, name) + if rerr == nil && repository != nil { + switch w.entry.method { + case "squash": + resolved = repository.GetAllowSquashMerge() + case "merge": + resolved = repository.GetAllowMergeCommit() + case "rebase": + resolved = repository.GetAllowRebaseMerge() + } + } + } if !resolved && w.entry.key == "app-permission" { // Positive evidence the App can act on the repo again: its // check-runs read on the blocked head succeeds. A real merge @@ -161,24 +191,55 @@ func (c *Client) revalidateMergeFailureAlerts(ctx context.Context, nowFn func() if !resolved { continue } - c.clearMergeFailureAlertID(w.repo, w.id) - c.logger.Info("merge alert cleared: blocking condition resolved", - slog.String("repo", w.repo), slog.Int("number", w.entry.number), slog.String("key", w.entry.key)) + c.mergeAlertMu.Lock() + retired, cleared := c.clearMergeFailureAlertEntryLocked(w.repo, w.id, w.entry) + c.mergeAlertMu.Unlock() + if retired { + c.logger.Info("merge alert origin retired: blocking condition resolved", + slog.String("repo", w.repo), slog.Int("number", w.entry.number), + slog.String("key", w.entry.key), slog.String("origin_head", w.entry.head), + slog.String("current_head", pr.GetHead().GetSHA()), slog.Bool("banner_retained", !cleared)) + } } } -func (c *Client) clearMergeFailureAlertID(repoKey, id string) { - c.mergeAlertMu.Lock() - sink := c.mergeAlertSink - if ids := c.mergeAlertIDsByRepo[repoKey]; ids != nil { - delete(ids, id) - if len(ids) == 0 { - delete(c.mergeAlertIDsByRepo, repoKey) +// The pointer is an observation token: a refusal raised during network reads +// replaces it and must survive the old observation. +func (c *Client) clearMergeFailureAlertEntryLocked(repoKey, member string, entry *mergeAlertEntry) (retired, bannerCleared bool) { + entries := c.mergeAlertIDsByRepo[repoKey] + if entries[member] != entry { + return false, false + } + delete(entries, member) + for _, other := range entries { + if other.key == entry.key { + return true, false } } - c.mergeAlertMu.Unlock() - if sink != nil { - sink.ClearSystemAlert(id) + if len(entries) == 0 { + delete(c.mergeAlertIDsByRepo, repoKey) + } + if c.mergeAlertSink != nil { + c.mergeAlertSink.ClearSystemAlert(mergeFailureAlertID(repoKey, entry.key)) + } + return true, true +} + +func mergeFailureMethod(message, requested string) string { + lower := strings.ToLower(message) + switch { + case strings.Contains(lower, "squash"): + return "squash" + case strings.Contains(lower, "rebase"): + return "rebase" + case strings.Contains(lower, "merge commits"): + return "merge" + } + switch requested { + case "squash", "merge", "rebase": + return requested + default: + return "" } } diff --git a/src/pkg/github/merge_failure_alert_checks.go b/src/pkg/github/merge_failure_alert_checks.go new file mode 100644 index 0000000000..499f03029c --- /dev/null +++ b/src/pkg/github/merge_failure_alert_checks.go @@ -0,0 +1,107 @@ +package github + +import ( + "context" + "strings" + + gh "github.com/google/go-github/v72/github" +) + +const mergeAlertEvidenceMaxPages = 3 + +// missingMergeChecksResolved only retires the missing-check banner. It neither +// grants a CI verdict nor schedules another merge, and never substitutes an +// old head's success for evidence on the current PR head. +func (c *Client) missingMergeChecksResolved(ctx context.Context, owner, repo string, pr *gh.PullRequest, names []string) bool { + head, base := pr.GetHead().GetSHA(), pr.GetBase().GetRef() + if head == "" || base == "" || len(names) == 0 { + return false + } + required, known := c.nativeObservationRequired(ctx, owner, repo, base) + if !known { + // In particular, a denied protection read cannot establish that an + // identically named check from another App satisfies this blocker. + return false + } + producers := make(map[string]int64, len(names)) + for _, name := range names { + appID, exists := required[name] + if !exists { + return false + } + producers[name] = appID + } + var checks []*gh.CheckRun + checkOpts := &gh.ListCheckRunsOptions{ListOptions: gh.ListOptions{PerPage: 100}} + for page := 0; ; page++ { + result, response, err := c.client.Checks.ListCheckRunsForRef(ctx, owner, repo, head, checkOpts) + if err != nil || result == nil || result.Total == nil { + return false + } + checks = append(checks, result.CheckRuns...) + if response == nil || response.NextPage == 0 { + if result.GetTotal() != len(checks) { + return false + } + break + } + if page+1 >= mergeAlertEvidenceMaxPages || response.NextPage <= checkOpts.Page { + return false + } + checkOpts.Page = response.NextPage + } + passed, blocked := make(map[string]bool), make(map[string]bool) + for _, check := range latestCheckRunsByNameAndApp(checks) { + name := check.GetName() + appID, wanted := producers[name] + if !wanted || (appID > 0 && check.GetApp().GetID() != appID) { + continue + } + conclusion := check.GetConclusion() + if check.GetID() > 0 && check.GetHeadSHA() == head && check.GetStatus() == "completed" && + (conclusion == "success" || conclusion == "neutral" || conclusion == "skipped") { + passed[name] = true + } else { + blocked[name] = true + } + } + statusOpts := &gh.ListOptions{PerPage: 100} + var statusCount int + for page := 0; ; page++ { + result, response, err := c.client.Repositories.GetCombinedStatus(ctx, owner, repo, head, statusOpts) + if err != nil || result == nil || result.TotalCount == nil || result.GetSHA() != head { + return false + } + statusCount += len(result.Statuses) + for _, status := range result.Statuses { + name := status.GetContext() + appID, wanted := producers[name] + if !wanted || appID > 0 { + continue + } + if status.GetID() > 0 && status.GetState() == "success" { + passed[name] = true + } else { + blocked[name] = true + } + } + if response == nil || response.NextPage == 0 { + if result.GetTotalCount() != statusCount { + return false + } + break + } + if page+1 >= mergeAlertEvidenceMaxPages || response.NextPage <= statusOpts.Page { + return false + } + statusOpts.Page = response.NextPage + } + for _, name := range names { + if !passed[name] || blocked[name] { + return false + } + } + current, _, err := c.client.PullRequests.Get(ctx, owner, repo, pr.GetNumber()) + return err == nil && current != nil && current.GetState() == "open" && + strings.EqualFold(current.GetHead().GetSHA(), head) && current.GetBase().GetRef() == base +} diff --git a/src/pkg/github/merge_failure_alert_checks_test.go b/src/pkg/github/merge_failure_alert_checks_test.go new file mode 100644 index 0000000000..94a273d2bc --- /dev/null +++ b/src/pkg/github/merge_failure_alert_checks_test.go @@ -0,0 +1,309 @@ +package github + +import ( + "context" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strconv" + "strings" + "sync/atomic" + "testing" + "time" +) + +const missingMergeCheckErr = `Required status check "guard" is expected` + +func TestMissingMergeCheckAlertExactCurrentHead(t *testing.T) { + tests := []struct { + name string + head string + checkHead string + status string + conclusion string + app int + requiredApp int + ruleApp int + protectionCode int + rulesCode int + checkCode int + statusCode int + finalCode int + finalHead string + finalBase string + checksMissing bool + statusSuccess bool + statusHead string + statusPaginate bool + statusPageFail bool + statusTruncate bool + secondMissing bool + paginate bool + truncated bool + pageError bool + pageLimit bool + newerPending bool + reraise bool + closed bool + wantClear bool + }{ + {name: "check now succeeds", wantClear: true}, + {name: "new current head succeeds after old refusal", head: "newer", checkHead: "newer", wantClear: true}, + {name: "old head success insufficient", checkHead: "old"}, + {name: "pending is not resolution", status: "in_progress", conclusion: ""}, + {name: "failed is not resolution", conclusion: "failure"}, + {name: "neutral follows existing policy", conclusion: "neutral", wantClear: true}, + {name: "scope skipped follows existing policy", conclusion: "skipped", wantClear: true}, + {name: "required App matches", requiredApp: 42, wantClear: true}, + {name: "different App cannot substitute", requiredApp: 99}, + {name: "ruleset producer matches", ruleApp: 42, wantClear: true}, + {name: "ruleset producer mismatch", ruleApp: 99}, + {name: "conflicting producer policy unknown", requiredApp: 42, ruleApp: 99}, + {name: "protection permission error", protectionCode: http.StatusForbidden}, + {name: "rules unavailable", rulesCode: http.StatusBadGateway}, + {name: "check read error", checkCode: http.StatusForbidden}, + {name: "status read error", statusCode: http.StatusBadGateway}, + {name: "PR recheck error", finalCode: http.StatusForbidden}, + {name: "head changes during observation", finalHead: "changed"}, + {name: "base changes during observation", finalBase: "other"}, + {name: "missing check retained", checksMissing: true}, + {name: "all originally missing checks must resolve", secondMissing: true}, + {name: "status success", checksMissing: true, statusSuccess: true, wantClear: true}, + {name: "old combined status head rejected", checksMissing: true, statusSuccess: true, statusHead: "old"}, + {name: "successful status on later page", checksMissing: true, statusSuccess: true, statusPaginate: true, wantClear: true}, + {name: "status pagination error retained", checksMissing: true, statusSuccess: true, statusPaginate: true, statusPageFail: true}, + {name: "truncated status evidence retained", checksMissing: true, statusSuccess: true, statusTruncate: true}, + {name: "status cannot substitute required App", checksMissing: true, statusSuccess: true, requiredApp: 42}, + {name: "positive evidence after first page", paginate: true, wantClear: true}, + {name: "partial pagination error retained", paginate: true, pageError: true}, + {name: "truncated evidence retained", truncated: true}, + {name: "bounded pagination retained", pageLimit: true}, + {name: "latest pending attempt overrides old success", newerPending: true}, + {name: "new refusal during observation retained", reraise: true}, + {name: "closed PR still resolves", closed: true, wantClear: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + head, checkHead := tt.head, tt.checkHead + if head == "" { + head = "new" + } + if checkHead == "" { + checkHead = head + } + status, conclusion, app := tt.status, tt.conclusion, tt.app + if status == "" { + status = "completed" + } + if conclusion == "" && status == "completed" { + conclusion = "success" + } + if app == 0 { + app = 42 + } + var c *Client + prReads, checkReads := 0, 0 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + if r.Method != http.MethodGet { + t.Errorf("alert observer attempted mutation: %s %s", r.Method, r.URL.Path) + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + switch { + case strings.HasSuffix(r.URL.Path, "/pulls/7"): + prReads++ + currentHead, base, state := head, "v5", "open" + if tt.closed { + state = "closed" + } + if prReads > 1 { + if tt.finalCode != 0 { + w.WriteHeader(tt.finalCode) + return + } + if tt.finalHead != "" { + currentHead = tt.finalHead + } + if tt.finalBase != "" { + base = tt.finalBase + } + if tt.reraise { + c.raiseMergeFailureAlert("o/r", 7, missingMergeCheckErr, head) + } + } + _, _ = fmt.Fprintf(w, `{"number":7,"state":%q,"head":{"sha":%q},"base":{"ref":%q}}`, state, currentHead, base) + case strings.HasSuffix(r.URL.Path, "/protection/required_status_checks"): + if tt.protectionCode != 0 { + w.WriteHeader(tt.protectionCode) + return + } + _, _ = fmt.Fprintf(w, `{"contexts":["guard"],"checks":[{"context":"guard","app_id":%d}]}`, tt.requiredApp) + case strings.Contains(r.URL.Path, "/rules/branches/"): + if tt.rulesCode != 0 { + w.WriteHeader(tt.rulesCode) + return + } + if tt.ruleApp != 0 { + _, _ = fmt.Fprintf(w, `[{"type":"required_status_checks","parameters":{"required_status_checks":[{"context":"guard","integration_id":%d}]}}]`, tt.ruleApp) + } else { + _, _ = io.WriteString(w, `[]`) + } + case strings.HasSuffix(r.URL.Path, "/check-runs"): + checkReads++ + if !strings.HasSuffix(r.URL.Path, "/commits/"+head+"/check-runs") { + t.Errorf("read evidence for wrong head: %s", r.URL.Path) + } + page, _ := strconv.Atoi(r.URL.Query().Get("page")) + if tt.checkCode != 0 || tt.pageError && page == 2 { + code := tt.checkCode + if code == 0 { + code = http.StatusBadGateway + } + w.WriteHeader(code) + return + } + if (tt.paginate && page != 2) || tt.pageLimit { + next := page + 1 + if next < 2 { + next = 2 + } + w.Header().Set("Link", fmt.Sprintf(`; rel="next"`, r.Host, r.URL.Path, next)) + _, _ = io.WriteString(w, `{"total_count":2,"check_runs":[{"id":1,"name":"unrelated"}]}`) + return + } + checks := fmt.Sprintf(`{"id":2,"name":"guard","head_sha":%q,"status":%q,"conclusion":%q,"app":{"id":%d}}`, checkHead, status, conclusion, app) + if tt.newerPending { + checks += fmt.Sprintf(`,{"id":3,"name":"guard","head_sha":%q,"status":"queued","app":{"id":%d}}`, head, app) + } + total := 1 + if tt.checksMissing { + checks, total = "", 0 + } + if tt.newerPending || tt.paginate || tt.truncated { + total = 2 + } + _, _ = fmt.Fprintf(w, `{"total_count":%d,"check_runs":[%s]}`, total, checks) + case strings.HasSuffix(r.URL.Path, "/status"): + page, _ := strconv.Atoi(r.URL.Query().Get("page")) + if tt.statusPaginate && page != 2 { + w.Header().Set("Link", fmt.Sprintf(`; rel="next"`, r.Host, r.URL.Path)) + _, _ = fmt.Fprintf(w, `{"sha":%q,"total_count":1,"statuses":[]}`, head) + return + } + if tt.statusPageFail && page == 2 { + w.WriteHeader(http.StatusBadGateway) + return + } + if tt.statusCode != 0 { + w.WriteHeader(tt.statusCode) + return + } + statusHead := head + if tt.statusHead != "" { + statusHead = tt.statusHead + } + statuses, total := "", 0 + if tt.statusSuccess { + statuses, total = `{"id":8,"context":"guard","state":"success"}`, 1 + } + if tt.statusTruncate { + total++ + } + _, _ = fmt.Fprintf(w, `{"sha":%q,"total_count":%d,"statuses":[%s]}`, statusHead, total, statuses) + default: + t.Errorf("unexpected observer request: %s", r.URL.Path) + w.WriteHeader(http.StatusNotFound) + } + })) + defer srv.Close() + c = testMergeClient(t, srv.URL) + sink := &recordingMergeAlertSink{} + c.SetMergeFailureAlertSink(sink) + trigger := missingMergeCheckErr + if tt.secondMissing { + trigger = "ci gate: required check(s) not yet reported on old: guard, lint" + } + c.raiseMergeFailureAlert("o/r", 7, trigger, "old") + c.revalidateMergeFailureAlerts(context.Background(), nil) + if got := len(sink.clears) == 1; got != tt.wantClear { + t.Fatalf("cleared=%v, want %v; PR reads=%d check pages=%d", got, tt.wantClear, prReads, checkReads) + } + if checkReads > mergeAlertEvidenceMaxPages { + t.Fatalf("unbounded evidence reads: %d", checkReads) + } + }) + } +} + +func TestMissingMergeCheckAlertPreservesOtherPRs(t *testing.T) { + var blocked atomic.Bool + blocked.Store(true) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + if r.Method != http.MethodGet { + t.Errorf("unexpected mutation: %s", r.Method) + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + switch { + case strings.Contains(r.URL.Path, "/pulls/"): + number := 7 + if strings.HasSuffix(r.URL.Path, "/8") { + number = 8 + } + _, _ = fmt.Fprintf(w, `{"number":%d,"state":"open","head":{"sha":"head%d"},"base":{"ref":"v5"}}`, number, number) + case strings.HasSuffix(r.URL.Path, "/protection/required_status_checks"): + _, _ = io.WriteString(w, `{"contexts":["guard"]}`) + case strings.Contains(r.URL.Path, "/rules/branches/"): + _, _ = io.WriteString(w, `[]`) + case strings.HasSuffix(r.URL.Path, "/check-runs"): + head := "head7" + if strings.Contains(r.URL.Path, "/head8/") { + head = "head8" + if blocked.Load() { + _, _ = io.WriteString(w, `{"total_count":0,"check_runs":[]}`) + return + } + } + _, _ = fmt.Fprintf(w, `{"total_count":1,"check_runs":[{"id":42,"name":"guard","head_sha":%q,"status":"completed","conclusion":"success","app":{"id":42}}]}`, head) + case strings.HasSuffix(r.URL.Path, "/status"): + head := "head7" + if strings.Contains(r.URL.Path, "/head8/") { + head = "head8" + } + _, _ = fmt.Fprintf(w, `{"sha":%q,"total_count":0,"statuses":[]}`, head) + default: + w.WriteHeader(http.StatusNotFound) + } + })) + defer srv.Close() + c := testMergeClient(t, srv.URL) + sink := &recordingMergeAlertSink{} + c.SetMergeFailureAlertSink(sink) + c.raiseMergeFailureAlert("O/R", 7, missingMergeCheckErr, "head7") + c.raiseMergeFailureAlert("o/r", 8, missingMergeCheckErr, "head8") + if sink.adds[0].id != sink.adds[1].id { + t.Fatal("aggregate banner identity changed") + } + now := time.Date(2026, 10, 31, 12, 0, 0, 0, time.UTC) + nowFn := func() time.Time { return now } + c.revalidateMergeFailureAlerts(context.Background(), nowFn) + if len(sink.clears) != 0 { + t.Fatal("resolved PR hid another PR's missing check") + } + c.RecordPRMergedAudit("O/R", 7, "squash", "head7", PRAuditPathSweep) + if len(sink.clears) != 0 { + t.Fatal("another PR's merge hid unresolved blocker") + } + blocked.Store(false) + now = now.Add(mergeAlertRevalidateInterval) + c.revalidateMergeFailureAlerts(context.Background(), nowFn) + if len(sink.clears) != 1 { + t.Fatalf("all originating blockers resolved but aggregate remains: %v", sink.clears) + } + if len(c.mergeAlertIDsByRepo) != 0 { + t.Fatal("resolved source entries retained") + } +} diff --git a/src/pkg/github/merge_failure_alert_scope_test.go b/src/pkg/github/merge_failure_alert_scope_test.go new file mode 100644 index 0000000000..b037dd8542 --- /dev/null +++ b/src/pkg/github/merge_failure_alert_scope_test.go @@ -0,0 +1,110 @@ +package github + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +func TestMergeFailureAlertSuccessfulMergeProofScope(t *testing.T) { + for _, tt := range []struct { + name, trigger, requested, merged string + wantClear bool + }{ + {"repository permission", appPermissionErr, "", "squash", true}, + {"same refused method", "squash commits are not allowed", "", "squash", true}, + {"different method insufficient", "squash commits are not allowed", "", "merge", false}, + {"requested method known", "merge method is not allowed", "rebase", "rebase", true}, + {"unknown method retained", "merge method is not allowed", "", "squash", false}, + {"actual error method beats requested method", "merge commits are not allowed", "squash", "squash", false}, + {"another PR review not proven", "At least 1 approving review is required", "", "squash", false}, + {"another head missing CI not proven", "required check was never created", "", "squash", false}, + {"another head missing check not proven", missingMergeCheckErr, "", "squash", false}, + {"another head run approval not proven", forkApprovalErr, "", "squash", false}, + } { + t.Run(tt.name, func(t *testing.T) { + c := NewClientForTest("http://127.0.0.1:0", "o", []string{"r"}, nil) + sink := &recordingMergeAlertSink{} + c.SetMergeFailureAlertSink(sink) + c.raiseMergeFailureAlert("o/r", 7, tt.trigger, "old", tt.requested) + c.RecordPRMergedAudit("O/R", 9, tt.merged, "head9", PRAuditPathSweep) + if got := len(sink.clears) == 1; got != tt.wantClear { + t.Fatalf("cleared=%v, want %v", got, tt.wantClear) + } + }) + } +} + +func TestMergeMethodAlertRevalidatesOriginalMethod(t *testing.T) { + for _, tt := range []struct { + name, trigger, flags string + repoError bool + wantClear bool + }{ + {"squash enabled", "squash commits are not allowed", `"allow_squash_merge":true`, false, true}, + {"rebase enabled", "rebase commits are not allowed", `"allow_rebase_merge":true`, false, true}, + {"merge enabled", "merge commits are not allowed", `"allow_merge_commit":true`, false, true}, + {"wrong method enabled", "squash commits are not allowed", `"allow_merge_commit":true`, false, false}, + {"disabled retains", "squash commits are not allowed", `"allow_squash_merge":false`, false, false}, + {"missing metadata retains", "squash commits are not allowed", "", false, false}, + {"API error retains", "squash commits are not allowed", "", true, false}, + {"unknown method retains", "merge method is not allowed", `"allow_squash_merge":true`, false, false}, + } { + t.Run(tt.name, func(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + if r.Method != http.MethodGet { + t.Errorf("unexpected mutation: %s", r.Method) + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + if strings.HasSuffix(r.URL.Path, "/pulls/7") { + _, _ = fmt.Fprint(w, `{"number":7,"state":"open","head":{"sha":"new"},"base":{"ref":"v5"}}`) + } else if r.URL.Path == "/repos/o/r" { + if tt.repoError { + w.WriteHeader(http.StatusForbidden) + } else { + _, _ = fmt.Fprintf(w, `{%s}`, tt.flags) + } + } else { + t.Errorf("unexpected read: %s", r.URL.Path) + w.WriteHeader(http.StatusNotFound) + } + })) + defer srv.Close() + c := testMergeClient(t, srv.URL) + sink := &recordingMergeAlertSink{} + c.SetMergeFailureAlertSink(sink) + c.raiseMergeFailureAlert("o/r", 7, tt.trigger, "old") + c.revalidateMergeFailureAlerts(context.Background(), nil) + if got := len(sink.clears) == 1; got != tt.wantClear { + t.Fatalf("cleared=%v, want %v", got, tt.wantClear) + } + }) + } +} + +func TestMergeFailureAlertOriginRetirementDistinguishesBannerRetention(t *testing.T) { + c := NewClientForTest("http://127.0.0.1:0", "o", []string{"r"}, nil) + sink := &recordingMergeAlertSink{} + c.SetMergeFailureAlertSink(sink) + c.raiseMergeFailureAlert("o/r", 7, missingMergeCheckErr, "head7") + c.raiseMergeFailureAlert("o/r", 8, missingMergeCheckErr, "head8") + c.mergeAlertMu.Lock() + defer c.mergeAlertMu.Unlock() + for member, entry := range c.mergeAlertIDsByRepo["o/r"] { + retired, cleared := c.clearMergeFailureAlertEntryLocked("o/r", member, entry) + if !retired || cleared != (len(c.mergeAlertIDsByRepo["o/r"]) == 0) { + t.Fatalf("retired=%v bannerCleared=%v", retired, cleared) + } + if retiredAgain, _ := c.clearMergeFailureAlertEntryLocked("o/r", member, entry); retiredAgain { + t.Fatal("old token retired twice") + } + } + if len(sink.clears) != 1 { + t.Fatalf("aggregate must clear only once: %v", sink.clears) + } +} diff --git a/src/pkg/github/merge_failure_alert_test.go b/src/pkg/github/merge_failure_alert_test.go index a7082449a9..4d8ef990a0 100644 --- a/src/pkg/github/merge_failure_alert_test.go +++ b/src/pkg/github/merge_failure_alert_test.go @@ -98,11 +98,11 @@ func TestMergeFailureAlertsDedupeAndClearOnSuccess(t *testing.T) { t.Fatalf("severity = %q, want error", sink.adds[0].severity) } - c.clearMergeFailureAlertsForRepo("o/r") + c.clearMergeFailureAlertsAfterMerge("o/r", 7, "squash") if len(sink.clears) != 1 || sink.clears[0] != sink.adds[0].id { t.Fatalf("clear on success = %+v, want %q", sink.clears, sink.adds[0].id) } - c.clearMergeFailureAlertsForRepo("o/r") + c.clearMergeFailureAlertsAfterMerge("o/r", 7, "squash") if len(sink.clears) != 1 { t.Fatalf("second clear should be no-op, got %+v", sink.clears) } @@ -141,7 +141,7 @@ func TestClassifyMergeFailureMergePUT403StillAppPermission(t *testing.T) { } // Every App merge path records its merge through RecordPRMergedAudit, so the -// self-authored sweep and the queue retire the repo's alerts too (#11924). +// self-authored sweep and the queue retire proven repository-permission alerts. func TestRecordPRMergedAuditClearsMergeFailureAlerts(t *testing.T) { c := NewClientForTest("http://127.0.0.1:0", "o", []string{"r"}, nil) sink := &recordingMergeAlertSink{} diff --git a/src/pkg/github/merge_request_watcher.go b/src/pkg/github/merge_request_watcher.go index e69c81a012..d625eba300 100644 --- a/src/pkg/github/merge_request_watcher.go +++ b/src/pkg/github/merge_request_watcher.go @@ -197,7 +197,6 @@ func (c *Client) handleForgeMergeRequest(ctx context.Context, path string, req M slog.String("repo", req.Repo), slog.Int("number", req.Number), slog.String("head_sha", req.ExpectSHA), slog.String("agent", req.Agent)) c.RecordPRMergedAudit(req.Repo, req.Number, method, req.ExpectSHA, PRAuditPathRelay) - c.clearMergeFailureAlertsForRepo(req.Repo) } // isRequiredCheckMergeBlocker classifies a merge-attempt error string as @@ -513,7 +512,6 @@ func (c *Client) handleOneMergeRequest(ctx context.Context, path string, nowFn f c.logger.Info("merge-request watcher: PR merged by App bot", slog.String("repo", req.Repo), slog.Int("number", req.Number), slog.String("sha", res.SHA), slog.String("agent", req.Agent)) - c.clearMergeFailureAlertsForRepo(req.Repo) } // recordMergeFailure writes the failed attempt's result and applies the retry @@ -524,7 +522,7 @@ func (c *Client) recordMergeFailure(path string, req MergeRequest, attempts int, now := nowFn() c.writeMergeResult(path, MergeResponse{Number: req.Number, Attempts: attempts, OK: false, Error: errMsg, At: now.UTC().Format(time.RFC3339)}) c.putMergeRequestState(req.Repo, req.Number, mergeRequestPRState{SHA: req.ExpectSHA, Attempts: attempts, Exhausted: attempts >= mergeRequestMaxAttempts, Reason: errMsg, At: now}) - c.raiseMergeFailureAlert(req.Repo, req.Number, errMsg) + c.raiseMergeFailureAlert(req.Repo, req.Number, errMsg, req.ExpectSHA, req.Method) if attempts >= mergeRequestMaxAttempts { // Terminal: a PR that still won't merge after N tries is blocked by // something a retry can't fix. Fix #2: classify WHY. If the blocker @@ -577,7 +575,7 @@ func (c *Client) quarantineMergeRequest(path string, req MergeRequest, attempts now := nowFn() c.writeMergeResult(path, MergeResponse{Number: req.Number, Attempts: attempts, OK: false, Error: reason, At: now.UTC().Format(time.RFC3339)}) c.putMergeRequestState(req.Repo, req.Number, mergeRequestPRState{SHA: req.ExpectSHA, Attempts: attempts, Exhausted: true, Reason: reason, At: now}) - c.raiseMergeFailureAlert(req.Repo, req.Number, reason) + c.raiseMergeFailureAlert(req.Repo, req.Number, reason, req.ExpectSHA, req.Method) _ = os.Rename(path, path+".exhausted") c.logger.Warn("merge-request watcher: merge cannot proceed on this head, quarantined without retry", slog.String("repo", req.Repo), slog.Int("number", req.Number),