diff --git a/changelog.d/fixed-16287-open-pr-guard-timeout.md b/changelog.d/fixed-16287-open-pr-guard-timeout.md new file mode 100644 index 0000000000..9435aaf31d --- /dev/null +++ b/changelog.d/fixed-16287-open-pr-guard-timeout.md @@ -0,0 +1 @@ +- Open-PR guard bounds its listing deadline and backs off per repo after a failure, logging a classified reason once per window (#16287) diff --git a/changelog.d/fixed-open-pr-guard-rate-limit.md b/changelog.d/fixed-open-pr-guard-rate-limit.md new file mode 100644 index 0000000000..20692fa45c --- /dev/null +++ b/changelog.d/fixed-open-pr-guard-rate-limit.md @@ -0,0 +1 @@ +- The open-PR pre-dispatch guard no longer hot-loops or fails open while the GitHub installation is rate-limited. A rate-limited open-PR listing (primary `RateLimitError` or secondary `AbuseRateLimitError`) is now remembered per repo until the limit resets (`Rate.Reset` / `Retry-After`, capped at an hour, one minute when unknown) and replayed without a GitHub call, warned about once per repo per window instead of once per candidate — one hive logged 47,000 `could not list open PRs` warnings in an hour. While the listing is rate-limited the guard refuses the dispatch (dispatch reason `open-PR guard deferred: GitHub rate limit`, withheld reason `open_pr_guard_rate_limited`) instead of admitting issues an open PR may already link; other listing failures still fail open. diff --git a/src/docs/operator-reference.md b/src/docs/operator-reference.md index 8d0f160576..e55555dbb3 100644 --- a/src/docs/operator-reference.md +++ b/src/docs/operator-reference.md @@ -1192,9 +1192,15 @@ between. Two mechanisms close that gap already links this issue (claim skipped)* with the PR as evidence (reason `open_pr_linked`). A failed listing fails **open**: the ledger-backed guards still apply, and a flaky listing must not stall every - claim. The gate also refuses an issue the confirming read says is already - **closed** (`issue closed`), so an issue closed between enumerations is - never dispatched from the cached snapshot. + claim; the failure is remembered per repo for a minute, so the listing is + not retried and the warning not repeated per candidate. A **rate-limited** + listing fails **closed** instead: the guard cannot see an already-linked + PR, so every candidate in the repo is deferred until the limit resets + (dispatch reason `open-PR guard deferred: GitHub rate limit`, withheld + reason `open_pr_guard_rate_limited`), with no GitHub call and one + warning per repo until then. The gate also refuses an issue the + confirming read says is already **closed** (`issue closed`), so an issue + closed between enumerations is never dispatched from the cached snapshot. - **Webhook claim linking.** A `pull_request` delivery (opened, reopened, edited, synchronize, ready-for-review, draft, closed) is parsed with the same closing-keyword and branch-name grammar and written into the claim diff --git a/src/pkg/dashboard/contribute_admission.go b/src/pkg/dashboard/contribute_admission.go index e96bde1467..4c518e33b0 100644 --- a/src/pkg/dashboard/contribute_admission.go +++ b/src/pkg/dashboard/contribute_admission.go @@ -100,7 +100,8 @@ type contributorAdmissionDecision struct { reason string claim ghpkg.IssueClaim // openPR carries the live open PR behind an open_pr_linked refusal - // (hivecommons/hive#14799). Zero for every other reason. + // (hivecommons/hive#14799), or the reset time behind an + // open_pr_guard_rate_limited one. Zero for every other reason. openPR ghpkg.OpenPRLink // churn carries the pull-request history behind an issue_churn refusal // (#7995): which PRs merged, which closed unmerged. Zero-valued for every @@ -127,6 +128,11 @@ const ( // It is what open_pr_claim becomes once the ledger catches up; the // distinct code tells an operator the guard, not the ledger, caught it. contributorAdmissionReasonOpenPRLinked = "open_pr_linked" + // contributorAdmissionReasonOpenPRGuardRateLimited: GitHub rate-limited + // the guard's open-PR listing, so it cannot tell whether a PR already + // links this issue. The candidate is deferred until the limit resets + // rather than dispatched blind. + contributorAdmissionReasonOpenPRGuardRateLimited = "open_pr_guard_rate_limited" // contributorAdmissionReasonIssueClaim: someone has claimed the issue on // the issue itself — a `hive:claim` marker comment or an assignee — and // the claim has not expired (hivecommons/hive#8380). It covers the window @@ -206,8 +212,12 @@ func (h *ContributeWSHub) evaluateContributorNeutralAdmission(sweep *contributor // a PR opened since is only visible on the forge. One listing per repo // per sweep, cached by the client. if link, ok := h.openPRLinkingIssue(candidate); ok { + reason := contributorAdmissionReasonOpenPRLinked + if link.RateLimited() { + reason = contributorAdmissionReasonOpenPRGuardRateLimited + } return contributorAdmissionDecision{ - reason: contributorAdmissionReasonOpenPRLinked, + reason: reason, openPR: link, } } @@ -324,8 +334,10 @@ func mergedReferenceClaim(c ghpkg.IssueClaim) bool { // openPRLinkingIssue asks the Dependencies.OpenPRLinkingIssue hook whether an // open PR already links the candidate (hivecommons/hive#14799). Repo // spellings are tried in the same order and for the same reason as -// issueClaimedByOpenPR. No hook wired (tests, or a hive booted without -// GitHub credentials) reports no link and selection proceeds as before. +// issueClaimedByOpenPR; a rate-limit deferral on the first spelling answers +// for both (one canonical listing backs them). No hook wired (tests, or a +// hive booted without GitHub credentials) reports no link and selection +// proceeds as before. func (h *ContributeWSHub) openPRLinkingIssue(candidate contributorAdmissionCandidate) (ghpkg.OpenPRLink, bool) { if h == nil || h.server == nil || h.server.deps == nil || h.server.deps.OpenPRLinkingIssue == nil { return ghpkg.OpenPRLink{}, false diff --git a/src/pkg/dashboard/contribute_select.go b/src/pkg/dashboard/contribute_select.go index 9dd7216249..b70509c12d 100644 --- a/src/pkg/dashboard/contribute_select.go +++ b/src/pkg/dashboard/contribute_select.go @@ -905,6 +905,9 @@ func (h *ContributeWSHub) selectTaskPass(c *ContributorConnection, skippedUnmint h.logger.Info("[contribute-ws] claim skipped: open PR already linked", "repo", repo.Full, "number", number, "pr", decision.openPR.PRNumber, "pr_url", decision.openPR.PRURL, "pr_author", decision.openPR.PRAuthor) + case contributorAdmissionReasonOpenPRGuardRateLimited: + // Not logged per item: the client warns once per repo + // per rate-limit window, and every candidate shares it. case contributorAdmissionReasonIssueClaim: // #8380: someone claimed the issue on the issue itself; the // claim expires on its own, so the log names when. diff --git a/src/pkg/dashboard/contribute_withheld.go b/src/pkg/dashboard/contribute_withheld.go index b85f78021d..677654f120 100644 --- a/src/pkg/dashboard/contribute_withheld.go +++ b/src/pkg/dashboard/contribute_withheld.go @@ -71,25 +71,26 @@ const ( // has to carry a second copy of this vocabulary (the #6902 "one admission path" // rule applies to the WORDS as much as to the decisions). var withheldReasonLabels = map[string]string{ - contributorAdmissionReasonOpenPRClaim: "An open pull request already claims this issue", - contributorAdmissionReasonOpenPRLinked: "An open pull request already links this issue (claim skipped)", - contributorAdmissionReasonIssueClaim: "Someone has claimed this issue; the claim expires on its own", - contributorAdmissionReasonMergedClaimStale: "Fixed by a merged pull request; the issue is still open — close it or say what remains", - contributorAdmissionReasonIssueChurn: "Too many pull requests on one issue — needs maintainer triage", - contributorAdmissionReasonWorkflowBlocked: "Workflow label: blocked", - contributorAdmissionReasonLabelSkipped: "Contribute skip label", - contributorAdmissionReasonDependencyBlocked: "A dependency is still open", - contributorAdmissionReasonDependencyUnknown: "A dependency could not be resolved", - withheldReasonDisabledRepo: "Repository is disabled for contribution", - withheldReasonRepoUnmintable: "Repository excluded after a GitHub App token mint failure", - withheldReasonTracker: "Tracker / umbrella issue", - withheldReasonCooldown: "Completion cooldown", - withheldReasonNoWorkNeeded: "Previous no_work_needed verdict", - completionVerdictNeedsDecision: "Waiting on a maintainer decision", - withheldReasonFailureCooldown: "Failure cooldown", - withheldReasonInFlight: "Already in flight", - withheldReasonContributorFilter: "Rejected by a contributor filter", - withheldReasonAssignedToOther: "Assigned to another contributor", + contributorAdmissionReasonOpenPRClaim: "An open pull request already claims this issue", + contributorAdmissionReasonOpenPRLinked: "An open pull request already links this issue (claim skipped)", + contributorAdmissionReasonOpenPRGuardRateLimited: "GitHub rate limit: the open-PR check is deferred until it resets", + contributorAdmissionReasonIssueClaim: "Someone has claimed this issue; the claim expires on its own", + contributorAdmissionReasonMergedClaimStale: "Fixed by a merged pull request; the issue is still open — close it or say what remains", + contributorAdmissionReasonIssueChurn: "Too many pull requests on one issue — needs maintainer triage", + contributorAdmissionReasonWorkflowBlocked: "Workflow label: blocked", + contributorAdmissionReasonLabelSkipped: "Contribute skip label", + contributorAdmissionReasonDependencyBlocked: "A dependency is still open", + contributorAdmissionReasonDependencyUnknown: "A dependency could not be resolved", + withheldReasonDisabledRepo: "Repository is disabled for contribution", + withheldReasonRepoUnmintable: "Repository excluded after a GitHub App token mint failure", + withheldReasonTracker: "Tracker / umbrella issue", + withheldReasonCooldown: "Completion cooldown", + withheldReasonNoWorkNeeded: "Previous no_work_needed verdict", + completionVerdictNeedsDecision: "Waiting on a maintainer decision", + withheldReasonFailureCooldown: "Failure cooldown", + withheldReasonInFlight: "Already in flight", + withheldReasonContributorFilter: "Rejected by a contributor filter", + withheldReasonAssignedToOther: "Assigned to another contributor", } // withheldReasonLabel returns the operator-facing prose for a reason code, @@ -112,6 +113,7 @@ func isConvergenceWithheldReason(reason string) bool { switch reason { case contributorAdmissionReasonOpenPRClaim, contributorAdmissionReasonOpenPRLinked, + contributorAdmissionReasonOpenPRGuardRateLimited, contributorAdmissionReasonIssueClaim, contributorAdmissionReasonMergedClaimStale, contributorAdmissionReasonIssueChurn, @@ -341,6 +343,10 @@ func withheldFromAdmissionDecision(c withheldCandidate, d contributorAdmissionDe item.Detail = fmt.Sprintf("Claim skipped: open pull request #%d already links this issue", d.openPR.PRNumber) } return item + case contributorAdmissionReasonOpenPRGuardRateLimited: + item := newWithheldItem(c, d.reason) + item.Detail = fmt.Sprintf("GitHub rate-limited the open-PR listing; dispatch deferred until %s", formatRunTime(d.openPR.RateLimitedUntil)) + return item case contributorAdmissionReasonMergedClaimStale: item := newWithheldItem(c, d.reason) item.ClaimURL = d.claim.PRURL diff --git a/src/pkg/dashboard/contribute_withheld_test.go b/src/pkg/dashboard/contribute_withheld_test.go index 744d6118b4..4722ca9873 100644 --- a/src/pkg/dashboard/contribute_withheld_test.go +++ b/src/pkg/dashboard/contribute_withheld_test.go @@ -239,6 +239,26 @@ func withheldCases() []withheldCase { } }, }, + { + // A rate-limited open-PR listing cannot rule out a linking PR, + // so the candidate is deferred to the reset, not dispatched. + name: "open-PR guard rate-limited", + reason: contributorAdmissionReasonOpenPRGuardRateLimited, + arrange: func(t *testing.T, hub *ContributeWSHub, s *Server) { + s.deps.IssueClaimed = func(string, int) (ghpkg.IssueClaim, bool) { return ghpkg.IssueClaim{}, false } + s.deps.OpenPRLinkingIssue = func(repo string, number int, labels []string) (ghpkg.OpenPRLink, bool) { + if number == 601 { + return ghpkg.OpenPRLink{Issue: 601, RateLimitedUntil: time.Now().Add(30 * time.Minute)}, true + } + return ghpkg.OpenPRLink{}, false + } + }, + evidence: func(t *testing.T, item AdmissionWithheldItem) { + if item.ClaimURL != "" || !strings.Contains(item.Detail, "deferred until") { + t.Errorf("claim_url = %q detail = %q; want no PR and the reset time", item.ClaimURL, item.Detail) + } + }, + }, { // #8003: chairlift#55 — fixed by merged #133 four days ago, still // open, re-offered every 72h. Withheld under its own reason that diff --git a/src/pkg/dashboard/deps.go b/src/pkg/dashboard/deps.go index 1b2c8b5503..280e700117 100644 --- a/src/pkg/dashboard/deps.go +++ b/src/pkg/dashboard/deps.go @@ -220,7 +220,9 @@ type Dependencies struct { // refreshed by the eval cycle, so a PR a person opened by hand stayed // invisible to the contribute lane until the next scan and the issue was // dispatched again in that window. labels are the issue's current labels - // so the label write is idempotent. Nil disables the live check; the + // so the label write is idempotent. ok with link.RateLimited() means + // GitHub rate-limited the listing and the candidate is deferred until + // link.RateLimitedUntil. Nil disables the live check; the // client honours the scanner open-PR switch itself. OpenPRLinkingIssue func(repo string, number int, labels []string) (ghpkg.OpenPRLink, bool) diff --git a/src/pkg/github/issue_claim_gate.go b/src/pkg/github/issue_claim_gate.go index 5a91b19480..6545249f8d 100644 --- a/src/pkg/github/issue_claim_gate.go +++ b/src/pkg/github/issue_claim_gate.go @@ -178,7 +178,8 @@ func (c *Client) issueClaimGate(ctx context.Context, repo string, number int) (i // using IssueClaimBlockReason: the agent whose own PR links the issue holds // its claim legitimately, and a guard that released it would un-stage the // very PR it exists to protect. Off with scanner.skip_issues_with_open_pr: -// false; a failed listing fails open with a warning. +// false; a failed listing fails open with a warning, except a rate-limited +// one, which refuses with ClaimBlockReasonOpenPRRateLimited until the reset. func (c *Client) IssueDispatchBlockReason(ctx context.Context, repo string, number int) (string, error) { gate, err := c.issueClaimGate(ctx, repo, number) if err != nil || gate.reason != "" { diff --git a/src/pkg/github/open_pr_links.go b/src/pkg/github/open_pr_links.go index e61f20e21d..effc67c6bd 100644 --- a/src/pkg/github/open_pr_links.go +++ b/src/pkg/github/open_pr_links.go @@ -2,6 +2,7 @@ package github import ( "context" + "errors" "fmt" "log/slog" "sort" @@ -36,15 +37,55 @@ const ( // issue an open PR already links: "open PR already linked: #14775". ClaimBlockReasonOpenPRPrefix = "open PR already linked: #" + // ClaimBlockReasonOpenPRRateLimited is the IssueDispatchBlockReason while + // GitHub rate-limits the guard's open-PR listing: the guard cannot tell + // whether a PR already links the issue, so the dispatch is deferred until + // the limit resets rather than admitted blind. + ClaimBlockReasonOpenPRRateLimited = "open-PR guard deferred: GitHub rate limit" + // openPRLinkCacheTTL bounds how long one repo's open-PR listing answers // the guard. A sweep (contributor selectTask pass, eval-cycle claim // burst) finishes well inside it, so the listing is paid once per sweep; // a PR opened after the listing is seen on the next one. openPRLinkCacheTTL = time.Minute + + // openPRLinkListTimeout bounds one full open-PR listing for the guard. + openPRLinkListTimeout = 45 * time.Second + // openPRLinkFailureTTL is how long a failed listing answers the guard + // for its repo: a surge of claims fails open at once instead of each one + // re-running a listing that just timed out and re-logging the warning. + // It is also the backoff for a rate limit whose reset time is unknown. + openPRLinkFailureTTL = time.Minute + // openPRLinkRateLimitMaxBackoff caps how long a rate-limited listing + // answers the guard; GitHub's primary window resets within the hour. + openPRLinkRateLimitMaxBackoff = time.Hour ) +// errOpenPRListingBackedOff marks an error replayed from the failure cache. +var errOpenPRListingBackedOff = errors.New("open-PR listing failed recently") + +type openPRLinkFailure struct { + err error + until time.Time +} + +// openPRListingRateLimitedError is a listing GitHub refused for its primary +// or secondary rate limit. until is when the guard lists again: the limit's +// reset (Rate.Reset / Retry-After), or openPRLinkFailureTTL when unknown. +type openPRListingRateLimitedError struct { + until time.Time + err error +} + +func (e *openPRListingRateLimitedError) Error() string { + return fmt.Sprintf("open-PR listing rate-limited until %s: %v", e.until.UTC().Format(time.RFC3339), e.err) +} + +func (e *openPRListingRateLimitedError) Unwrap() error { return e.err } + // OpenPRLink is one open pull request that links an issue with a closing -// keyword, as the guard found it. +// keyword, as the guard found it — or, when RateLimitedUntil is set, the +// guard's refusal for an issue it could not check (no PR fields are set). type OpenPRLink struct { // Repo is the issue's repository in the configured spelling. Repo string @@ -58,8 +99,15 @@ type OpenPRLink struct { // CreatedAt is when the PR was opened; with several open PRs linking // one issue the earliest is reported. CreatedAt time.Time + // RateLimitedUntil is non-zero when GitHub rate-limited the open-PR + // listing: no PR is known to link the issue, but the guard refuses the + // dispatch until this time rather than admitting it blind. + RateLimitedUntil time.Time } +// RateLimited reports whether l is a rate-limit deferral rather than a PR. +func (l OpenPRLink) RateLimited() bool { return !l.RateLimitedUntil.IsZero() } + // openPRLinkIndex is one repo's open PRs indexed by the issue they close. type openPRLinkIndex struct { fetchedAt time.Time @@ -75,6 +123,9 @@ type openPRLinkIndex struct { type openPRLinkCache struct { mu sync.Mutex repos map[string]*openPRLinkIndex + // failures holds each repo's last listing failure until its until: + // openPRLinkFailureTTL, or a rate limit's reset. + failures map[string]openPRLinkFailure // disabled is scanner.skip_issues_with_open_pr: false. disabled bool now func() time.Time @@ -82,7 +133,7 @@ type openPRLinkCache struct { func (c *Client) openPRLinks() *openPRLinkCache { c.openPRLinkOnce.Do(func() { - c.openPRLinkState = &openPRLinkCache{repos: map[string]*openPRLinkIndex{}, now: time.Now} + c.openPRLinkState = &openPRLinkCache{repos: map[string]*openPRLinkIndex{}, failures: map[string]openPRLinkFailure{}, now: time.Now} }) return c.openPRLinkState } @@ -122,9 +173,11 @@ func (c *Client) InvalidateOpenPRLinks(repo string) { defer cache.mu.Unlock() if repo == "" { cache.repos = map[string]*openPRLinkIndex{} + cache.failures = map[string]openPRLinkFailure{} return } delete(cache.repos, c.canonicalRepo(repo)) + delete(cache.failures, c.canonicalRepo(repo)) } // canonicalRepo maps repo onto the configured spelling, so a caller naming @@ -134,6 +187,12 @@ func (c *Client) canonicalRepo(repo string) string { } // openPRIndexFor returns the (possibly cached) open-PR link index for repo. +// A failed listing is remembered per canonical repo and replayed without a +// GitHub call until it expires: openPRLinkFailureTTL for an ordinary +// failure, the limit's reset for a rate limit (wrapped in +// openPRListingRateLimitedError). go-github answers every call made under +// an exhausted limit with a local RateLimitError, so without the replay each +// claim candidate re-ran the listing and re-logged the failure. func (c *Client) openPRIndexFor(ctx context.Context, repo string) (*openPRLinkIndex, error) { repo = c.canonicalRepo(repo) cache := c.openPRLinks() @@ -143,11 +202,26 @@ func (c *Client) openPRIndexFor(ctx context.Context, repo string) (*openPRLinkIn cache.mu.Unlock() return idx, nil } + if f, ok := cache.failures[repo]; ok && now.Before(f.until) { + cache.mu.Unlock() + return nil, fmt.Errorf("%w: %w", errOpenPRListingBackedOff, f.err) + } cache.mu.Unlock() owner, repoName := c.splitRepo(repo) - prs, err := c.listOpenPRsForSupersessionSweep(WithRESTCaller(ctx, "hive:open_pr_links"), owner, repoName) + listCtx, cancel := context.WithTimeout(WithRESTCaller(ctx, "hive:open_pr_links"), openPRLinkListTimeout) + defer cancel() + prs, err := c.listOpenPRsForSupersessionSweep(listCtx, owner, repoName) if err != nil { + cache.mu.Lock() + failedAt := cache.now() + until := failedAt.Add(openPRLinkFailureTTL) + if isRateLimitErr(err) { + until = failedAt.Add(retryableGitHubDelay(err, failedAt, openPRLinkFailureTTL, openPRLinkFailureTTL, openPRLinkRateLimitMaxBackoff)) + err = &openPRListingRateLimitedError{until: until, err: err} + } + cache.failures[repo] = openPRLinkFailure{err: err, until: until} + cache.mu.Unlock() return nil, err } identity := c.getHiveIdentity() @@ -157,6 +231,7 @@ func (c *Client) openPRIndexFor(ctx context.Context, repo string) (*openPRLinkIn idx := buildOpenPRLinkIndex(prs, repo, identity, now) cache.mu.Lock() cache.repos[repo] = idx + delete(cache.failures, repo) cache.mu.Unlock() return idx, nil } @@ -221,19 +296,45 @@ func (c *Client) OpenPRLinkingIssue(ctx context.Context, repo string, number int // the issue HasOpenPRLabel when it does not carry it yet (label created on // first use) and logs `claim skipped: open PR already linked`. ok is false // when the guard is off (scanner.skip_issues_with_open_pr: false), no open -// PR links the issue, or the listing failed — a failure fails OPEN with a -// warning: the ledger-backed guards downstream still stand, and a flaky -// listing must not stall every claim in the hive. labels are the issue's -// current labels as the caller read them (nil when unknown). +// PR links the issue, or the listing failed for any reason but a rate +// limit — such a failure fails OPEN with one warning per repo per backoff +// window: the ledger-backed guards downstream still stand, and a flaky +// listing must not stall every claim in the hive. +// +// A rate-limited listing fails CLOSED instead: ok is true and the link +// carries only RateLimitedUntil (no PR fields, no label written). While the +// installation's limit is exhausted the guard cannot see a PR that already +// links the issue, and the work a dispatch starts needs the same exhausted +// limit anyway, so deferring to the reset costs nothing; admitting handed +// already-linked issues out again. The limit is warned about once per repo +// per window. labels are the issue's current labels as the caller read them +// (nil when unknown). func (c *Client) OpenPRLinkGuard(ctx context.Context, repo string, number int, labels []string) (OpenPRLink, bool) { if c == nil || !c.SkipIssuesWithOpenPR() { return OpenPRLink{}, false } link, ok, err := c.OpenPRLinkingIssue(ctx, repo, number) if err != nil { - if c.logger != nil { + // A replayed failure was already warned about when it happened. + warn := c.logger != nil && !errors.Is(err, errOpenPRListingBackedOff) + var limited *openPRListingRateLimitedError + if errors.As(err, &limited) { + if warn { + c.logger.Warn("open-PR guard: open-PR listing rate-limited, deferring claims until the limit resets", + slog.String("repo", repo), slog.Int("issue", number), + slog.String("reason", "open-pr-listing-rate-limited"), + slog.Time("retry_at", limited.until), slog.String("error", err.Error())) + } + return OpenPRLink{Repo: c.canonicalRepo(repo), Issue: number, RateLimitedUntil: limited.until}, true + } + if warn { + reason := "open-pr-listing-failed" + if errors.Is(err, context.DeadlineExceeded) { + reason = "open-pr-listing-timeout" + } c.logger.Warn("open-PR guard: could not list open PRs, admitting the claim on ledger evidence alone", - slog.String("repo", repo), slog.Int("issue", number), slog.String("error", err.Error())) + slog.String("repo", repo), slog.Int("issue", number), + slog.String("reason", reason), slog.String("error", err.Error())) } return OpenPRLink{}, false } @@ -250,12 +351,17 @@ func (c *Client) OpenPRLinkGuard(ctx context.Context, repo string, number int, l } // openPRLinkBlockReason is the IssueClaimBlockReason step for the guard: -// the refusal reason for an issue an open PR already links, "" otherwise. +// the refusal reason for an issue an open PR already links, or +// ClaimBlockReasonOpenPRRateLimited while the listing is rate-limited; "" +// otherwise. func (c *Client) openPRLinkBlockReason(ctx context.Context, repo string, number int, labels []string) string { link, ok := c.OpenPRLinkGuard(ctx, repo, number, labels) if !ok { return "" } + if link.RateLimited() { + return ClaimBlockReasonOpenPRRateLimited + } return fmt.Sprintf("%s%d", ClaimBlockReasonOpenPRPrefix, link.PRNumber) } diff --git a/src/pkg/github/open_pr_links_test.go b/src/pkg/github/open_pr_links_test.go index eaa7304cfe..6d587eb990 100644 --- a/src/pkg/github/open_pr_links_test.go +++ b/src/pkg/github/open_pr_links_test.go @@ -1,12 +1,15 @@ package github import ( + "bytes" "context" "encoding/json" "fmt" "io" + "log/slog" "net/http" "net/http/httptest" + "strconv" "strings" "sync" "testing" @@ -21,6 +24,12 @@ type openPRGuardServer struct { addedLabels []string createdLbl []string failPulls bool + // rateLimitReset, when set, answers the pulls listing as GitHub does + // once the installation's primary limit is spent (403, remaining 0). + rateLimitReset time.Time + // secondaryRetryAfter, when positive, answers it with a secondary + // rate limit carrying Retry-After in seconds. + secondaryRetryAfter int } func newOpenPRGuardServer(t *testing.T, prs []map[string]any, issueLabels []string) (*httptest.Server, *openPRGuardServer) { @@ -35,11 +44,27 @@ func newOpenPRGuardServer(t *testing.T, prs []map[string]any, issueLabels []stri obs.mu.Lock() obs.pullsCalls++ fail := obs.failPulls + reset := obs.rateLimitReset + retryAfter := obs.secondaryRetryAfter obs.mu.Unlock() if fail { w.WriteHeader(http.StatusBadGateway) return } + if !reset.IsZero() { + w.Header().Set("X-RateLimit-Limit", "6450") + w.Header().Set("X-RateLimit-Remaining", "0") + w.Header().Set("X-RateLimit-Reset", strconv.FormatInt(reset.Unix(), 10)) + w.WriteHeader(http.StatusForbidden) + _, _ = io.WriteString(w, `{"message":"API rate limit of 6450 exceeded for installation ID 1."}`) + return + } + if retryAfter > 0 { + w.Header().Set("Retry-After", strconv.Itoa(retryAfter)) + w.WriteHeader(http.StatusForbidden) + _, _ = io.WriteString(w, `{"message":"You have exceeded a secondary rate limit.","documentation_url":"https://docs.github.com/rest/overview/rate-limits-for-the-rest-api#about-secondary-rate-limits"}`) + return + } _ = json.NewEncoder(w).Encode(prs) }) mux.HandleFunc("/repos/o/r/issues/42", func(w http.ResponseWriter, r *http.Request) { @@ -209,6 +234,133 @@ func TestOpenPRGuard_ListingFailureFailsOpen(t *testing.T) { } } +func TestOpenPRGuard_ListingFailureIsNotRetriedWithinWindow(t *testing.T) { + srv, obs := newOpenPRGuardServer(t, nil, nil) + defer srv.Close() + obs.failPulls = true + c := newTestClient(t, srv, "o", []string{"r"}) + if _, ok := c.OpenPRLinkGuard(context.Background(), "o/r", 42, nil); ok { + t.Fatal("guard refused; want fail-open") + } + calls := obs.pullsCalls + if calls == 0 { + t.Fatal("first claim did not attempt a listing") + } + for i := 0; i < 5; i++ { + if _, ok := c.OpenPRLinkGuard(context.Background(), "o/r", 42, nil); ok { + t.Fatal("guard refused; want fail-open") + } + } + if obs.pullsCalls != calls { + t.Fatalf("listing attempts = %d, want %d: failure must be replayed within the window", obs.pullsCalls, calls) + } +} + +// rateLimitedGuardClient is a client whose pulls listing is rate-limited +// until reset, on a fake clock starting at t0, with PR 14775 open and +// closing #42 — the PR a rate-limited guard cannot see. +func rateLimitedGuardClient(t *testing.T, t0, reset time.Time) (*Client, *openPRGuardServer, *time.Time) { + t.Helper() + srv, obs := newOpenPRGuardServer(t, []map[string]any{ + openPRFixture(14775, "Fixes #42", "clubanderson", t0.Add(-time.Hour)), + }, nil) + t.Cleanup(srv.Close) + // The 403s below engage the process-wide pacer; keep it out of other tests. + ResetRateLimitPacingForTest() + t.Cleanup(ResetRateLimitPacingForTest) + obs.rateLimitReset = reset + c := newTestClient(t, srv, "o", []string{"r"}) + clock := t0 + c.openPRLinks().now = func() time.Time { return clock } + return c, obs, &clock +} + +// Production: an exhausted installation limit made go-github fail every +// listing locally, and each claim candidate re-listed and re-warned (47k +// warnings an hour). The failure must be replayed until the reset — past +// the one-minute failure TTL, across both repo spellings — and warned once. +func TestOpenPRGuard_RateLimitedListingIsNotRetriedUntilReset(t *testing.T) { + t0 := time.Date(2026, 10, 27, 1, 0, 0, 0, time.UTC) + reset := t0.Add(30 * time.Minute) + c, obs, clock := rateLimitedGuardClient(t, t0, reset) + var logs bytes.Buffer + c.logger = slog.New(slog.NewTextHandler(&logs, &slog.HandlerOptions{Level: slog.LevelWarn})) + for i := range 20 { + *clock = t0.Add(time.Duration(i) * time.Minute) + repo := "o/r" + if i%2 == 1 { + repo = "r" + } + c.OpenPRLinkGuard(context.Background(), repo, 42, nil) + } + if obs.pullsCalls != 1 { + t.Fatalf("listing attempts = %d, want 1 until the rate limit resets", obs.pullsCalls) + } + if n := strings.Count(logs.String(), "open-PR guard"); n != 1 { + t.Fatalf("guard warnings = %d, want 1 per repo per window:\n%s", n, logs.String()) + } +} + +// While rate-limited the guard cannot see PR 14775, so it must refuse the +// claim (fail closed) instead of admitting an already-linked issue — and +// write no has-open-pr label, since no PR is known. +func TestOpenPRGuard_RateLimitedListingRefusesClaim(t *testing.T) { + t0 := time.Date(2026, 10, 27, 1, 0, 0, 0, time.UTC) + reset := t0.Add(30 * time.Minute) + c, obs, _ := rateLimitedGuardClient(t, t0, reset) + got, err := c.IssueDispatchBlockReason(context.Background(), "o/r", 42) + if err != nil || got != ClaimBlockReasonOpenPRRateLimited { + t.Fatalf("reason = %q, %v; want %q while rate-limited", got, err, ClaimBlockReasonOpenPRRateLimited) + } + link, ok := c.OpenPRLinkGuard(context.Background(), "r", 42, nil) + if !ok || !link.RateLimited() || !link.RateLimitedUntil.Equal(reset) || link.PRNumber != 0 { + t.Fatalf("guard = %+v, %v; want a refusal deferred until %s", link, ok, reset) + } + if len(obs.addedLabels) != 0 || len(obs.createdLbl) != 0 { + t.Fatalf("labels added = %v created = %v; want none for a rate-limit deferral", obs.addedLabels, obs.createdLbl) + } +} + +func TestOpenPRGuard_RateLimitedListingRetriesAfterReset(t *testing.T) { + t0 := time.Date(2026, 10, 27, 1, 0, 0, 0, time.UTC) + reset := t0.Add(30 * time.Minute) + c, obs, clock := rateLimitedGuardClient(t, t0, reset) + if link, ok := c.OpenPRLinkGuard(context.Background(), "o/r", 42, nil); !ok || !link.RateLimited() { + t.Fatalf("guard = %+v, %v; want deferred while rate-limited", link, ok) + } + *clock = reset.Add(-time.Second) + if link, ok := c.OpenPRLinkGuard(context.Background(), "o/r", 42, nil); !ok || !link.RateLimited() || obs.pullsCalls != 1 { + t.Fatalf("guard = %+v, %v after %d listings; want still deferred just before the reset", link, ok, obs.pullsCalls) + } + obs.mu.Lock() + obs.rateLimitReset = time.Time{} + obs.mu.Unlock() + ResetRateLimitPacingForTest() // the reset has passed for the pacer too + *clock = reset.Add(time.Second) + got, err := c.IssueDispatchBlockReason(context.Background(), "o/r", 42) + if want := ClaimBlockReasonOpenPRPrefix + "14775"; err != nil || got != want { + t.Fatalf("reason = %q, %v; want %q once the limit reset", got, err, want) + } + if obs.pullsCalls != 2 { + t.Fatalf("listing attempts = %d, want 2 (one relist after the reset)", obs.pullsCalls) + } +} + +// A secondary limit backs off for its Retry-After, not the failure TTL. +func TestOpenPRGuard_SecondaryRateLimitRefusesForRetryAfter(t *testing.T) { + t0 := time.Date(2026, 10, 27, 1, 0, 0, 0, time.UTC) + c, obs, clock := rateLimitedGuardClient(t, t0, time.Time{}) + obs.secondaryRetryAfter = 120 + link, ok := c.OpenPRLinkGuard(context.Background(), "o/r", 42, nil) + if want := t0.Add(2 * time.Minute); !ok || !link.RateLimitedUntil.Equal(want) { + t.Fatalf("guard = %+v, %v; want deferred until %s", link, ok, want) + } + *clock = t0.Add(90 * time.Second) + if _, ok := c.OpenPRLinkGuard(context.Background(), "o/r", 42, nil); !ok || obs.pullsCalls != 1 { + t.Fatalf("guard ok=%v after %d listings; want deferred without relisting inside Retry-After", ok, obs.pullsCalls) + } +} + func TestOpenPRGuard_InvalidateForcesRelist(t *testing.T) { srv, obs := newOpenPRGuardServer(t, nil, nil) defer srv.Close()