From d43425185073d3eaca8a8d6432afbd80c2810941 Mon Sep 17 00:00:00 2001 From: Danathar Date: Fri, 6 Nov 2026 12:26:15 -0500 Subject: [PATCH] =?UTF-8?q?=F0=9F=90=9B=20fix(dashboard):=20Let=20hive=20d?= =?UTF-8?q?ecide=20preview=20names=20GitHub's=20rate=20limit,=20not=20"hiv?= =?UTF-8?q?e=20unavailable"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The preview gathers the item live from GitHub and answered any failure with 502. On a hosted hive the Ingress swaps every 502 and 503 for its canned "hive unavailable" body, so while the App was out of quota every preview read "Could not preview (hive unavailable)" with the hive up. The plan now maps the gather error: a primary or secondary rate limit (or a request the pacer shed) answers 429 with Retry-After and the time it lifts; any other GitHub answer (404, permission 403), a closed item or a missing snapshot answers 424 with the real message; only a request that never got an answer stays 502. pkg/github gains RateLimitRetryDelay, built on the existing rate-limit classifier and retry-delay reader. The dashboard shows the server's message and, for a 429 without a time, "retry after HH:MMZ" from Retry-After. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BQYnPGFJx81c8JXR3o1sg1 Signed-off-by: Danathar --- .../fixed-delegate-plan-github-rate-limit.md | 1 + src/docs/api-reference/packs-and-acmm.md | 2 +- .../dashboard-sections/activity-monitor.md | 2 +- .../api_repo_item_decide_delegated_plan.go | 2 +- src/pkg/dashboard/delegated_gather_error.go | 66 +++++++ .../dashboard/delegated_gather_error_test.go | 176 ++++++++++++++++++ .../stage_tracker_row_actions_test.go | 1 + .../static-src/js/tracker-actions.js | 17 +- src/pkg/dashboard/static/index.html | 17 +- src/pkg/github/github_retryable.go | 12 ++ .../github/github_retryable_ratelimit_test.go | 41 ++++ 11 files changed, 332 insertions(+), 5 deletions(-) create mode 100644 changelog.d/fixed-delegate-plan-github-rate-limit.md create mode 100644 src/pkg/dashboard/delegated_gather_error.go create mode 100644 src/pkg/dashboard/delegated_gather_error_test.go create mode 100644 src/pkg/github/github_retryable_ratelimit_test.go diff --git a/changelog.d/fixed-delegate-plan-github-rate-limit.md b/changelog.d/fixed-delegate-plan-github-rate-limit.md new file mode 100644 index 0000000000..ee8dcb4c3b --- /dev/null +++ b/changelog.d/fixed-delegate-plan-github-rate-limit.md @@ -0,0 +1 @@ +- "Let hive decide" now says why a preview failed instead of "hive unavailable". While the hive's GitHub App was out of quota, every preview failed with "Could not preview owner/repo#N (hive unavailable)": the server answered 502, and the hosted Ingress replaces every 502 with its own "hive is restarting" body. The preview now answers 429 with a `Retry-After` header and reads "GitHub API rate limit for this hive is exhausted; resets at 17:32Z" (or the secondary limit's retry time); other GitHub refusals (a 404, a permission 403), a closed item or a missing dashboard snapshot answer 424 with the real message. 502 is left for a GitHub that could not be reached at all. diff --git a/src/docs/api-reference/packs-and-acmm.md b/src/docs/api-reference/packs-and-acmm.md index 3b2a9c07a5..360fcf7da3 100644 --- a/src/docs/api-reference/packs-and-acmm.md +++ b/src/docs/api-reference/packs-and-acmm.md @@ -19,7 +19,7 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `POST` | `/api/repos/{owner}/{repo}/items/{number}/hold` | Owner or repo write/maintain/admin | Add or remove a dashboard hold chip for an issue or PR. Body `{"held": true}` adds the hive's exact-match canonical `hive-pause/` label; `{"held": false}` removes exactly the labels causing the hold (`hive-pause/` and/or generic hold labels) and never removes `hive/` provenance. Every toggle writes an audit-log entry; the #8851 upgrade migration writes `/data/hive-hold-migration-.json` with audit-backed, provenance-only, and ambiguous legacy `hive/` classifications. | `pkg/dashboard/api.go:364`, `pkg/dashboard/api_repo_hold.go` | | `POST` | `/api/repos/{owner}/{repo}/items/{number}/decision` | Owner or repo write/maintain/admin, signed-in GitHub user | Record an operator decision from the activity monitor's row menu / selection bar. Body `{"decision": "approve"|"direction"|"close-superseded"|"close-wontfix"|"adopt"|"reroute"|"re-author"|"merge-now"|"queue-merge", "comment": "…", "type": "issue"|"pr", "superseded_by": "#458", "force": true, "to": ""|"any"}` (`force` only with `adopt`, `to` only with `reroute`; an unknown `decision` is a 400). On an issue parked at 045, `approve`/`direction` act exactly as a maintainer's `/hive approve` / `/hive decision ` comment would (labels, `approved-direction`, stage-engine Unparked fact, reply comment naming the operator). On a PR parked at 085 (`type: "pr"`), `approve` posts the `/lgtm` and `/approve` commands as two comments attributed to the operator, records the approval against the head as the human answer to a `requires_human` / decision-requested verdict (the lanes and the stage engine honour it like a trusted merger's Approve review), lifts `needs-human`, clears the sentinel label and acknowledges the bundle's sentinel findings for that head, re-derives the stage and drops the tracker cache — the response adds `released` (`approval-recorded`, `needs-human`, `sentinel-label`, `sentinel-acknowledged`) and `stage`; the merge gate decides from there (no review as the operator, no merge) — and `direction` records the text as a human change request against the head — the `/hive decision` fix lane — on a hive-authored PR, and on a contributor's clanker PR (`author/clanker` / hive attribution markers) where the text is posted to the contributor's hive and the PR moves to `stage/072-returned-to-contributor` (`status: "direction-returned-to-contributor"`); a person's PR, which no hive can carry direction to, answers 409. The close variants post a comment naming the operator (and the superseding PR when `superseded_by` is given) and close the item as not planned. `adopt` and `re-author` require `type: "pr"` and queue adoption via the `adoption/supersede` label (409 unless `governor.labels.stage.adoption` is enabled). `adopt` with `"force": true` is *Adopt now* ([#15038](https://github.com/hivecommons/hive/issues/15038)): it bypasses the author's presence, the grace window and `maintainer_policy`, marks the stage record and nudges the automerge sweep so the adoption engine supersedes the head at once (responds `status: "already-adopted"` when nothing is left to do; 400 on the hive's own PR). `reroute` (`type: "pr"`, `"to": ""|"any"`) hands the PR's follow-up work to another online clanker through the contributor fix queue, posts a `` note and arms a pickup timer that falls back to Adopt now after `governor.reroute.pickup_timeout`; answers `status: "rerouted"`, 409 when no clanker is online or adoption is disabled. On a PR at 090, `queue-merge` (*Approve and let hive merge*, the lead action) records an approval attributed to the operator and applies the automerge label, returning `status: "queued-for-merge"` (on the hive's own PR, which the App cannot review, the queue note is a comment and the operator's approval is recorded as `approve` records it — `released` lists `approval-recorded`, `admitted`); hive's merge sweep merges on its next tick once CI is green and the head has soaked, and that recorded operator approval is the one thing that lets the sweep merge an untrusted-author PR (the merge comment names the operator). `merge-now` (*Merge now as maintainer*, secondary) posts an attributed comment and merges immediately through the hive's GitHub App, returning `status: "merged"`. Both require `type: "pr"` and a verified owner or `merger` role in addition to the route's repo permission check; both refuse closed/merged PRs and the operator's own PR. `merge-now` on a PR whose `mergeable_state` is `dirty`/`behind` instead answers `202` with `status: "rebase-queued"` (approval recorded, rebase kicked, merge follows via the follow-through once the hive's own rebase is green; a head pushed by anyone else fails the follow-through closed; `head-not-pushable` refusal on a fork without maintainer edits). `approve` on such a PR adds `rebase: {attempt, max, exhausted, head, text}` for the rebase episode it opened. Every decision writes an audit-log entry (`repo_item_decision_*`, e.g. `repo_item_decision_adopt_now`, `repo_item_decision_reroute`). | `pkg/dashboard/api.go:365`, `pkg/dashboard/api_repo_item_decision.go` | | `POST` / `GET` | `/api/repos/{owner}/{repo}/items/{number}/decide-delegated` | Same as `…/hold` (owner or repo write), signed-in GitHub user; `decisions.delegate.enabled` (403 when off) | *Let hive decide*: enqueue a delegated-decision job in which the hive decides the parked item itself as the owner's delegate under `pkg/delegatedecision.PolicyText` and records it as a human decision (`/hive decision` comment with the `` marker, park labels off, `triage/accepted`, queue approval on a PR head). Body `{"type": "issue"|"pr", "reason", "caption", "sentinel_reason"}`. `POST` answers 202 with the job (`{id, status: "running"|"done"|"failed", decision: {verdict, rule, text, rationale, actions, keeps_park, ladder?: {decision, reason, ruled_out[]}}, notes, error}`; `ladder` is set on contributor-PR decisions and names the rung that decided — `already-fixed` → `mergeable-as-is` → `rebase-then-merge` → `return-to-contributor` → `contributor-offline` — and why each rung above it did not apply); 409 with the running job while one is in flight; `GET` reads it. Rails: never merges, refuses items wearing `hold`, never grants secrets/permissions (`needs-owner` + `hold` instead), recommends only on hive policy/governance questions (keeps 085). Audited as `repo_item_decision_delegated[_failed]`. See [dashboard-sections.md](../dashboard-sections/activity-monitor.md#let-hive-decide). | `pkg/dashboard/api.go:329`, `pkg/dashboard/api_repo_item_decide_delegated.go` | -| `POST` | `/api/repos/{owner}/{repo}/items/{number}/decide-delegated/plan` | Same as `…/decide-delegated` | Preview for *Let hive decide*: runs the same owner-delegate policy with **no side effects** (nothing written, no job) and answers `{would, rule, acting, title, summary, why, consequences[], ladder, mergeable_as_is, alternatives[], writes: false, prior, notes, saw[], rule_plain, rationale, why_not[]}` — `summary` is one verb-first sentence with no "Hive would:" prefix (*Send #15332 back to its author with one instruction: make CI green, nothing else.*), `why` one short line (*CI is red.*), `consequences` at most two items, `ladder` the rung that decided with its `ruled_out` list, `saw` every fact the rules read as `{label, value, state}` (`state` is `ok`, `blocks` or `info`), `rule_plain` and `rationale` the deciding rule in plain words and its whole reason (`rationale` omitted when it only repeats `why`), `why_not` the ruled-out rungs as `{option, because}`, and `mergeable_as_is` whether an as-is merge is honest (the queue-merge and merge-now alternatives are dropped when it is not). The dialog shows **Cancel** / **Let hive do that**, plus **Approve and let hive merge** (or **Merge now as maintainer** when the hive would approve anyway) only when `mergeable_as_is`; only then does the SPA post `…/decide-delegated` with `chosen` naming the button. See [dashboard-sections.md](../dashboard-sections/activity-monitor.md#let-hive-decide). | `pkg/dashboard/api.go:369`, `pkg/dashboard/api_repo_item_decide_delegated_plan.go` | +| `POST` | `/api/repos/{owner}/{repo}/items/{number}/decide-delegated/plan` | Same as `…/decide-delegated` | Preview for *Let hive decide*: runs the same owner-delegate policy with **no side effects** (nothing written, no job) and answers `{would, rule, acting, title, summary, why, consequences[], ladder, mergeable_as_is, alternatives[], writes: false, prior, notes, saw[], rule_plain, rationale, why_not[]}` — `summary` is one verb-first sentence with no "Hive would:" prefix (*Send #15332 back to its author with one instruction: make CI green, nothing else.*), `why` one short line (*CI is red.*), `consequences` at most two items, `ladder` the rung that decided with its `ruled_out` list, `saw` every fact the rules read as `{label, value, state}` (`state` is `ok`, `blocks` or `info`), `rule_plain` and `rationale` the deciding rule in plain words and its whole reason (`rationale` omitted when it only repeats `why`), `why_not` the ruled-out rungs as `{option, because}`, and `mergeable_as_is` whether an as-is merge is honest (the queue-merge and merge-now alternatives are dropped when it is not). The dialog shows **Cancel** / **Let hive do that**, plus **Approve and let hive merge** (or **Merge now as maintainer** when the hive would approve anyway) only when `mergeable_as_is`; only then does the SPA post `…/decide-delegated` with `chosen` naming the button. A plan that cannot read the item says why, never with 502 or 503 (the hosted Ingress replaces those with its own *hive unavailable* page): 429 with `Retry-After` while GitHub rate-limits the hive (*GitHub API rate limit for this hive is exhausted; resets at 17:32Z*, or the secondary limit's *retry after* time); 424 with the real message for any other GitHub refusal (a 404, a permission 403), a closed item, or no dashboard snapshot yet; 502 only when GitHub could not be reached at all. See [dashboard-sections.md](../dashboard-sections/activity-monitor.md#let-hive-decide). | `pkg/dashboard/api.go:369`, `pkg/dashboard/api_repo_item_decide_delegated_plan.go`, `pkg/dashboard/delegated_gather_error.go` | | `POST` | `/api/repos/{owner}/{repo}/items/{number}/blocker/recheck` | Same as `…/hold` (owner or repo write), signed-in GitHub user | *Re-check permissions* on an issue parked on a capability blocker (#15099): probes the GitHub App installation's permissions; when a non-workflows permission is granted, clears its ledger entries and un-parks the item; workflows blockers require a separate hand-off (`{result: {outcome: "released", …}}`), otherwise `{result: {outcome: "no-effect", reason: "blocker", options: [...]}}`; 404 when the item has no blocker, 409 when the blocker is not a capability gap | `pkg/dashboard/api.go:370` | | `POST` | `/api/repos/{owner}/{repo}/items/{number}/blocker/handoff` | Same as `…/hold`, signed-in GitHub user | With a workflows blocker, verified App permission and capable lane, records the decision, clears the park and routes the issue to that lane; rejects held or ineligible issues | `pkg/dashboard/api.go:371` | | `POST` | `/api/repos/{owner}/{repo}/items/{number}/blocker/close-partial` | Same as `…/hold` (owner or repo write), signed-in GitHub user | Option (c) on a capability blocker (#15099): labels the issue `hive/partially-addressed`, posts the operator's `{"comment"}` naming what remains and closes it; the ledger entry is cleared | `pkg/dashboard/api.go:372` | diff --git a/src/docs/dashboard-sections/activity-monitor.md b/src/docs/dashboard-sections/activity-monitor.md index 8b3e4bd7ca..231c278405 100644 --- a/src/docs/dashboard-sections/activity-monitor.md +++ b/src/docs/dashboard-sections/activity-monitor.md @@ -677,7 +677,7 @@ The item leaves 085/045 and proceeds without the owner. The action is audited as #### How it runs -1. The preview opens at once as a skeleton: the title, the item, **Asking the hive…** and placeholder lines (still under *prefers-reduced-motion*). **Cancel** works straight away; **Let hive do that** stays disabled until the plan lands and fills the dialog. Until then the row's *Let hive decide* button and its caption read **Asking the hive…** too, and a second click (button, ⋯ entry or popover) opens nothing. If the plan fails, the dialog says so inline with **Retry** — no toast, no browser alert. +1. The preview opens at once as a skeleton: the title, the item, **Asking the hive…** and placeholder lines (still under *prefers-reduced-motion*). **Cancel** works straight away; **Let hive do that** stays disabled until the plan lands and fills the dialog. Until then the row's *Let hive decide* button and its caption read **Asking the hive…** too, and a second click (button, ⋯ entry or popover) opens nothing. If the plan fails, the dialog says so inline with **Retry** — no toast, no browser alert — and gives the hive's reason: while GitHub rate-limits the hive it reads *Could not preview owner/repo#N (GitHub API rate limit for this hive is exhausted; resets at 17:32Z)*, so wait until then before retrying. *hive unavailable* means the hive itself did not answer (it is restarting or unreachable). 2. Read the one-time explanation and confirmation. **Don't ask again for this action** is optional; reset it in ⚙ → **Confirmations**. 3. The row reads **Hive deciding…** while the job runs. 4. Read the persistent result toast. It says what was decided and what happens next; it stays until the target row changes state. diff --git a/src/pkg/dashboard/api_repo_item_decide_delegated_plan.go b/src/pkg/dashboard/api_repo_item_decide_delegated_plan.go index 4b289494cc..77d87c38ce 100644 --- a/src/pkg/dashboard/api_repo_item_decide_delegated_plan.go +++ b/src/pkg/dashboard/api_repo_item_decide_delegated_plan.go @@ -129,7 +129,7 @@ func (s *Server) handleRepoItemDecideDelegatedPlan(w http.ResponseWriter, r *htt var err error plan, err = s.planDelegatedDecision(ctx, repoFull, number, itemType, mergerRole, body) if err != nil { - jsonError(w, err.Error(), http.StatusBadGateway) + delegatedGatherError(w, err, time.Now()) return } s.delegatedJobs.plans.put(key, plan, time.Now()) diff --git a/src/pkg/dashboard/delegated_gather_error.go b/src/pkg/dashboard/delegated_gather_error.go new file mode 100644 index 0000000000..f4a9b9f1b3 --- /dev/null +++ b/src/pkg/dashboard/delegated_gather_error.go @@ -0,0 +1,66 @@ +package dashboard + +import ( + "context" + "errors" + "math" + "net" + "net/http" + "strconv" + "time" + + gh "github.com/google/go-github/v72/github" + ghpkg "github.com/hivecommons/hive/pkg/github" +) + +// delegatedGatherRetryFallback is the wait named when GitHub refused for +// quota without saying until when: GitHub's own rule for that case is to +// wait at least a minute. +const delegatedGatherRetryFallback = time.Minute + +// delegatedGatherError answers a "Let hive decide" request whose live read +// of the item failed. On a hosted hive the Ingress swaps every 502 and 503 +// for its canned "hive unavailable" body (custom-http-errors, see +// pkg/hub/saas_provision.go), so an error GitHub explained must not use +// them: projectbluefin's previews all read "(hive unavailable)" for half an +// hour while the hive was up and its App was out of quota. +func delegatedGatherError(w http.ResponseWriter, err error, now time.Time) { + status, msg, retry := delegatedGatherErrorStatus(err, now) + if retry > 0 { + w.Header().Set("Retry-After", strconv.Itoa(int(math.Ceil(retry.Seconds())))) + } + jsonError(w, msg, status) +} + +// delegatedGatherErrorStatus maps a gather error to its answer: a GitHub +// rate limit (primary, secondary, or a request the pacer shed while one is +// in force) is 429 with the wait and the time it lifts; any other answer +// from GitHub (404, a permission 403, …) or a hive-side refusal (the item +// is closed, no snapshot yet) is 424 with the real message; only a request +// that never got an answer (network error, timeout) stays 502. +func delegatedGatherErrorStatus(err error, now time.Time) (int, string, time.Duration) { + if delay, limited := ghpkg.RateLimitRetryDelay(err, now); limited || errors.Is(err, ghpkg.ErrPacingShed) { + if delay <= 0 { + delay = delegatedGatherRetryFallback + } + var primary *gh.RateLimitError + if errors.As(err, &primary) { + return http.StatusTooManyRequests, "GitHub API rate limit for this hive is exhausted; resets at " + now.Add(delay).UTC().Format("15:04Z"), delay + } + // Round up so "retry after" is never early. + at := now.Add(delay + time.Minute - time.Nanosecond).UTC().Truncate(time.Minute).Format("15:04Z") + if !limited { + return http.StatusTooManyRequests, "This hive is pacing its GitHub requests after a rate limit; retry after " + at, delay + } + return http.StatusTooManyRequests, "GitHub secondary rate limit reached for this hive; retry after " + at, delay + } + var ghErr *gh.ErrorResponse + if errors.As(err, &ghErr) { + return http.StatusFailedDependency, err.Error(), 0 + } + var netErr net.Error + if errors.As(err, &netErr) || errors.Is(err, context.DeadlineExceeded) { + return http.StatusBadGateway, err.Error(), 0 + } + return http.StatusFailedDependency, err.Error(), 0 +} diff --git a/src/pkg/dashboard/delegated_gather_error_test.go b/src/pkg/dashboard/delegated_gather_error_test.go new file mode 100644 index 0000000000..2f18f19ac5 --- /dev/null +++ b/src/pkg/dashboard/delegated_gather_error_test.go @@ -0,0 +1,176 @@ +package dashboard + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "net/url" + "strconv" + "strings" + "testing" + "time" + + gh "github.com/google/go-github/v72/github" + "github.com/hivecommons/hive/pkg/github" +) + +// projectbluefin, 2026-11-06: the App was out of quota for half an hour and +// every "Let hive decide" preview read "(hive unavailable)": the plan +// answered 502, which the hosted Ingress swaps for its canned body. The +// plan now answers 429 with Retry-After and when to retry, for GitHub's +// primary and secondary limits alike. +func TestDelegatedDecisionPlanRateLimitedAnswers429WithReset(t *testing.T) { + // A rate-limit 403 engages the shared pacer; release it so the rest of + // the package is not paced. X-RateLimit-Limit 60 keeps the shared + // core-bucket snapshot unmoved (an unauthenticated bucket). + t.Cleanup(github.ResetRateLimitPacingForTest) + t.Cleanup(github.ResetRateLimitBudgetForTest) + reset := time.Now().Add(21 * time.Minute).Truncate(time.Second) + for _, tc := range []struct { + name string + refuse func(w http.ResponseWriter) + minS int + maxS int + wantError string + }{ + {"primary", func(w http.ResponseWriter) { + w.Header().Set("X-RateLimit-Limit", "60") + w.Header().Set("X-RateLimit-Remaining", "0") + w.Header().Set("X-RateLimit-Reset", strconv.FormatInt(reset.Unix(), 10)) + w.WriteHeader(http.StatusForbidden) + _, _ = io.WriteString(w, `{"message":"API rate limit exceeded for installation ID 1."}`) + }, 19 * 60, 22 * 60, "GitHub API rate limit for this hive is exhausted; resets at " + reset.UTC().Format("15:04Z")}, + {"secondary", func(w http.ResponseWriter) { + w.Header().Set("Retry-After", "90") + w.WriteHeader(http.StatusForbidden) + _, _ = io.WriteString(w, `{"message":"You have exceeded a secondary rate limit.","documentation_url":"https://docs.github.com/rest/overview/rate-limits-for-the-rest-api#about-secondary-rate-limits"}`) + }, 85, 90, "GitHub secondary rate limit reached for this hive; retry after "}, + } { + t.Run(tc.name, func(t *testing.T) { + github.ResetRateLimitPacingForTest() + ghSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + if strings.HasSuffix(r.URL.Path, "/collaborators/writer/permission") { + _, _ = io.WriteString(w, `{"permission":"write"}`) + return + } + tc.refuse(w) + })) + t.Cleanup(ghSrv.Close) + srv := newDelegatedServer(t, &repoItemDecisionMock{}) + srv.deps.GHClient = github.NewClient("token", "testorg", []string{"testrepo"}, slog.Default(), ghSrv.URL) + srv.deps.GHClient.SetAppBotLogin("hive[bot]") + srv.deps.GHClient.SetPRDetailTTLFunc(func() time.Duration { return 0 }) + + w := postDelegatedPlan(t, srv, 8, "writer", `{"type":"pr"}`, nil) + if w.Code == http.StatusBadGateway || w.Code == http.StatusServiceUnavailable { + t.Fatalf("code=%d body=%s: the Ingress rewrites this to \"hive unavailable\"", w.Code, w.Body.String()) + } + if w.Code != http.StatusTooManyRequests { + t.Fatalf("code=%d body=%s, want 429", w.Code, w.Body.String()) + } + secs, err := strconv.Atoi(w.Header().Get("Retry-After")) + if err != nil || secs < tc.minS || secs > tc.maxS { + t.Fatalf("Retry-After = %q, want %d..%d s", w.Header().Get("Retry-After"), tc.minS, tc.maxS) + } + var body struct { + Error string `json:"error"` + } + if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil { + t.Fatal(err) + } + if !strings.HasPrefix(body.Error, tc.wantError) { + t.Fatalf("error = %q, want it to start %q", body.Error, tc.wantError) + } + }) + } +} + +// 502 and 503 are the Ingress's "hive unavailable": no GitHub refusal may +// use them, and each answer carries the real reason. +func TestDelegatedGatherErrorNeverAnswers502ForGitHubRefusals(t *testing.T) { + now := time.Date(2026, 11, 6, 17, 10, 7, 0, time.UTC) + reset := time.Date(2026, 11, 6, 17, 32, 6, 0, time.UTC) + retry := 75 * time.Second + resp := func(code int) *http.Response { + return &http.Response{StatusCode: code, Header: http.Header{}, Request: &http.Request{Method: http.MethodGet, URL: &url.URL{Path: "/repos/o/r/issues/1"}}} + } + for _, tc := range []struct { + name string + err error + status int + contains string + retry time.Duration + }{ + {"primary rate limit", fmt.Errorf("reading o/r#1: %w", &gh.RateLimitError{Rate: gh.Rate{Limit: 7000, Reset: gh.Timestamp{Time: reset}}, Response: resp(http.StatusForbidden), Message: "API rate limit of 7000 still exceeded"}), + http.StatusTooManyRequests, "resets at 17:32Z", reset.Sub(now)}, + {"secondary rate limit", fmt.Errorf("reading PR o/r#1: %w", &gh.AbuseRateLimitError{Response: resp(http.StatusForbidden), Message: "You have exceeded a secondary rate limit", RetryAfter: &retry}), + http.StatusTooManyRequests, "secondary rate limit reached for this hive; retry after 17:12Z", retry}, + {"secondary without a time", &gh.AbuseRateLimitError{Response: resp(http.StatusForbidden)}, + http.StatusTooManyRequests, "retry after 17:12Z", delegatedGatherRetryFallback}, + {"pacer shed", fmt.Errorf("reading o/r#1: %w", &url.Error{Op: "Get", URL: "https://api.github.com/repos/o/r/issues/1", Err: github.ErrPacingShed}), + http.StatusTooManyRequests, "pacing its GitHub requests", delegatedGatherRetryFallback}, + {"not found", fmt.Errorf("reading o/r#1: %w", &gh.ErrorResponse{Response: resp(http.StatusNotFound), Message: "Not Found"}), + http.StatusFailedDependency, "404 Not Found", 0}, + {"permission", fmt.Errorf("reading o/r#1: %w", &gh.ErrorResponse{Response: resp(http.StatusForbidden), Message: "Resource not accessible by integration"}), + http.StatusFailedDependency, "Resource not accessible by integration", 0}, + {"no snapshot", errDelegatedNoSnapshot, http.StatusFailedDependency, "no dashboard snapshot yet", 0}, + {"transport", fmt.Errorf("reading o/r#1: %w", &url.Error{Op: "Get", URL: "https://api.github.com/repos/o/r/issues/1", Err: errors.New("connection refused")}), + http.StatusBadGateway, "connection refused", 0}, + } { + t.Run(tc.name, func(t *testing.T) { + status, msg, d := delegatedGatherErrorStatus(tc.err, now) + if status != tc.status || !strings.Contains(msg, tc.contains) || d != tc.retry { + t.Fatalf("got (%d, %q, %v), want (%d, …%q…, %v)", status, msg, d, tc.status, tc.contains, tc.retry) + } + if tc.status != http.StatusBadGateway && (status == http.StatusBadGateway || status == http.StatusServiceUnavailable) { + t.Fatalf("%s answered %d, which the Ingress rewrites to \"hive unavailable\"", tc.name, status) + } + w := httptest.NewRecorder() + delegatedGatherError(w, tc.err, now) + if got := w.Header().Get("Retry-After"); (tc.retry > 0) != (got != "") { + t.Fatalf("Retry-After = %q for wait %v", got, tc.retry) + } + }) + } +} + +// The preview shows the server's reason; a 429 says when to retry, from +// the message or else Retry-After; the Ingress's own 502 page (HTML, not +// JSON) still reads "hive unavailable". +func TestStageTrackerDelegatePlanRateLimitMessage(t *testing.T) { + html := indexHTML(t) + runStageTrackerRowActionsScript(t, stageTrackerRowActionsPreamble(t, html)+stageTrackerConfirmOptOutJS(t, html)+` +(async () => { + const item = { repo: 'projectbluefin/testsuite', kind: 'pr', number: 1026, url: 'u', stage: 85, modifiers: ['needs-decision'] }; + const req = stageTrackerActionRequest(item, 'decide:delegate', {}); + let reply; + global.fetch = async () => reply; + const hdr = h => ({ get: k => (k in h ? h[k] : null) }); + + reply = { ok: false, status: 429, headers: hdr({ 'Retry-After': '1319' }), json: async () => ({ ok: false, error: 'GitHub API rate limit for this hive is exhausted; resets at 17:32Z' }) }; + let pl = await stageTrackerPostDelegatePlan(item, req, false); + assert.strictEqual(pl.error, 'GitHub API rate limit for this hive is exhausted; resets at 17:32Z'); + const err = stageTrackerDelegatePlanErrorContent('projectbluefin/testsuite#1026', [pl]); + assert.ok(err.includes('Could not preview projectbluefin/testsuite#1026 (GitHub API rate limit for this hive is exhausted; resets at 17:32Z)'), err); + assert.ok(!err.includes('hive unavailable'), err); + + // A 429 whose message names no time: Retry-After does. + const now = Date.parse('2026-11-06T17:10:07Z'); + assert.strictEqual(stageTrackerDelegatePlanErrorText(429, { error: 'slow down' }, '90', now), 'slow down; retry after 17:12Z'); + assert.strictEqual(stageTrackerDelegatePlanErrorText(429, {}, null, now), 'GitHub rate limit reached; retry in a minute'); + // The Ingress's canned 502: JSON or its HTML page. + assert.strictEqual(stageTrackerDelegatePlanErrorText(502, { ok: false, error: 'hive unavailable' }, null, now), 'hive unavailable'); + assert.strictEqual(stageTrackerDelegatePlanErrorText(502, {}, null, now), 'hive unavailable'); + reply = { ok: false, status: 502, headers: hdr({}), json: async () => { throw new SyntaxError('Unexpected token <'); } }; + pl = await stageTrackerPostDelegatePlan(item, req, false); + assert.strictEqual(pl.error, 'hive unavailable'); + // A 424 carries GitHub's own words. + assert.strictEqual(stageTrackerDelegatePlanErrorText(424, { error: 'reading o/r#1: 404 Not Found' }, null, now), 'reading o/r#1: 404 Not Found'); +})().catch(e => { console.error(e); process.exit(1); }); +`) +} diff --git a/src/pkg/dashboard/stage_tracker_row_actions_test.go b/src/pkg/dashboard/stage_tracker_row_actions_test.go index 2ad454b870..92523ddf3e 100644 --- a/src/pkg/dashboard/stage_tracker_row_actions_test.go +++ b/src/pkg/dashboard/stage_tracker_row_actions_test.go @@ -887,6 +887,7 @@ func stageTrackerConfirmOptOutJS(t *testing.T, html string) string { jsConstLine(t, html, "STAGE_TRACKER_PLAN_PREFETCH_MS"), jsConstLine(t, html, "_stageTrackerPlanPrefetch"), jsFunc(t, html, "stageTrackerPlanRequestKey"), + jsFunc(t, html, "stageTrackerDelegatePlanErrorText"), jsFunc(t, html, "stageTrackerPostDelegatePlan"), jsFunc(t, html, "stageTrackerPrefetchDelegatePlan"), jsFunc(t, html, "stageTrackerFetchDelegatePlan"), diff --git a/src/pkg/dashboard/static-src/js/tracker-actions.js b/src/pkg/dashboard/static-src/js/tracker-actions.js index 80088bbc04..973e8f868c 100644 --- a/src/pkg/dashboard/static-src/js/tracker-actions.js +++ b/src/pkg/dashboard/static-src/js/tracker-actions.js @@ -1776,13 +1776,28 @@ const STAGE_TRACKER_PLAN_PREFETCH_MS = 30000; const _stageTrackerPlanPrefetch = new Map(); function stageTrackerPlanRequestKey(req) { return req.url + '/plan ' + JSON.stringify(req.body); } + // Pure: why a plan failed, in the server's words. A 429 is GitHub's + // rate limit and says when to retry: the server's message names the + // time, else Retry-After does. 502/503 on a hosted hive are the + // Ingress's own "hive unavailable" page, which may not be JSON. + function stageTrackerDelegatePlanErrorText(status, data, retryAfter, nowMs) { + const msg = data && typeof data.error === 'string' ? data.error.trim() : ''; + if (status === 429 && !/\d\d:\d\dZ/.test(msg)) { + const secs = Number(retryAfter); + const at = secs > 0 ? nowMs + secs * 1000 : Date.parse(retryAfter || ''); + const when = at > 0 ? '; retry after ' + new Date(Math.ceil(at / 60000) * 60000).toISOString().slice(11, 16) + 'Z' : '; retry in a minute'; + return (msg || 'GitHub rate limit reached') + when; + } + if (msg) return msg; + return status === 502 || status === 503 ? 'hive unavailable' : 'HTTP ' + status; + } async function stageTrackerPostDelegatePlan(item, req, prefetch) { try { const headers = { 'Content-Type': 'application/json' }; if (prefetch) headers['X-Hive-Prefetch'] = '1'; const r = await fetch(req.url + '/plan', { method: 'POST', headers, body: JSON.stringify(req.body) }); const data = await r.json().catch(() => ({})); - if (!r.ok) return { ref: stageTrackerRefOf(item), number: item.number, error: data.error || ('HTTP ' + r.status) }; + if (!r.ok) return { ref: stageTrackerRefOf(item), number: item.number, error: stageTrackerDelegatePlanErrorText(r.status, data, r.headers && r.headers.get ? r.headers.get('Retry-After') : '', Date.now()) }; return Object.assign({ ref: stageTrackerRefOf(item) + (typeof stageTrackerLineageClause === 'function' ? stageTrackerLineageClause(item) : '') }, data); } catch (e) { return { ref: stageTrackerRefOf(item), number: item.number, error: String(e && e.message || e) }; } } diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html index 0caa15a8e4..b273297a20 100644 --- a/src/pkg/dashboard/static/index.html +++ b/src/pkg/dashboard/static/index.html @@ -24610,13 +24610,28 @@

All labels

const STAGE_TRACKER_PLAN_PREFETCH_MS = 30000; const _stageTrackerPlanPrefetch = new Map(); function stageTrackerPlanRequestKey(req) { return req.url + '/plan ' + JSON.stringify(req.body); } + // Pure: why a plan failed, in the server's words. A 429 is GitHub's + // rate limit and says when to retry: the server's message names the + // time, else Retry-After does. 502/503 on a hosted hive are the + // Ingress's own "hive unavailable" page, which may not be JSON. + function stageTrackerDelegatePlanErrorText(status, data, retryAfter, nowMs) { + const msg = data && typeof data.error === 'string' ? data.error.trim() : ''; + if (status === 429 && !/\d\d:\d\dZ/.test(msg)) { + const secs = Number(retryAfter); + const at = secs > 0 ? nowMs + secs * 1000 : Date.parse(retryAfter || ''); + const when = at > 0 ? '; retry after ' + new Date(Math.ceil(at / 60000) * 60000).toISOString().slice(11, 16) + 'Z' : '; retry in a minute'; + return (msg || 'GitHub rate limit reached') + when; + } + if (msg) return msg; + return status === 502 || status === 503 ? 'hive unavailable' : 'HTTP ' + status; + } async function stageTrackerPostDelegatePlan(item, req, prefetch) { try { const headers = { 'Content-Type': 'application/json' }; if (prefetch) headers['X-Hive-Prefetch'] = '1'; const r = await fetch(req.url + '/plan', { method: 'POST', headers, body: JSON.stringify(req.body) }); const data = await r.json().catch(() => ({})); - if (!r.ok) return { ref: stageTrackerRefOf(item), number: item.number, error: data.error || ('HTTP ' + r.status) }; + if (!r.ok) return { ref: stageTrackerRefOf(item), number: item.number, error: stageTrackerDelegatePlanErrorText(r.status, data, r.headers && r.headers.get ? r.headers.get('Retry-After') : '', Date.now()) }; return Object.assign({ ref: stageTrackerRefOf(item) + (typeof stageTrackerLineageClause === 'function' ? stageTrackerLineageClause(item) : '') }, data); } catch (e) { return { ref: stageTrackerRefOf(item), number: item.number, error: String(e && e.message || e) }; } } diff --git a/src/pkg/github/github_retryable.go b/src/pkg/github/github_retryable.go index deb83af597..480cad26a7 100644 --- a/src/pkg/github/github_retryable.go +++ b/src/pkg/github/github_retryable.go @@ -17,6 +17,18 @@ import ( // outside the package. func IsRetryableError(err error) bool { return isRetryableGitHubError(err) } +// RateLimitRetryDelay reports whether err is GitHub refusing for quota (a +// primary or secondary rate limit, or a bare 429) and how long until the +// refusal lifts by GitHub's own reset time or Retry-After; the delay is 0 +// when GitHub named neither. +func RateLimitRetryDelay(err error, now time.Time) (time.Duration, bool) { + var ghErr *gh.ErrorResponse + if !isRateLimitErr(err) && (!errors.As(err, &ghErr) || ghErr.Response == nil || ghErr.Response.StatusCode != http.StatusTooManyRequests) { + return 0, false + } + return retryableGitHubDelay(err, now, 0, 0, time.Hour), true +} + // isRetryableGitHubError reports whether err is a transient GitHub/API // transport failure where a guarded mutation should be retried instead of // proceeding with incomplete preflight information. diff --git a/src/pkg/github/github_retryable_ratelimit_test.go b/src/pkg/github/github_retryable_ratelimit_test.go new file mode 100644 index 0000000000..c42f516f52 --- /dev/null +++ b/src/pkg/github/github_retryable_ratelimit_test.go @@ -0,0 +1,41 @@ +package github + +import ( + "fmt" + "net/http" + "net/url" + "testing" + "time" + + gh "github.com/google/go-github/v72/github" +) + +// RateLimitRetryDelay names GitHub's quota refusals and the wait GitHub +// gave; every other error is not a rate limit. +func TestRateLimitRetryDelay(t *testing.T) { + now := time.Date(2026, 11, 6, 17, 10, 7, 0, time.UTC) + resp := func(code int, h http.Header) *http.Response { + return &http.Response{StatusCode: code, Header: h, Request: &http.Request{Method: http.MethodGet, URL: &url.URL{Path: "/x"}}} + } + ninety := 90 * time.Second + for _, tc := range []struct { + name string + err error + wantOK bool + wantDur time.Duration + }{ + {"primary", fmt.Errorf("wrapped: %w", &gh.RateLimitError{Rate: gh.Rate{Reset: gh.Timestamp{Time: now.Add(22 * time.Minute)}}, Response: resp(http.StatusForbidden, http.Header{})}), true, 22 * time.Minute}, + {"secondary", &gh.AbuseRateLimitError{Response: resp(http.StatusForbidden, http.Header{}), RetryAfter: &ninety}, true, ninety}, + {"secondary without a time", &gh.AbuseRateLimitError{Response: resp(http.StatusForbidden, http.Header{})}, true, 0}, + {"bare 429", &gh.ErrorResponse{Response: resp(http.StatusTooManyRequests, http.Header{"Retry-After": {"30"}})}, true, 30 * time.Second}, + {"not found", &gh.ErrorResponse{Response: resp(http.StatusNotFound, http.Header{})}, false, 0}, + {"permission", &gh.ErrorResponse{Response: resp(http.StatusForbidden, http.Header{})}, false, 0}, + } { + t.Run(tc.name, func(t *testing.T) { + d, ok := RateLimitRetryDelay(tc.err, now) + if ok != tc.wantOK || d != tc.wantDur { + t.Fatalf("RateLimitRetryDelay = (%v, %v), want (%v, %v)", d, ok, tc.wantDur, tc.wantOK) + } + }) + } +}