From 8990b59247dbf9fed74d59795c3432a8cda6d78f Mon Sep 17 00:00:00 2001 From: Danathar Date: Sat, 7 Nov 2026 21:11:50 -0500 Subject: [PATCH 1/2] =?UTF-8?q?=F0=9F=90=9B=20fix(webhooks):=20a=20spoke?= =?UTF-8?q?=20without=20a=20webhook=20secret=20says=20so=20in=20status=20a?= =?UTF-8?q?nd=20deep=20health?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A hosted spoke with no GITHUB_WEBHOOK_SECRET answers every relayed delivery with 503 and polls forever, but the only trace was one WARN per process: /api/status.webhooks read healthy:false, mode:polling with no reason, the same as "GitHub isn't sending". - /api/status.webhooks gains mode_reason_code (no_secret, signature_failed, deliveries_rejected, no_deliveries), rejected_1h (fixed per-minute ring, no per-request allocation) and last_rejected_at; mode_reason explains the no-secret case in words. - The no-secret 503 is recorded through RecordWebhookDeliveryFailure, which now takes the reason; the WARN stays once per process. - /api/health/deep and the health summary get a `webhooks` check that warns (never fails, never changes the verdict) once deliveries are being turned away. - Only whether the secret is set is ever reported. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BQYnPGFJx81c8JXR3o1sg1 Signed-off-by: Danathar --- changelog.d/fixed-webhook-no-secret-reason.md | 1 + src/docs/api-reference.md | 2 +- .../api-reference/health-status-events.md | 2 +- src/docs/env-vars.md | 2 +- .../operator-reference/runtime-and-quota.md | 9 +- src/docs/webhooks.md | 34 ++- src/pkg/dashboard/api_github_webhook.go | 11 +- .../api_github_webhook_reason_test.go | 197 ++++++++++++++++++ src/pkg/dashboard/health_webhooks.go | 57 +++++ src/pkg/dashboard/server.go | 20 ++ src/pkg/github/webhook_cache.go | 49 ++++- src/pkg/github/webhook_mode.go | 116 ++++++++++- src/pkg/github/webhook_mode_test.go | 57 ++++- 13 files changed, 524 insertions(+), 33 deletions(-) create mode 100644 changelog.d/fixed-webhook-no-secret-reason.md create mode 100644 src/pkg/dashboard/api_github_webhook_reason_test.go create mode 100644 src/pkg/dashboard/health_webhooks.go diff --git a/changelog.d/fixed-webhook-no-secret-reason.md b/changelog.d/fixed-webhook-no-secret-reason.md new file mode 100644 index 0000000000..6dd15e493c --- /dev/null +++ b/changelog.d/fixed-webhook-no-secret-reason.md @@ -0,0 +1 @@ +- A spoke that rejects webhook deliveries now says why. `/api/status.webhooks` (and `/api/gh-rate-limits`) gain `mode_reason_code` — `no_secret` when `GITHUB_WEBHOOK_SECRET` is unset, `signature_failed` or `deliveries_rejected` when deliveries arrive but fail, `no_deliveries` when the secret is set but nothing verified has arrived — plus `rejected_1h` and `last_rejected_at`; `mode_reason` gives the same cause in plain words. The 503 a spoke without a secret returns is now counted like any other rejected delivery (the log stays at one WARN per process), and `/api/health/deep` plus the dashboard health tooltip carry a `webhooks` warning such as "polling — no GITHUB_WEBHOOK_SECRET on this spoke; relayed deliveries are rejected with 503". Before this, a hosted spoke whose hub never provisioned the secret read `healthy:false, mode:polling` with no reason, the same as "GitHub isn't sending". Only whether the secret is set is reported, never its value. See [webhooks.md](src/docs/webhooks.md). diff --git a/src/docs/api-reference.md b/src/docs/api-reference.md index 800e9784f0..af6c52b7a7 100644 --- a/src/docs/api-reference.md +++ b/src/docs/api-reference.md @@ -34,7 +34,7 @@ History point limits, timestamp cutoffs, incremental cost reads and ETag/304 beh | `GET` | `/api/version/release-notes` | Dashboard auth/session | Changelog gap between two revisions: `?from=&to=` returns `{from:{sha,version?}, to:{sha,ref,version?}, sections:[{version,date,categories:{added,changed,fixed,security,deprecated}}], unreleased?, truncated, source, error?}` — the `CHANGELOG.md` release sections present at `to` and absent at `from`, newest first, plus (when `to` is not a release tag) the `changelog.d/` fragments new at `to` under an `Unreleased (in this build)` section. Fetch failures return `source: "unavailable"` with `error` rather than an error status; results are cached per (from, to) for an hour ([#11065](https://github.com/hivecommons/hive/issues/11065)) | `pkg/dashboard/api.go:46` | | `POST` | `/api/release-channel` | Owner only | Hosted spoke self-service release-channel selector; relays `stable`/`candidate`/`edge` to the hub's existing switch-branch endpoint with the spoke dashboard-token proof, and reports the requested channel as pending until the Deployment image lands | `pkg/dashboard/api.go:76` | | `GET` | `/api/health` | Public | Basic health probe | `pkg/dashboard/server.go:1811` | -| `GET` | `/api/health/deep` | Public | Deep health probe | `pkg/dashboard/server.go:1812` | +| `GET` | `/api/health/deep` | Public | Deep health probe; the `webhooks` check (`status`, `detail`, `mode`, `mode_reason_code`, `rejected_1h`) warns when deliveries arrive but cannot be used — no `GITHUB_WEBHOOK_SECRET`, signature mismatch, undecodable payload — and never changes the overall verdict | `pkg/dashboard/server.go:1812` | | `GET` | `/api/livez` | Public | Kubernetes liveness probe | `pkg/dashboard/server.go:1813` | | `GET` | `/metrics` | Registered only when `HIVE_METRICS_ENABLED`; requires `Authorization: ****** (403 if the token is unset) | Prometheus metrics | `pkg/dashboard/server.go:1819` | | `GET` | `/api/status` | Dashboard auth/session | Dashboard aggregate status; `labelPresets[]` (`{key,label,description,labels[],states[]}`) lists the built-in label-filter presets for the Projects cards, led by `needs-human` whose `labels[]` are the effective hard-suppress labels plus `needs-signal`, the hold labels, `stage/071-needs-author` and the reporter-trust awaiting label (#12172); accepts `?fields=a,b` for top-level selection and `?omit=repos` to drop heavy sections, and honors `Accept-Encoding: gzip`; `agents[]` includes busy-pane visibility fields (`kicksUndeliverable`, `busySince`, `lastTranscriptActivity`, `condition`, `conditionMessage`) when a Working pane cannot accept kicks; `sla` (fleet) and `repos[].sla` (per repo) carry the Delivery SLA block derived from the lifecycle timeline with no extra GitHub calls: `windows.{7d,30d}.{open_to_triage,triage_to_start,start_to_pr,pr_to_merge,open_to_merge}` as `{p50_h,p95_h,n}` (hours, nearest-rank), `history[]` of up to 30 daily buckets `{day,start_to_pr_p50,pr_to_merge_p50,open_to_merge_p50,n_started,n_pr,n_merged}` persisted under `/data/sla-history.json`, `stale_in_progress[]` (`{number,repo,started_at,age_h,agent}` — issues carrying `hive/in-progress` longer than the agent claim TTL with no linked PR), `stale_count`, `merged_7d`, `claim_ttl_s` and, on the fleet block, the effective `targets` (`start_to_pr_h`, `pr_to_merge_h`, `open_to_merge_h`); `staleSince` and `repos[].countsStale` / `countsStaleSince` mark totals served from last-good figures — the previous enumeration (#13382) or the ForgeState cache's last-good counts (#13376); every your-turn issue and PR row carries the critical path (#13646): `waits_on[]` (`{kind,ref}`, kind one of `pr`, `issue`, `external`, `base-check` (ref = the check red on the base branch), `approval`, `merge`; ref `#N` in the same repository or `owner/repo#N`), `unblocks` (transitive count of your-turn items waiting on it, cycle-guarded), `may_conflict_with[]` (PR numbers of the same repository sharing a closing issue or a known changed file; never an edge), `action` (`merge`, `release-hold`, `rebase`, `rerun` or `approve`), `action_ref`, `root_cause` and `blocker` (the root cause it collapses under); `repos[].yourTurn` has `issues`, `prs` (your turn, `CLOSE` excluded), `root_issues`, `root_prs`, `root_cause_count`, `order[]` (`{kind,repo,number}` by unblocks then age), `groups[]` (`{blocker,blocker_title,dependents[]}`) and `main_red` (`{check,since,fix,fix_your_turn}`); the top-level `yourTurn` sums the counts | `pkg/dashboard/server.go:1824` | diff --git a/src/docs/api-reference/health-status-events.md b/src/docs/api-reference/health-status-events.md index bc61f09f3b..e92cfd5633 100644 --- a/src/docs/api-reference/health-status-events.md +++ b/src/docs/api-reference/health-status-events.md @@ -25,7 +25,7 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `GET` | `/api/version/release-notes` | Dashboard auth/session | Changelog gap between two revisions: `?from=&to=` returns `{from:{sha,version?}, to:{sha,ref,version?}, sections:[{version,date,categories:{added,changed,fixed,security,deprecated}}], unreleased?, truncated, source, error?}` — the `CHANGELOG.md` release sections present at `to` and absent at `from`, newest first, plus (when `to` is not a release tag) the `changelog.d/` fragments new at `to` under an `Unreleased (in this build)` section. Fetch failures return `source: "unavailable"` with `error` rather than an error status; results are cached per (from, to) for an hour ([#11065](https://github.com/hivecommons/hive/issues/11065)) | `pkg/dashboard/api.go:46` | | `POST` | `/api/release-channel` | Owner only | Hosted spoke self-service release-channel selector; relays `stable`/`candidate`/`edge` to the hub's existing switch-branch endpoint with the spoke dashboard-token proof, and reports the requested channel as pending until the Deployment image lands | `pkg/dashboard/api.go:76` | | `GET` | `/api/health` | Public | Basic health probe | `pkg/dashboard/server.go:1811` | -| `GET` | `/api/health/deep` | Public | Deep health probe | `pkg/dashboard/server.go:1812` | +| `GET` | `/api/health/deep` | Public | Deep health probe; the `webhooks` check (`status`, `detail`, `mode`, `mode_reason_code`, `rejected_1h`) warns when deliveries arrive but cannot be used — no `GITHUB_WEBHOOK_SECRET`, signature mismatch, undecodable payload — and never changes the overall verdict | `pkg/dashboard/server.go:1812` | | `GET` | `/api/livez` | Public | Kubernetes liveness probe | `pkg/dashboard/server.go:1813` | | `GET` | `/metrics` | Registered only when `HIVE_METRICS_ENABLED`; requires `Authorization: ****** (403 if the token is unset) | Prometheus metrics | `pkg/dashboard/server.go:1819` | | `GET` | `/api/status` | Dashboard auth/session | Dashboard aggregate status; `labelPresets[]` (`{key,label,description,labels[],states[]}`) lists the built-in label-filter presets for the Projects cards, led by `needs-human` whose `labels[]` are the effective hard-suppress labels plus `needs-signal`, the hold labels, `stage/071-needs-author` and the reporter-trust awaiting label (#12172); accepts `?fields=a,b` for top-level selection and `?omit=repos` to drop heavy sections, and honors `Accept-Encoding: gzip`; `agents[]` includes busy-pane visibility fields (`kicksUndeliverable`, `busySince`, `lastTranscriptActivity`, `condition`, `conditionMessage`) when a Working pane cannot accept kicks; a working agent also carries `agents[].run` — the live per-run line (#14576): `{since, elapsedS, costUSD, tokens, priced, tests, filesChanged}`, the estimated spend/tokens since the kick was delivered (`lastKickAt`, falling back to `busySince`) (delta of the token collector's per-agent cumulative figure; `priced:false` when the agent has none yet), the last visible test-runner result (`pass`/`fail`/empty) and the last `N files changed` git summary (`-1` when none) parsed from the captured pane; top-level `fleetRun` `{running, hourUSD, hourPriced, dayUSD, weekUSD, monthUSD, windowsPriced}` is the governor strip's rotating `running N · $X this hour / today / this week / this month` figure (#15614) — the hour is the rise since the top of the clock hour, the wider windows are the daily cost ledger summed from 00:00 UTC today, Monday 00:00 UTC (ISO week) and the 1st 00:00 UTC plus the rise since the ledger's last snapshot, `windowsPriced:false` when there is no ledger yet; `sla` (fleet) and `repos[].sla` (per repo) carry the Delivery SLA block derived from the lifecycle timeline with no extra GitHub calls: `windows.{7d,30d}.{open_to_triage,triage_to_start,start_to_pr,pr_to_merge,open_to_merge}` as `{p50_h,p95_h,n}` (hours, nearest-rank), `history[]` of up to 30 daily buckets `{day,start_to_pr_p50,pr_to_merge_p50,open_to_merge_p50,n_started,n_pr,n_merged}` persisted under `/data/sla-history.json`, `stale_in_progress[]` (`{number,repo,started_at,age_h,agent}` — issues carrying `hive/in-progress` longer than the agent claim TTL with no linked PR), `stale_count`, `merged_7d`, `claim_ttl_s` and, on the fleet block, the effective `targets` (`start_to_pr_h`, `pr_to_merge_h`, `open_to_merge_h`); `staleSince` and `repos[].countsStale` / `countsStaleSince` mark totals served from last-good figures — the previous enumeration (#13382) or the ForgeState cache's last-good counts (#13376); `ci_failure_signals` `{count, last, kicks, unmatched}` is the webhook CI-failure signal store — distinct red (repo, head, check) signals retained, the most recent one `{repo, number, head_sha, check, conclusion, details_url, agent, at}`, webhook-triggered kicks/nudges since boot, and failure deliveries whose head matched no open PR; `workflow_approvals` `{approved, denied, permission_errors, last}` counts the hive's approvals of GitHub-gated workflow runs since boot (#16603) — runs approved, runs refused by the trust policy (once per head), `403`/`404` refusals for want of `actions: write`, and the most recent decision `{repo, number, head, run_id, outcome, at}`; `hotspots` `{count, actionable, filed}` is the merge-hotspot advisor's one-line summary (files at/above the thresholds, those with a split recommendation, those with a modularize issue filed), absent when the advisor is not wired — details on `GET /api/hotspots`; `writeback.dedupe` `{enabled, dedupe_same_item, dedupe_burst, rate_capped, hourly_cap, burst_items, burst_window_s, same_item_ttl_s}` is the write-back gate's content layer — the effective `writeback.dedupe.*` settings and the last-hour counts of writes held as `duplicate`, `burst` and `rate-capped` (each also listed under `GET /api/writeback/suppressed`); `config.gate_regressions[]` `{gate, previous, effective, source}` is the boot-time safety-gate regression list (#16438) — gates a previous runtime overlay on the PVC carried that the running config lacks or loosens (`review.require_labels`, `project.issue_filter.require_labels`, `project.paused_repos[]`, `writeback.per_repo[].mode`), empty on a healthy boot | `pkg/dashboard/server.go:1824` | diff --git a/src/docs/env-vars.md b/src/docs/env-vars.md index 5e4459a679..d55218f370 100644 --- a/src/docs/env-vars.md +++ b/src/docs/env-vars.md @@ -102,7 +102,7 @@ behavior. [Maintenance limitations](#keeping-this-reference-current) still apply | `HIVE_CONTRIBUTE_SKIP_LABELS` | No | `blocked,tracking,epic,discussion,question,needs-decision,needs-triage` | Comma-separated, case-insensitive label patterns for issues that are not contributor work and must never be offered by the relay. Patterns use `path.Match`-style `*` globs; `blocked` is always unioned into the effective set even if omitted. Same setting as `hub.contribute_skip_labels`. | | `HIVE_FEDERATION_REGISTRY_PATH` | No | `/data/federation/registry.json` | Federation registry path override. | | `HIVE_WEBHOOK_SECRET` | No | none | HMAC secret for the spoke `/webhook` channel. | -| `GITHUB_WEBHOOK_SECRET` | No | `/data/saas/webhook-secret.key` when present (hub only) | GitHub App webhook HMAC secret. The hub verifies `/api/github/webhook` with it; a spoke verifies `/api/webhook/github` (direct or hub-relayed deliveries) with the same secret and rejects every delivery while it is unset. | +| `GITHUB_WEBHOOK_SECRET` | No | `/data/saas/webhook-secret.key` when present (hub only) | GitHub App webhook HMAC secret. The hub verifies `/api/github/webhook` with it; a spoke verifies `/api/webhook/github` (direct or hub-relayed deliveries) with the same secret and rejects every delivery while it is unset; such a spoke reports `/api/status.webhooks.mode_reason_code: no_secret` and a `webhooks` warning on `/api/health/deep` once deliveries arrive. | | `HIVE_DASHBOARD_URL` | No | none | Base URL the `hive tui` client targets (`pkg/tui/client`). A bad value surfaces as a request error on the first call, not at startup. On the hub it is also consulted (fourth) in the hub public-origin chain (see `HIVE_HUB_PUBLIC_URL`). | | `HIVE_CONVERGENCE_MODE` | No | `convergence.mode` in `hive.yaml`, else `shadow` | Process-level override of the convergence mode (`off`, `shadow`, `enforce`) so an operator can flip the mode without editing `hive.yaml`. An **unset** mode takes the default, `shadow`, which computes and records admission decisions without withholding any work. A **non-empty but unrecognised** value — a typo, or a mode this build does not know — still resolves to `off`. | | `HIVE_RELEASE_LINE_LAG_MAX` | No | `5` | Alarm threshold for the release-line drift surface ([#6960](https://github.com/hivecommons/hive/issues/6960)): how many commits the hosted edge line (`v5`) may sit behind the stable default branch (`v4`) before the dashboard status flags the line as drifting (`releaseLineLag.exceeded`). An unset, empty, non-numeric, or negative value falls back to the default — it can never silently disable the alarm. An unknown lag (tips unresolved or the compare failed) is reported as `unknown`, never a healthy zero. | diff --git a/src/docs/operator-reference/runtime-and-quota.md b/src/docs/operator-reference/runtime-and-quota.md index a68564f24b..04c69177ba 100644 --- a/src/docs/operator-reference/runtime-and-quota.md +++ b/src/docs/operator-reference/runtime-and-quota.md @@ -556,10 +556,13 @@ periodic housekeeping cannot starve the governor: `/api/status` and `/api/gh-rate-limits` expose `webhooks: {healthy, last_event_at, events_1h, invalidations_1h, -targeted_refreshes_1h, tracker_invalidations_1h, mode, since, mode_reason, +targeted_refreshes_1h, tracker_invalidations_1h, rejected_1h, +last_rejected_at, mode, since, mode_reason, mode_reason_code, healthy_windows}` (the middle two count the single-item re-reads and the -tracker cache drops webhooks caused; `mode` is `webhook` | `polling` and -`since` when it began — see [webhooks.md](../webhooks.md)); +tracker cache drops webhooks caused; `rejected_1h` and `last_rejected_at` +count deliveries the receiver turned away; `mode` is `webhook` | `polling` +and `since` when it began; `mode_reason_code` says why it is polling — see +[webhooks.md](../webhooks.md)); `/api/status` adds `evalMode` (`webhook` | `poll`), `evalIntervalS`, `lastWakeReason`, `lastWakeAt` and `webhookWakes`; `graphql_pr_batch.webhook_skips` counts batch queries skipped because nothing diff --git a/src/docs/webhooks.md b/src/docs/webhooks.md index 163196c375..d25dc814d8 100644 --- a/src/docs/webhooks.md +++ b/src/docs/webhooks.md @@ -132,10 +132,10 @@ Defaults: `src/pkg/config/defaults.go:30-32` (`DefaultEvalIntervalWebhookS`, `DefaultWebhookWakeDebounceS`, `DefaultWebhookWakeMinGapS`), `src/pkg/config/forgestate.go:8` (`DefaultForgeStateReconcileInterval`, `DefaultForgeStateFreshnessBound`, `DefaultForgeStateDriftThreshold`), -`src/pkg/github/webhook_cache.go:14` (`DefaultWebhookHealthWindow`, replaced -by `SetWebhookHealthWindow` at `src/pkg/github/webhook_cache.go:102` once the +`src/pkg/github/webhook_cache.go:17` (`DefaultWebhookHealthWindow`, replaced +by `SetWebhookHealthWindow` at `src/pkg/github/webhook_cache.go:206` once the governor loop publishes 2 × `eval_interval_s`), and -`src/pkg/github/webhook_mode.go:34` (`WebhookRecoveryHealthyWindows`). The +`src/pkg/github/webhook_mode.go:38` (`WebhookRecoveryHealthyWindows`). The effective tick is composed in `webhookAwareEvalInterval` (`src/cmd/hive/api_budget_decision.go:82`): the API-budget stretch still applies and the larger interval wins. @@ -158,7 +158,7 @@ forgestate: Webhook mode is only safe while deliveries actually arrive. The governor therefore keeps a small cadence **mode** machine on top of the raw health -window (`src/pkg/github/webhook_mode.go:100`, `observe`) and recomputes it +window (`src/pkg/github/webhook_mode.go:149`, `observe`) and recomputes it wherever it recomputes the eval interval (`evalIntervalAndModeForConfig`, `src/cmd/hive/api_budget_decision.go:115`), including the re-check timer that fires as the health window closes @@ -174,16 +174,16 @@ States and transitions: - the health window lapsed **and** GitHub was active meanwhile: a cadence pass listed an item whose `updated_at` is newer than the newest delivery (plus a 30 s clock-skew allowance, `webhookActivityClockSkew`, - `src/pkg/github/webhook_mode.go:40`). The reconcile pass reports that + `src/pkg/github/webhook_mode.go:44`). The reconcile pass reports that newest `updated_at` through `RecordGitHubActivity` - (`src/pkg/github/webhook_mode.go:183`). Silence alone — a quiet + (`src/pkg/github/webhook_mode.go:232`). Silence alone — a quiet repository overnight — is **not** degradation and keeps the stretched tick; or - deliveries are failing: the receiver saw a delivery (direct or relayed by the hub) whose signature did not verify or whose payload did not decode, newer than the last good delivery and inside the window - (`RecordWebhookDeliveryFailure`, `src/pkg/github/webhook_mode.go:199`, - called from `src/pkg/dashboard/api_github_webhook.go:58`). A wrong + (`RecordWebhookDeliveryFailure`, `src/pkg/github/webhook_mode.go:251`, + called from `src/pkg/dashboard/api_github_webhook.go:84`). A wrong `GITHUB_WEBHOOK_SECRET` on a spoke therefore shows up as polling mode with `mode_reason: deliveries failing verification`, not as a silent stretch to 900 s. @@ -208,8 +208,12 @@ The dashboard reads the same machine: - `/api/status.webhooks.mode` — `webhook` | `polling`; `/api/status.webhooks.since` — when the current mode began; - `mode_reason` while polling after a degradation; `healthy_windows` while - recovering (`src/pkg/github/webhook_cache.go:307`, `snapshot`). + `mode_reason` while polling after a degradation or while webhooks cannot + work at all; `mode_reason_code` (`no_secret`, `signature_failed`, + `deliveries_rejected`, `no_deliveries`) whenever it is polling for a + reason; `rejected_1h` and `last_rejected_at` for deliveries the receiver + turned away; `healthy_windows` while recovering + (`src/pkg/github/webhook_cache.go:430`, `snapshot`). `webhooks.healthy` stays the raw window (a delivery inside 2 × `eval_interval_s`), so `healthy: false, mode: webhook` reads "quiet but trusted" and `healthy: true, mode: polling` reads "recovering". @@ -228,7 +232,15 @@ Operator checks: `mode: webhook` on a busy repo with the App webhook pointed at the hub or the spoke; `polling` with a `mode_reason` is a configuration fault to fix (secret, App subscription, hub relay target), `polling` without one - means no delivery has ever arrived. + means no delivery has ever arrived. `mode_reason_code` tells the causes + apart: `no_secret` means `GITHUB_WEBHOOK_SECRET` is unset on this spoke + and every delivery, relayed or direct, is answered 503 (`rejected_1h` and + `last_rejected_at` show they are arriving); `signature_failed` means the + secret does not match the sender's; `deliveries_rejected` means signed + payloads did not decode; `no_deliveries` means the secret is set but + nothing verified has arrived, so look at the App webhook URL or the hub + relay. `GET /api/health/deep` carries the same cause as a `webhooks` + warning. 2. `grep 'webhooks degraded' hive.log` — one line per outage with the reason; the matching `webhooks healthy; webhook cadence engaged` closes it. diff --git a/src/pkg/dashboard/api_github_webhook.go b/src/pkg/dashboard/api_github_webhook.go index b8ad183a36..67996df260 100644 --- a/src/pkg/dashboard/api_github_webhook.go +++ b/src/pkg/dashboard/api_github_webhook.go @@ -23,7 +23,7 @@ const ( // GITHUB_WEBHOOK_SECRET and verifies X-Hub-Signature-256 over the raw body. githubWebhookPath = "/api/webhook/github" - githubWebhookSecretEnvVar = "GITHUB_WEBHOOK_SECRET" + githubWebhookSecretEnvVar = github.WebhookSecretEnvVar githubWebhookMaxBodyBytes = 1 << 20 ) @@ -71,12 +71,17 @@ func (s *Server) handleGitHubWebhook(w http.ResponseWriter, r *http.Request) { if !warned { s.webhookLogger().Debug(msg) } + // Counted like any other rejected delivery so /api/status.webhooks + // (mode_reason_code no_secret, rejected_1h, last_rejected_at) and + // /api/health/deep say why the spoke is polling; the log stays at one + // WARN per process (#13666). + github.RecordWebhookDeliveryFailure(github.WebhookReasonNoSecret) jsonError(w, "webhooks disabled: no secret configured", http.StatusServiceUnavailable) return } if !verifyGitHubWebhookSignature(body, r.Header.Get("X-Hub-Signature-256"), secret) { s.webhookLogger().Warn("github webhook signature verification failed", "event", event) - github.RecordWebhookDeliveryFailure() + github.RecordWebhookDeliveryFailure(github.WebhookReasonSignatureFailed) jsonError(w, "invalid signature", http.StatusUnauthorized) return } @@ -97,7 +102,7 @@ func (s *Server) handleGitHubWebhook(w http.ResponseWriter, r *http.Request) { } res, err := s.deps.GHClient.HandleWebhookEvent(event, body) if err != nil { - github.RecordWebhookDeliveryFailure() + github.RecordWebhookDeliveryFailure(github.WebhookReasonDeliveriesRejected) jsonError(w, "invalid webhook payload", http.StatusBadRequest) return } diff --git a/src/pkg/dashboard/api_github_webhook_reason_test.go b/src/pkg/dashboard/api_github_webhook_reason_test.go new file mode 100644 index 0000000000..45824b52c1 --- /dev/null +++ b/src/pkg/dashboard/api_github_webhook_reason_test.go @@ -0,0 +1,197 @@ +package dashboard + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/github" +) + +// readStatusWebhooks returns /api/status?fields=webhooks as a map. +func readStatusWebhooks(t *testing.T, s *Server) map[string]any { + t.Helper() + s.statusMu.Lock() + if s.status == nil { + s.status = &StatusPayload{HiveID: "webhook-reason-test"} + } + s.statusMu.Unlock() + rec := httptest.NewRecorder() + s.handleStatus(rec, httptest.NewRequest(http.MethodGet, "/api/status?fields=webhooks", nil)) + var status map[string]any + if err := json.Unmarshal(rec.Body.Bytes(), &status); err != nil { + t.Fatal(err) + } + webhooks, ok := status["webhooks"].(map[string]any) + if !ok { + t.Fatalf("/api/status has no webhooks block: %s", rec.Body.String()) + } + return webhooks +} + +// readDeepWebhookCheck returns the body of GET /api/health/deep and its +// `webhooks` check. +func readDeepWebhookCheck(t *testing.T, s *Server) (string, map[string]any) { + t.Helper() + rec := httptest.NewRecorder() + s.handleHealthDeep(rec, httptest.NewRequest(http.MethodGet, "/api/health/deep", nil)) + var out struct { + Status string `json:"status"` + Checks map[string]any `json:"checks"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &out); err != nil { + t.Fatal(err) + } + check, ok := out.Checks[healthCheckWebhooks].(map[string]any) + if !ok { + t.Fatalf("/api/health/deep has no %q check: %s", healthCheckWebhooks, rec.Body.String()) + } + return rec.Body.String(), check +} + +// summaryWebhookCheck returns the webhooks entry of the heartbeat/dashboard +// health summary (the navbar health tooltip reads it). +func summaryWebhookCheck(t *testing.T, s *Server) (status, detail string) { + t.Helper() + raw, err := json.Marshal(s.HealthSummary()) + if err != nil { + t.Fatal(err) + } + var out struct { + Checks []struct { + Name string `json:"name"` + Status string `json:"status"` + Detail string `json:"detail"` + } `json:"checks"` + } + if err := json.Unmarshal(raw, &out); err != nil { + t.Fatal(err) + } + for _, c := range out.Checks { + if c.Name == healthCheckWebhooks { + return c.Status, c.Detail + } + } + t.Fatalf("health summary has no %q check: %s", healthCheckWebhooks, raw) + return "", "" +} + +// A spoke with no GITHUB_WEBHOOK_SECRET answers every relayed delivery 503. +// That used to leave /api/status at `healthy:false, mode:polling` with no +// reason, the same as "GitHub is not sending". The status now names the +// cause and counts the rejections, while the log stays at one WARN. +func TestGitHubWebhookNoSecretReportsReasonAndCountsRejections(t *testing.T) { + t.Cleanup(github.ResetWebhookTrackerForTest()) + t.Setenv(githubWebhookSecretEnvVar, "") + s := newGitHubWebhookTestServer(t) + body := []byte(`{"number":5,"repository":{"full_name":"webhookorg/webhookrepo"}}`) + + for i := 0; i < 3; i++ { + if rec := postGitHubWebhook(s, "pull_request", "sha256=00", body); rec.Code != http.StatusServiceUnavailable { + t.Fatalf("attempt %d: status = %d, want 503", i, rec.Code) + } + } + got := readStatusWebhooks(t, s) + if got["mode"] != string(github.WebhookModePolling) || got["mode_reason_code"] != github.WebhookReasonNoSecret { + t.Fatalf("webhooks = %v, want polling with mode_reason_code %q", got, github.WebhookReasonNoSecret) + } + if reason, _ := got["mode_reason"].(string); !strings.Contains(reason, "no GITHUB_WEBHOOK_SECRET") || !strings.Contains(reason, "503") { + t.Fatalf("mode_reason = %q, want the plain-words no-secret cause", reason) + } + if n, _ := got["rejected_1h"].(float64); n != 3 { + t.Fatalf("rejected_1h = %v, want 3", got["rejected_1h"]) + } + if at, _ := got["last_rejected_at"].(string); at == "" { + t.Fatalf("last_rejected_at = %v, want the time of the last 503", got["last_rejected_at"]) + } else if _, err := time.Parse(time.RFC3339Nano, at); err != nil { + t.Fatalf("last_rejected_at = %q: %v", at, err) + } +} + +// The deep health check (health_deep_read) and the dashboard health summary +// carry the no-secret cause as a warning that never changes the verdict. +func TestHealthDeepWarnsWhenWebhooksRejectedForMissingSecret(t *testing.T) { + t.Cleanup(github.ResetWebhookTrackerForTest()) + t.Setenv(githubWebhookSecretEnvVar, "") + s := newGitHubWebhookTestServer(t) + s.statusMu.Lock() + s.status = &StatusPayload{HiveID: "webhook-health-test"} + s.ready = true + s.statusMu.Unlock() + + // Not configured and nobody sending: polling by design, not a warning. + if _, check := readDeepWebhookCheck(t, s); check["status"] != "pass" || check["mode_reason_code"] != github.WebhookReasonNoSecret { + t.Fatalf("before any delivery webhooks check = %v, want pass with no_secret", check) + } + + if rec := postGitHubWebhook(s, "pull_request", "", []byte(`{}`)); rec.Code != http.StatusServiceUnavailable { + t.Fatalf("status = %d, want 503", rec.Code) + } + body, check := readDeepWebhookCheck(t, s) + detail, _ := check["detail"].(string) + if check["status"] != "warn" || check["mode_reason_code"] != github.WebhookReasonNoSecret || + !strings.Contains(detail, "no GITHUB_WEBHOOK_SECRET on this spoke") || !strings.Contains(detail, "rejected with 503") { + t.Fatalf("webhooks check = %v, want a no-secret warning", check) + } + if !strings.Contains(body, `"status":"ok"`) { + t.Fatalf("a webhook warning must not change the overall verdict: %s", body) + } + if st, d := summaryWebhookCheck(t, s); st != "warn" || !strings.Contains(d, "no GITHUB_WEBHOOK_SECRET") { + t.Fatalf("health summary webhooks = %s %q, want the same warning", st, d) + } +} + +// With the secret set, the cause is "no deliveries" until a verified delivery +// arrives; then it clears and the check passes. The secret itself never +// appears in the status or health output. +func TestWebhookReasonClearsAfterVerifiedDelivery(t *testing.T) { + const secret = "s3cr3t-never-shown" + t.Cleanup(github.ResetWebhookTrackerForTest()) + t.Setenv(githubWebhookSecretEnvVar, secret) + s := newGitHubWebhookTestServer(t) + s.statusMu.Lock() + s.status = &StatusPayload{HiveID: "webhook-clear-test"} + s.ready = true + s.statusMu.Unlock() + + if got := readStatusWebhooks(t, s); got["mode_reason_code"] != github.WebhookReasonNoDeliveries { + t.Fatalf("before any delivery webhooks = %v, want mode_reason_code %q", got, github.WebhookReasonNoDeliveries) + } + + known := []byte(`{"number":5,"pull_request":{"number":5},"repository":{"full_name":"webhookorg/webhookrepo"}}`) + if rec := postGitHubWebhook(s, "pull_request", "sha256=00", known); rec.Code != http.StatusUnauthorized { + t.Fatalf("bad signature: status = %d, want 401", rec.Code) + } + if got := readStatusWebhooks(t, s); got["mode_reason_code"] != github.WebhookReasonSignatureFailed { + t.Fatalf("after a bad signature webhooks = %v, want mode_reason_code %q", got, github.WebhookReasonSignatureFailed) + } + if _, check := readDeepWebhookCheck(t, s); check["status"] != "warn" { + t.Fatalf("after a bad signature webhooks check = %v, want warn", check) + } + + if rec := postGitHubWebhook(s, "pull_request", signGitHubWebhookForTest(secret, known), known); rec.Code != http.StatusOK { + t.Fatalf("delivery: status = %d (%s)", rec.Code, rec.Body.String()) + } + if got := readStatusWebhooks(t, s); got["mode_reason_code"] != nil || got["healthy"] != true { + t.Fatalf("after a verified delivery webhooks = %v, want healthy with no reason code", got) + } + github.ObserveWebhookMode() + got := readStatusWebhooks(t, s) + if got["mode"] != string(github.WebhookModeWebhook) || got["mode_reason_code"] != nil { + t.Fatalf("after observe webhooks = %v, want webhook mode with no reason", got) + } + body, check := readDeepWebhookCheck(t, s) + if check["status"] != "pass" { + t.Fatalf("webhooks check = %v, want pass", check) + } + raw, _ := json.Marshal(got) + summary, _ := json.Marshal(s.HealthSummary()) + for _, out := range []string{string(raw), body, string(summary)} { + if strings.Contains(out, secret) { + t.Fatalf("webhook secret leaked into output: %s", out) + } + } +} diff --git a/src/pkg/dashboard/health_webhooks.go b/src/pkg/dashboard/health_webhooks.go new file mode 100644 index 0000000000..7c6e4f754a --- /dev/null +++ b/src/pkg/dashboard/health_webhooks.go @@ -0,0 +1,57 @@ +package dashboard + +import ( + "fmt" + + "github.com/hivecommons/hive/pkg/github" +) + +// Webhook receiver health. A hosted spoke that never received +// GITHUB_WEBHOOK_SECRET answers every relayed delivery with 503 and stays on +// the polling cadence; before this check the only trace was one WARN line +// per process, and /api/status read the same as "GitHub is not sending". +// The check names the cause (webhooks.mode_reason_code) as a warning: the +// hive still works on polling, so it is never a failure and never raises the +// hub's health-check-failing alert. +// +// Only whether the secret is set is ever reported, never its value, length +// or a digest of it. + +const healthCheckWebhooks = "webhooks" + +// webhookHealth grades a webhook snapshot: "warn" when deliveries arrive but +// cannot be used, or webhooks degraded while GitHub stayed busy; "pass" +// otherwise, with the mode and cause in the detail. +func webhookHealth(snap github.WebhookHealth) (status, detail string) { + if snap.Mode != github.WebhookModePolling { + return "pass", fmt.Sprintf("webhook mode — %d deliveries in the last hour", snap.Events1h) + } + rejected := fmt.Sprintf("%d rejected in the last hour", snap.Rejected1h) + switch snap.ReasonCode { + case github.WebhookReasonNoSecret: + if snap.LastRejectedAt != nil { + return "warn", "polling — no GITHUB_WEBHOOK_SECRET on this spoke; relayed deliveries are rejected with 503 (" + rejected + ")" + } + return "pass", "polling — no GITHUB_WEBHOOK_SECRET on this spoke; webhooks are off and no delivery has arrived" + case github.WebhookReasonSignatureFailed: + return "warn", "polling — deliveries fail signature verification: GITHUB_WEBHOOK_SECRET does not match the secret the sender signs with (" + rejected + ")" + case github.WebhookReasonDeliveriesRejected: + return "warn", "polling — deliveries arrive but are rejected as undecodable payloads (" + rejected + ")" + case github.WebhookReasonNoDeliveries: + if snap.Reason != "" { + return "warn", "polling — " + snap.Reason + "; check the GitHub App webhook URL or the hub relay" + } + return "pass", "polling — no verified delivery yet" + } + if snap.HealthyWindows > 0 { + return "pass", fmt.Sprintf("polling — recovering, %d of %d healthy windows", snap.HealthyWindows, github.WebhookRecoveryHealthyWindows) + } + return "pass", "polling — verified deliveries arriving; webhook mode engages at the next eval cycle" +} + +// webhookHealthCheck grades the live webhook tracker. +func webhookHealthCheck() (status, detail string, snap github.WebhookHealth) { + snap = github.WebhookHealthSnapshot() + status, detail = webhookHealth(snap) + return status, detail, snap +} diff --git a/src/pkg/dashboard/server.go b/src/pkg/dashboard/server.go index f09df0f037..231e6d5a6d 100644 --- a/src/pkg/dashboard/server.go +++ b/src/pkg/dashboard/server.go @@ -3978,6 +3978,18 @@ func (s *Server) handleHealthDeep(w http.ResponseWriter, r *http.Request) { } } + // Webhook receiver: why the spoke is polling (no secret, deliveries + // failing, none arriving). A warning never changes the overall verdict: + // the hive still works on the polling cadence. + { + st, detail, snap := webhookHealthCheck() + wh := map[string]any{"status": st, "detail": detail, "mode": string(snap.Mode), "rejected_1h": snap.Rejected1h} + if snap.ReasonCode != "" { + wh["mode_reason_code"] = snap.ReasonCode + } + checks[healthCheckWebhooks] = wh + } + // 12. Queue trend (is work being processed?) s.statusMu.RLock() if s.status != nil { @@ -5330,6 +5342,14 @@ func (s *Server) healthSummaryFor(status *StatusPayload, ready bool) map[string] fails++ } + // Webhook receiver cause (see health_webhooks.go): a warning, never a fail. + if st, detail, _ := webhookHealthCheck(); st != "" { + checks = append(checks, check{Name: healthCheckWebhooks, Status: st, Detail: detail}) + if st == "warn" { + warns++ + } + } + // 8. Queue if status != nil { total := 0 diff --git a/src/pkg/github/webhook_cache.go b/src/pkg/github/webhook_cache.go index 5835d8137d..e4ce4db285 100644 --- a/src/pkg/github/webhook_cache.go +++ b/src/pkg/github/webhook_cache.go @@ -42,6 +42,11 @@ type WebhookHealth struct { CIEvents1h int `json:"ci_events_1h"` CIEventsApplied1h int `json:"ci_events_applied_1h"` CIRefetches1h int `json:"ci_refetches_1h"` + // Rejected1h counts deliveries the receiver turned away in the last hour + // (no secret configured, signature mismatch, undecodable payload) and + // LastRejectedAt is the newest one; mode_reason_code says which kind. + Rejected1h int `json:"rejected_1h"` + LastRejectedAt *time.Time `json:"last_rejected_at"` // WebhookModeStatus is the reconcile cadence the health drives: // `mode` webhook|polling and `since` (hivecommons/hive#13381). WebhookModeStatus @@ -137,7 +142,42 @@ type webhookTracker struct { // the cadence mode machine (webhook_mode.go). lastActivity time.Time lastFailure time.Time - mode webhookModeMachine + // lastFailureReason is the WebhookReason* code of lastFailure and + // rejected the per-minute count of rejected deliveries (rejected_1h). + lastFailureReason string + rejected webhookMinuteCounter + mode webhookModeMachine +} + +// webhookMinuteCounter counts events over the last hour in sixty one-minute +// buckets: fixed size, so recording a delivery never allocates however many +// arrive. +type webhookMinuteCounter struct { + minute [60]int64 + count [60]uint32 +} + +func (c *webhookMinuteCounter) add(now time.Time) { + m := now.Unix() / 60 + i := int(m % 60) + if c.minute[i] != m { + c.minute[i] = m + c.count[i] = 0 + } + c.count[i]++ +} + +// total is the count over the hour ending at now (the current minute and the +// 59 before it). +func (c *webhookMinuteCounter) total(now time.Time) int { + m := now.Unix() / 60 + n := 0 + for i := range c.minute { + if c.count[i] > 0 && c.minute[i] > m-60 && c.minute[i] <= m { + n += int(c.count[i]) + } + } + return n } func newWebhookTracker() *webhookTracker { @@ -402,7 +442,12 @@ func (t *webhookTracker) snapshot() WebhookHealth { out := WebhookHealth{Healthy: healthy, Events1h: len(t.events), Invalidations1h: len(t.invalidations), TargetedRefreshes1h: len(t.refreshes), TrackerInvalidations1h: len(t.trackerDrops), CIEvents1h: len(t.ciEvents), CIEventsApplied1h: len(t.ciApplied), CIRefetches1h: len(t.ciRefetches), - WebhookModeStatus: t.mode.status(t.window, now)} + Rejected1h: t.rejected.total(now), WebhookModeStatus: t.mode.status(t.window, now)} + t.annotateModeLocked(&out.WebhookModeStatus, now) + if !t.lastFailure.IsZero() { + at := t.lastFailure.UTC() + out.LastRejectedAt = &at + } if !latest.IsZero() { at := latest.UTC() out.LastEventAt = &at diff --git a/src/pkg/github/webhook_mode.go b/src/pkg/github/webhook_mode.go index 3750f8281b..6277b8d130 100644 --- a/src/pkg/github/webhook_mode.go +++ b/src/pkg/github/webhook_mode.go @@ -1,6 +1,10 @@ package github -import "time" +import ( + "os" + "strings" + "time" +) // Webhook-outage fallback to polling (hivecommons/hive#13381, ForgeState // phase 4). @@ -45,13 +49,58 @@ const ( WebhookDegradedReasonFailures = "deliveries failing verification" ) +// Machine-readable causes for polling, reported as webhooks.mode_reason_code +// so an operator (or a tool) can tell "this spoke cannot verify deliveries" +// from "GitHub is not sending any". They are also the reasons +// RecordWebhookDeliveryFailure takes for a rejected delivery. +const ( + // WebhookReasonNoSecret: GITHUB_WEBHOOK_SECRET is unset on this process, + // so every delivery (direct or hub-relayed) is answered 503. + WebhookReasonNoSecret = "no_secret" + // WebhookReasonSignatureFailed: deliveries arrive but their + // X-Hub-Signature-256 does not verify against the configured secret. + WebhookReasonSignatureFailed = "signature_failed" + // WebhookReasonDeliveriesRejected: deliveries arrive signed but are + // rejected for another reason (the payload did not decode). + WebhookReasonDeliveriesRejected = "deliveries_rejected" + // WebhookReasonNoDeliveries: the secret is set but no verified delivery + // is inside the health window (none ever, or none while GitHub was busy). + WebhookReasonNoDeliveries = "no_deliveries" +) + +// webhookReasonNoSecret*Text are the plain-words mode_reason for +// WebhookReasonNoSecret: with deliveries arriving, and with none yet. +const ( + webhookReasonNoSecretRejectedText = "no GITHUB_WEBHOOK_SECRET on this spoke; relayed deliveries are rejected with 503" + webhookReasonNoSecretIdleText = "no GITHUB_WEBHOOK_SECRET on this spoke; webhooks are off" +) + +// WebhookSecretEnvVar names the GitHub App webhook HMAC secret a spoke +// verifies /api/webhook/github deliveries with. +const WebhookSecretEnvVar = "GITHUB_WEBHOOK_SECRET" + +// WebhookSecretConfigured reports whether GITHUB_WEBHOOK_SECRET is set (non-blank) +// in this process. Only the yes/no ever leaves this function: the value, its +// length and any digest of it must never reach status, logs or health output. +func WebhookSecretConfigured() bool { + return strings.TrimSpace(os.Getenv(WebhookSecretEnvVar)) != "" +} + // WebhookModeStatus is the mode block /api/status serves inside `webhooks`. type WebhookModeStatus struct { Mode WebhookMode `json:"mode"` Since *time.Time `json:"since,omitempty"` - // Reason is set while polling after a degradation (empty at boot before - // webhooks were ever healthy, and in webhook mode). + // Reason is the plain-words cause while polling: the degradation reason + // after a degradation, or why webhooks cannot work at all (no secret, + // deliveries failing). Empty in webhook mode and while polling only + // because no delivery has arrived yet. Reason string `json:"mode_reason,omitempty"` + // ReasonCode is the machine-readable cause while polling: one of + // WebhookReasonNoSecret, WebhookReasonSignatureFailed, + // WebhookReasonDeliveriesRejected or WebhookReasonNoDeliveries. Empty in + // webhook mode, and while polling with a verified delivery in the window + // and no newer failure (about to engage webhook mode). + ReasonCode string `json:"mode_reason_code,omitempty"` // HealthyWindows counts the consecutive healthy windows observed while // polling after a degradation; recovery happens at // WebhookRecoveryHealthyWindows. Non-zero means "recovering". @@ -193,16 +242,58 @@ func RecordGitHubActivity(updatedAt time.Time) { } // RecordWebhookDeliveryFailure notes a delivery that reached the receiver -// (directly or via the hub relay) but could not be applied — a signature -// that did not verify or a payload that did not decode. Failures newer than -// the newest good delivery degrade the cadence to polling. -func RecordWebhookDeliveryFailure() { +// (directly or via the hub relay) but could not be applied, with why: +// WebhookReasonNoSecret (no secret to verify with), WebhookReasonSignatureFailed +// or WebhookReasonDeliveriesRejected (payload did not decode). Failures newer +// than the newest good delivery degrade the cadence to polling. Each call +// costs one counter bump in a fixed per-minute ring, never an allocation, so +// a flood of rejected deliveries cannot grow memory. +func RecordWebhookDeliveryFailure(reason string) { t := sharedWebhookTracker() t.mu.Lock() - t.lastFailure = t.now() + now := t.now() + t.lastFailure = now + t.lastFailureReason = reason + t.rejected.add(now) t.mu.Unlock() } +// annotateModeLocked fills the polling cause (mode_reason_code, and a +// plain-words mode_reason where the machine has none) on a mode status. +func (t *webhookTracker) annotateModeLocked(st *WebhookModeStatus, now time.Time) { + if st.Mode != WebhookModePolling { + return + } + healthy, latest := t.healthyLocked(now) + recentFailure := !t.lastFailure.IsZero() && now.Sub(t.lastFailure) < t.window && t.lastFailure.After(latest) + switch { + case !WebhookSecretConfigured(): + st.ReasonCode = WebhookReasonNoSecret + if st.Reason == "" { + st.Reason = webhookReasonNoSecretIdleText + if !t.lastFailure.IsZero() && t.lastFailureReason == WebhookReasonNoSecret { + st.Reason = webhookReasonNoSecretRejectedText + } + } + case recentFailure: + st.ReasonCode = webhookFailureCode(t.lastFailureReason) + if st.Reason == "" { + st.Reason = WebhookDegradedReasonFailures + } + case !healthy: + st.ReasonCode = WebhookReasonNoDeliveries + } +} + +// webhookFailureCode maps a recorded failure reason to the reported code; a +// failure recorded without a known reason reads as a rejected delivery. +func webhookFailureCode(reason string) string { + if reason == WebhookReasonSignatureFailed { + return reason + } + return WebhookReasonDeliveriesRejected +} + func (t *webhookTracker) modeInputLocked(now time.Time) webhookModeInput { healthy, latest := t.healthyLocked(now) in := webhookModeInput{healthy: healthy} @@ -219,13 +310,18 @@ func (t *webhookTracker) observeMode() webhookModeTransition { t.mu.Lock() defer t.mu.Unlock() now := t.now() - return t.mode.observe(t.modeInputLocked(now), t.window, now) + tr := t.mode.observe(t.modeInputLocked(now), t.window, now) + t.annotateModeLocked(&tr.Status, now) + return tr } func (t *webhookTracker) modeSnapshot() WebhookModeStatus { t.mu.Lock() defer t.mu.Unlock() - return t.mode.status(t.window, t.now()) + now := t.now() + st := t.mode.status(t.window, now) + t.annotateModeLocked(&st, now) + return st } // newestEnumeratedUpdate is the newest `updated_at` across one repo's listed diff --git a/src/pkg/github/webhook_mode_test.go b/src/pkg/github/webhook_mode_test.go index ae65829d65..0c60922cbe 100644 --- a/src/pkg/github/webhook_mode_test.go +++ b/src/pkg/github/webhook_mode_test.go @@ -118,7 +118,7 @@ func TestWebhookTrackerModeInputs(t *testing.T) { // A failure newer than the last good delivery degrades; one older than a // later good delivery does not. - RecordWebhookDeliveryFailure() + RecordWebhookDeliveryFailure(WebhookReasonSignatureFailed) if status, degraded, _ = ObserveWebhookMode(); status.Mode != WebhookModePolling || !degraded || status.Reason != WebhookDegradedReasonFailures { t.Fatalf("failure: %+v degraded=%v", status, degraded) } @@ -128,3 +128,58 @@ func TestWebhookTrackerModeInputs(t *testing.T) { t.Fatalf("good delivery after the failure should start the streak: %+v", status) } } + +// TestWebhookModeReasonCodes covers webhooks.mode_reason_code and the +// rejected-delivery counters: no secret wins, a failure newer than the last +// good delivery names its kind, silence is no_deliveries, and a verified +// delivery clears the code. rejected_1h ages out after an hour. +func TestWebhookModeReasonCodes(t *testing.T) { + clock := &webhookTestClock{now: time.Date(2026, 11, 7, 12, 0, 0, 0, time.UTC)} + t.Cleanup(resetWebhookTrackerForTest(clock.Now)) + SetWebhookHealthWindow(10 * time.Minute) + + t.Setenv(WebhookSecretEnvVar, " ") + if snap := WebhookHealthSnapshot(); snap.ReasonCode != WebhookReasonNoSecret || snap.Reason != webhookReasonNoSecretIdleText || snap.LastRejectedAt != nil { + t.Fatalf("blank secret, nothing sent: %+v", snap) + } + for i := 0; i < 4; i++ { + RecordWebhookDeliveryFailure(WebhookReasonNoSecret) + } + snap := WebhookHealthSnapshot() + if snap.ReasonCode != WebhookReasonNoSecret || snap.Reason != webhookReasonNoSecretRejectedText || snap.Rejected1h != 4 || + snap.LastRejectedAt == nil || !snap.LastRejectedAt.Equal(clock.now) { + t.Fatalf("no secret, deliveries rejected: %+v", snap) + } + if got := WebhookModeSnapshot(); got.ReasonCode != WebhookReasonNoSecret { + t.Fatalf("mode snapshot = %+v, want no_secret", got) + } + clock.now = clock.now.Add(59 * time.Minute) + if snap = WebhookHealthSnapshot(); snap.Rejected1h != 4 { + t.Fatalf("rejected_1h inside the hour = %d, want 4", snap.Rejected1h) + } + clock.now = clock.now.Add(time.Minute) + if snap = WebhookHealthSnapshot(); snap.Rejected1h != 0 || snap.LastRejectedAt == nil { + t.Fatalf("after an hour rejected_1h = %d (want 0), last_rejected_at = %v (want kept)", snap.Rejected1h, snap.LastRejectedAt) + } + + t.Setenv(WebhookSecretEnvVar, "set") + if snap = WebhookHealthSnapshot(); snap.ReasonCode != WebhookReasonNoDeliveries || snap.Reason != "" { + t.Fatalf("secret set, silent: %+v", snap) + } + RecordWebhookDeliveryFailure(WebhookReasonSignatureFailed) + if snap = WebhookHealthSnapshot(); snap.ReasonCode != WebhookReasonSignatureFailed || snap.Reason != WebhookDegradedReasonFailures { + t.Fatalf("bad signature: %+v", snap) + } + RecordWebhookDeliveryFailure(WebhookReasonDeliveriesRejected) + if snap = WebhookHealthSnapshot(); snap.ReasonCode != WebhookReasonDeliveriesRejected || snap.Rejected1h != 2 { + t.Fatalf("undecodable payload: %+v", snap) + } + clock.now = clock.now.Add(time.Second) + RecordWebhookEventForTest("org/repo") + if snap = WebhookHealthSnapshot(); snap.ReasonCode != "" || snap.Reason != "" || !snap.Healthy { + t.Fatalf("verified delivery should clear the cause: %+v", snap) + } + if status, _, recovered := ObserveWebhookMode(); status.Mode != WebhookModeWebhook || !recovered || status.ReasonCode != "" { + t.Fatalf("observe: %+v recovered=%v, want webhook mode with no code", status, recovered) + } +} From 5e992cf7b373e6bb0e33c826c8509ee1b53f0c78 Mon Sep 17 00:00:00 2001 From: Danathar Date: Sat, 7 Nov 2026 21:34:56 -0500 Subject: [PATCH 2/2] fix(webhooks): warn only on rejections inside the hour; never move the summary verdict Two findings from the hive reviewer on #17173: - The no-secret warning keyed on LastRejectedAt, which never resets, so a single rejected delivery left the spoke at warn for good. It now follows Rejected1h, the same hourly window the other counters use. - The health summary counted the webhook warning in warns, which turned the summary verdict (header pill, DeepHealth) to "warning". The PR and docs said it never changes the overall verdict. The check is still listed but no longer counted, so that is now true on both endpoints. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BQYnPGFJx81c8JXR3o1sg1 Signed-off-by: Danathar --- .../api_github_webhook_reason_test.go | 20 +++++++++++++++++++ src/pkg/dashboard/health_webhooks.go | 6 ++++-- src/pkg/dashboard/server.go | 7 +++---- 3 files changed, 27 insertions(+), 6 deletions(-) diff --git a/src/pkg/dashboard/api_github_webhook_reason_test.go b/src/pkg/dashboard/api_github_webhook_reason_test.go index 45824b52c1..f198843381 100644 --- a/src/pkg/dashboard/api_github_webhook_reason_test.go +++ b/src/pkg/dashboard/api_github_webhook_reason_test.go @@ -142,6 +142,26 @@ func TestHealthDeepWarnsWhenWebhooksRejectedForMissingSecret(t *testing.T) { if st, d := summaryWebhookCheck(t, s); st != "warn" || !strings.Contains(d, "no GITHUB_WEBHOOK_SECRET") { t.Fatalf("health summary webhooks = %s %q, want the same warning", st, d) } + // The summary feeds the header pill and DeepHealth: the warning is listed + // there but does not move its verdict either (#17173 review). + if sum := s.HealthSummary(); sum["warns"] != 0 || sum["status"] != "ok" { + t.Fatalf("health summary verdict = %v (warns %v), want ok with the webhook check not counted", sum["status"], sum["warns"]) + } +} + +// One rejection outside the hourly window must not keep the spoke at warn +// for good: the no-secret warning follows Rejected1h, not LastRejectedAt +// (#17173 review). +func TestWebhookHealthNoSecretWarnsOnlyWithinTheHour(t *testing.T) { + old := time.Now().Add(-26 * time.Hour) + snap := github.WebhookHealth{WebhookModeStatus: github.WebhookModeStatus{Mode: github.WebhookModePolling, ReasonCode: github.WebhookReasonNoSecret}, LastRejectedAt: &old} + if st, d := webhookHealth(snap); st != "pass" { + t.Fatalf("stale rejection: webhookHealth = %s %q, want pass", st, d) + } + snap.Rejected1h = 2 + if st, _ := webhookHealth(snap); st != "warn" { + t.Fatalf("rejections in the last hour: webhookHealth = %s, want warn", st) + } } // With the secret set, the cause is "no deliveries" until a verified delivery diff --git a/src/pkg/dashboard/health_webhooks.go b/src/pkg/dashboard/health_webhooks.go index 7c6e4f754a..1c91c3f88f 100644 --- a/src/pkg/dashboard/health_webhooks.go +++ b/src/pkg/dashboard/health_webhooks.go @@ -29,10 +29,12 @@ func webhookHealth(snap github.WebhookHealth) (status, detail string) { rejected := fmt.Sprintf("%d rejected in the last hour", snap.Rejected1h) switch snap.ReasonCode { case github.WebhookReasonNoSecret: - if snap.LastRejectedAt != nil { + // Only rejections inside the window count: one delivery days ago must + // not leave the spoke at warn for good. + if snap.Rejected1h > 0 { return "warn", "polling — no GITHUB_WEBHOOK_SECRET on this spoke; relayed deliveries are rejected with 503 (" + rejected + ")" } - return "pass", "polling — no GITHUB_WEBHOOK_SECRET on this spoke; webhooks are off and no delivery has arrived" + return "pass", "polling — no GITHUB_WEBHOOK_SECRET on this spoke; webhooks are off and no delivery was rejected in the last hour" case github.WebhookReasonSignatureFailed: return "warn", "polling — deliveries fail signature verification: GITHUB_WEBHOOK_SECRET does not match the secret the sender signs with (" + rejected + ")" case github.WebhookReasonDeliveriesRejected: diff --git a/src/pkg/dashboard/server.go b/src/pkg/dashboard/server.go index 231e6d5a6d..20c165bb9d 100644 --- a/src/pkg/dashboard/server.go +++ b/src/pkg/dashboard/server.go @@ -5342,12 +5342,11 @@ func (s *Server) healthSummaryFor(status *StatusPayload, ready bool) map[string] fails++ } - // Webhook receiver cause (see health_webhooks.go): a warning, never a fail. + // Webhook receiver cause (see health_webhooks.go): listed, but it does not + // count toward warns, so it never changes the summary verdict that drives + // the header pill and DeepHealth — the hive works on polling. if st, detail, _ := webhookHealthCheck(); st != "" { checks = append(checks, check{Name: healthCheckWebhooks, Status: st, Detail: detail}) - if st == "warn" { - warns++ - } } // 8. Queue