diff --git a/changelog.d/fixed-webhook-no-secret-reason.md b/changelog.d/fixed-webhook-no-secret-reason.md new file mode 100644 index 0000000000..6dd15e493c --- /dev/null +++ b/changelog.d/fixed-webhook-no-secret-reason.md @@ -0,0 +1 @@ +- A spoke that rejects webhook deliveries now says why. `/api/status.webhooks` (and `/api/gh-rate-limits`) gain `mode_reason_code` — `no_secret` when `GITHUB_WEBHOOK_SECRET` is unset, `signature_failed` or `deliveries_rejected` when deliveries arrive but fail, `no_deliveries` when the secret is set but nothing verified has arrived — plus `rejected_1h` and `last_rejected_at`; `mode_reason` gives the same cause in plain words. The 503 a spoke without a secret returns is now counted like any other rejected delivery (the log stays at one WARN per process), and `/api/health/deep` plus the dashboard health tooltip carry a `webhooks` warning such as "polling — no GITHUB_WEBHOOK_SECRET on this spoke; relayed deliveries are rejected with 503". Before this, a hosted spoke whose hub never provisioned the secret read `healthy:false, mode:polling` with no reason, the same as "GitHub isn't sending". Only whether the secret is set is reported, never its value. See [webhooks.md](src/docs/webhooks.md). diff --git a/src/docs/api-reference.md b/src/docs/api-reference.md index 800e9784f0..af6c52b7a7 100644 --- a/src/docs/api-reference.md +++ b/src/docs/api-reference.md @@ -34,7 +34,7 @@ History point limits, timestamp cutoffs, incremental cost reads and ETag/304 beh | `GET` | `/api/version/release-notes` | Dashboard auth/session | Changelog gap between two revisions: `?from=&to=` returns `{from:{sha,version?}, to:{sha,ref,version?}, sections:[{version,date,categories:{added,changed,fixed,security,deprecated}}], unreleased?, truncated, source, error?}` — the `CHANGELOG.md` release sections present at `to` and absent at `from`, newest first, plus (when `to` is not a release tag) the `changelog.d/` fragments new at `to` under an `Unreleased (in this build)` section. Fetch failures return `source: "unavailable"` with `error` rather than an error status; results are cached per (from, to) for an hour ([#11065](https://github.com/hivecommons/hive/issues/11065)) | `pkg/dashboard/api.go:46` | | `POST` | `/api/release-channel` | Owner only | Hosted spoke self-service release-channel selector; relays `stable`/`candidate`/`edge` to the hub's existing switch-branch endpoint with the spoke dashboard-token proof, and reports the requested channel as pending until the Deployment image lands | `pkg/dashboard/api.go:76` | | `GET` | `/api/health` | Public | Basic health probe | `pkg/dashboard/server.go:1811` | -| `GET` | `/api/health/deep` | Public | Deep health probe | `pkg/dashboard/server.go:1812` | +| `GET` | `/api/health/deep` | Public | Deep health probe; the `webhooks` check (`status`, `detail`, `mode`, `mode_reason_code`, `rejected_1h`) warns when deliveries arrive but cannot be used — no `GITHUB_WEBHOOK_SECRET`, signature mismatch, undecodable payload — and never changes the overall verdict | `pkg/dashboard/server.go:1812` | | `GET` | `/api/livez` | Public | Kubernetes liveness probe | `pkg/dashboard/server.go:1813` | | `GET` | `/metrics` | Registered only when `HIVE_METRICS_ENABLED`; requires `Authorization: ****** (403 if the token is unset) | Prometheus metrics | `pkg/dashboard/server.go:1819` | | `GET` | `/api/status` | Dashboard auth/session | Dashboard aggregate status; `labelPresets[]` (`{key,label,description,labels[],states[]}`) lists the built-in label-filter presets for the Projects cards, led by `needs-human` whose `labels[]` are the effective hard-suppress labels plus `needs-signal`, the hold labels, `stage/071-needs-author` and the reporter-trust awaiting label (#12172); accepts `?fields=a,b` for top-level selection and `?omit=repos` to drop heavy sections, and honors `Accept-Encoding: gzip`; `agents[]` includes busy-pane visibility fields (`kicksUndeliverable`, `busySince`, `lastTranscriptActivity`, `condition`, `conditionMessage`) when a Working pane cannot accept kicks; `sla` (fleet) and `repos[].sla` (per repo) carry the Delivery SLA block derived from the lifecycle timeline with no extra GitHub calls: `windows.{7d,30d}.{open_to_triage,triage_to_start,start_to_pr,pr_to_merge,open_to_merge}` as `{p50_h,p95_h,n}` (hours, nearest-rank), `history[]` of up to 30 daily buckets `{day,start_to_pr_p50,pr_to_merge_p50,open_to_merge_p50,n_started,n_pr,n_merged}` persisted under `/data/sla-history.json`, `stale_in_progress[]` (`{number,repo,started_at,age_h,agent}` — issues carrying `hive/in-progress` longer than the agent claim TTL with no linked PR), `stale_count`, `merged_7d`, `claim_ttl_s` and, on the fleet block, the effective `targets` (`start_to_pr_h`, `pr_to_merge_h`, `open_to_merge_h`); `staleSince` and `repos[].countsStale` / `countsStaleSince` mark totals served from last-good figures — the previous enumeration (#13382) or the ForgeState cache's last-good counts (#13376); every your-turn issue and PR row carries the critical path (#13646): `waits_on[]` (`{kind,ref}`, kind one of `pr`, `issue`, `external`, `base-check` (ref = the check red on the base branch), `approval`, `merge`; ref `#N` in the same repository or `owner/repo#N`), `unblocks` (transitive count of your-turn items waiting on it, cycle-guarded), `may_conflict_with[]` (PR numbers of the same repository sharing a closing issue or a known changed file; never an edge), `action` (`merge`, `release-hold`, `rebase`, `rerun` or `approve`), `action_ref`, `root_cause` and `blocker` (the root cause it collapses under); `repos[].yourTurn` has `issues`, `prs` (your turn, `CLOSE` excluded), `root_issues`, `root_prs`, `root_cause_count`, `order[]` (`{kind,repo,number}` by unblocks then age), `groups[]` (`{blocker,blocker_title,dependents[]}`) and `main_red` (`{check,since,fix,fix_your_turn}`); the top-level `yourTurn` sums the counts | `pkg/dashboard/server.go:1824` | diff --git a/src/docs/api-reference/health-status-events.md b/src/docs/api-reference/health-status-events.md index bc61f09f3b..e92cfd5633 100644 --- a/src/docs/api-reference/health-status-events.md +++ b/src/docs/api-reference/health-status-events.md @@ -25,7 +25,7 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `GET` | `/api/version/release-notes` | Dashboard auth/session | Changelog gap between two revisions: `?from=&to=` returns `{from:{sha,version?}, to:{sha,ref,version?}, sections:[{version,date,categories:{added,changed,fixed,security,deprecated}}], unreleased?, truncated, source, error?}` — the `CHANGELOG.md` release sections present at `to` and absent at `from`, newest first, plus (when `to` is not a release tag) the `changelog.d/` fragments new at `to` under an `Unreleased (in this build)` section. Fetch failures return `source: "unavailable"` with `error` rather than an error status; results are cached per (from, to) for an hour ([#11065](https://github.com/hivecommons/hive/issues/11065)) | `pkg/dashboard/api.go:46` | | `POST` | `/api/release-channel` | Owner only | Hosted spoke self-service release-channel selector; relays `stable`/`candidate`/`edge` to the hub's existing switch-branch endpoint with the spoke dashboard-token proof, and reports the requested channel as pending until the Deployment image lands | `pkg/dashboard/api.go:76` | | `GET` | `/api/health` | Public | Basic health probe | `pkg/dashboard/server.go:1811` | -| `GET` | `/api/health/deep` | Public | Deep health probe | `pkg/dashboard/server.go:1812` | +| `GET` | `/api/health/deep` | Public | Deep health probe; the `webhooks` check (`status`, `detail`, `mode`, `mode_reason_code`, `rejected_1h`) warns when deliveries arrive but cannot be used — no `GITHUB_WEBHOOK_SECRET`, signature mismatch, undecodable payload — and never changes the overall verdict | `pkg/dashboard/server.go:1812` | | `GET` | `/api/livez` | Public | Kubernetes liveness probe | `pkg/dashboard/server.go:1813` | | `GET` | `/metrics` | Registered only when `HIVE_METRICS_ENABLED`; requires `Authorization: ****** (403 if the token is unset) | Prometheus metrics | `pkg/dashboard/server.go:1819` | | `GET` | `/api/status` | Dashboard auth/session | Dashboard aggregate status; `labelPresets[]` (`{key,label,description,labels[],states[]}`) lists the built-in label-filter presets for the Projects cards, led by `needs-human` whose `labels[]` are the effective hard-suppress labels plus `needs-signal`, the hold labels, `stage/071-needs-author` and the reporter-trust awaiting label (#12172); accepts `?fields=a,b` for top-level selection and `?omit=repos` to drop heavy sections, and honors `Accept-Encoding: gzip`; `agents[]` includes busy-pane visibility fields (`kicksUndeliverable`, `busySince`, `lastTranscriptActivity`, `condition`, `conditionMessage`) when a Working pane cannot accept kicks; a working agent also carries `agents[].run` — the live per-run line (#14576): `{since, elapsedS, costUSD, tokens, priced, tests, filesChanged}`, the estimated spend/tokens since the kick was delivered (`lastKickAt`, falling back to `busySince`) (delta of the token collector's per-agent cumulative figure; `priced:false` when the agent has none yet), the last visible test-runner result (`pass`/`fail`/empty) and the last `N files changed` git summary (`-1` when none) parsed from the captured pane; top-level `fleetRun` `{running, hourUSD, hourPriced, dayUSD, weekUSD, monthUSD, windowsPriced}` is the governor strip's rotating `running N · $X this hour / today / this week / this month` figure (#15614) — the hour is the rise since the top of the clock hour, the wider windows are the daily cost ledger summed from 00:00 UTC today, Monday 00:00 UTC (ISO week) and the 1st 00:00 UTC plus the rise since the ledger's last snapshot, `windowsPriced:false` when there is no ledger yet; `sla` (fleet) and `repos[].sla` (per repo) carry the Delivery SLA block derived from the lifecycle timeline with no extra GitHub calls: `windows.{7d,30d}.{open_to_triage,triage_to_start,start_to_pr,pr_to_merge,open_to_merge}` as `{p50_h,p95_h,n}` (hours, nearest-rank), `history[]` of up to 30 daily buckets `{day,start_to_pr_p50,pr_to_merge_p50,open_to_merge_p50,n_started,n_pr,n_merged}` persisted under `/data/sla-history.json`, `stale_in_progress[]` (`{number,repo,started_at,age_h,agent}` — issues carrying `hive/in-progress` longer than the agent claim TTL with no linked PR), `stale_count`, `merged_7d`, `claim_ttl_s` and, on the fleet block, the effective `targets` (`start_to_pr_h`, `pr_to_merge_h`, `open_to_merge_h`); `staleSince` and `repos[].countsStale` / `countsStaleSince` mark totals served from last-good figures — the previous enumeration (#13382) or the ForgeState cache's last-good counts (#13376); `ci_failure_signals` `{count, last, kicks, unmatched}` is the webhook CI-failure signal store — distinct red (repo, head, check) signals retained, the most recent one `{repo, number, head_sha, check, conclusion, details_url, agent, at}`, webhook-triggered kicks/nudges since boot, and failure deliveries whose head matched no open PR; `workflow_approvals` `{approved, denied, permission_errors, last}` counts the hive's approvals of GitHub-gated workflow runs since boot (#16603) — runs approved, runs refused by the trust policy (once per head), `403`/`404` refusals for want of `actions: write`, and the most recent decision `{repo, number, head, run_id, outcome, at}`; `hotspots` `{count, actionable, filed}` is the merge-hotspot advisor's one-line summary (files at/above the thresholds, those with a split recommendation, those with a modularize issue filed), absent when the advisor is not wired — details on `GET /api/hotspots`; `writeback.dedupe` `{enabled, dedupe_same_item, dedupe_burst, rate_capped, hourly_cap, burst_items, burst_window_s, same_item_ttl_s}` is the write-back gate's content layer — the effective `writeback.dedupe.*` settings and the last-hour counts of writes held as `duplicate`, `burst` and `rate-capped` (each also listed under `GET /api/writeback/suppressed`); `config.gate_regressions[]` `{gate, previous, effective, source}` is the boot-time safety-gate regression list (#16438) — gates a previous runtime overlay on the PVC carried that the running config lacks or loosens (`review.require_labels`, `project.issue_filter.require_labels`, `project.paused_repos[]`, `writeback.per_repo[].mode`), empty on a healthy boot | `pkg/dashboard/server.go:1824` | diff --git a/src/docs/env-vars.md b/src/docs/env-vars.md index cc224eb9ec..a5b03ebaf7 100644 --- a/src/docs/env-vars.md +++ b/src/docs/env-vars.md @@ -102,7 +102,7 @@ behavior. [Maintenance limitations](#keeping-this-reference-current) still apply | `HIVE_CONTRIBUTE_SKIP_LABELS` | No | `blocked,tracking,epic,discussion,question,needs-decision,needs-triage` | Comma-separated, case-insensitive label patterns for issues that are not contributor work and must never be offered by the relay. Patterns use `path.Match`-style `*` globs; `blocked` is always unioned into the effective set even if omitted. Same setting as `hub.contribute_skip_labels`. | | `HIVE_FEDERATION_REGISTRY_PATH` | No | `/data/federation/registry.json` | Federation registry path override. | | `HIVE_WEBHOOK_SECRET` | No | none | HMAC secret for the spoke `/webhook` channel. | -| `GITHUB_WEBHOOK_SECRET` | No | `/data/saas/webhook-secret.key` when present (hub only) | GitHub App webhook HMAC secret. The hub verifies `/api/github/webhook` with it; a spoke verifies `/api/webhook/github` (direct or hub-relayed deliveries) with the same secret and rejects every delivery while it is unset. | +| `GITHUB_WEBHOOK_SECRET` | No | `/data/saas/webhook-secret.key` when present (hub only) | GitHub App webhook HMAC secret. The hub verifies `/api/github/webhook` with it; a spoke verifies `/api/webhook/github` (direct or hub-relayed deliveries) with the same secret and rejects every delivery while it is unset; such a spoke reports `/api/status.webhooks.mode_reason_code: no_secret` and a `webhooks` warning on `/api/health/deep` once deliveries arrive. | | `HIVE_DASHBOARD_URL` | No | none | Base URL the `hive tui` client targets (`pkg/tui/client`). A bad value surfaces as a request error on the first call, not at startup. On the hub it is also consulted (fourth) in the hub public-origin chain (see `HIVE_HUB_PUBLIC_URL`). | | `HIVE_CONVERGENCE_MODE` | No | `convergence.mode` in `hive.yaml`, else `shadow` | Process-level override of the convergence mode (`off`, `shadow`, `enforce`) so an operator can flip the mode without editing `hive.yaml`. An **unset** mode takes the default, `shadow`, which computes and records admission decisions without withholding any work. A **non-empty but unrecognised** value — a typo, or a mode this build does not know — still resolves to `off`. | | `HIVE_RELEASE_LINE_LAG_MAX` | No | `5` | Alarm threshold for the release-line drift surface ([#6960](https://github.com/hivecommons/hive/issues/6960)): how many commits the hosted edge line (`v5`) may sit behind the stable default branch (`v4`) before the dashboard status flags the line as drifting (`releaseLineLag.exceeded`). An unset, empty, non-numeric, or negative value falls back to the default — it can never silently disable the alarm. An unknown lag (tips unresolved or the compare failed) is reported as `unknown`, never a healthy zero. | diff --git a/src/docs/operator-reference/runtime-and-quota.md b/src/docs/operator-reference/runtime-and-quota.md index a68564f24b..04c69177ba 100644 --- a/src/docs/operator-reference/runtime-and-quota.md +++ b/src/docs/operator-reference/runtime-and-quota.md @@ -556,10 +556,13 @@ periodic housekeeping cannot starve the governor: `/api/status` and `/api/gh-rate-limits` expose `webhooks: {healthy, last_event_at, events_1h, invalidations_1h, -targeted_refreshes_1h, tracker_invalidations_1h, mode, since, mode_reason, +targeted_refreshes_1h, tracker_invalidations_1h, rejected_1h, +last_rejected_at, mode, since, mode_reason, mode_reason_code, healthy_windows}` (the middle two count the single-item re-reads and the -tracker cache drops webhooks caused; `mode` is `webhook` | `polling` and -`since` when it began — see [webhooks.md](../webhooks.md)); +tracker cache drops webhooks caused; `rejected_1h` and `last_rejected_at` +count deliveries the receiver turned away; `mode` is `webhook` | `polling` +and `since` when it began; `mode_reason_code` says why it is polling — see +[webhooks.md](../webhooks.md)); `/api/status` adds `evalMode` (`webhook` | `poll`), `evalIntervalS`, `lastWakeReason`, `lastWakeAt` and `webhookWakes`; `graphql_pr_batch.webhook_skips` counts batch queries skipped because nothing diff --git a/src/docs/webhooks.md b/src/docs/webhooks.md index 163196c375..d25dc814d8 100644 --- a/src/docs/webhooks.md +++ b/src/docs/webhooks.md @@ -132,10 +132,10 @@ Defaults: `src/pkg/config/defaults.go:30-32` (`DefaultEvalIntervalWebhookS`, `DefaultWebhookWakeDebounceS`, `DefaultWebhookWakeMinGapS`), `src/pkg/config/forgestate.go:8` (`DefaultForgeStateReconcileInterval`, `DefaultForgeStateFreshnessBound`, `DefaultForgeStateDriftThreshold`), -`src/pkg/github/webhook_cache.go:14` (`DefaultWebhookHealthWindow`, replaced -by `SetWebhookHealthWindow` at `src/pkg/github/webhook_cache.go:102` once the +`src/pkg/github/webhook_cache.go:17` (`DefaultWebhookHealthWindow`, replaced +by `SetWebhookHealthWindow` at `src/pkg/github/webhook_cache.go:206` once the governor loop publishes 2 × `eval_interval_s`), and -`src/pkg/github/webhook_mode.go:34` (`WebhookRecoveryHealthyWindows`). The +`src/pkg/github/webhook_mode.go:38` (`WebhookRecoveryHealthyWindows`). The effective tick is composed in `webhookAwareEvalInterval` (`src/cmd/hive/api_budget_decision.go:82`): the API-budget stretch still applies and the larger interval wins. @@ -158,7 +158,7 @@ forgestate: Webhook mode is only safe while deliveries actually arrive. The governor therefore keeps a small cadence **mode** machine on top of the raw health -window (`src/pkg/github/webhook_mode.go:100`, `observe`) and recomputes it +window (`src/pkg/github/webhook_mode.go:149`, `observe`) and recomputes it wherever it recomputes the eval interval (`evalIntervalAndModeForConfig`, `src/cmd/hive/api_budget_decision.go:115`), including the re-check timer that fires as the health window closes @@ -174,16 +174,16 @@ States and transitions: - the health window lapsed **and** GitHub was active meanwhile: a cadence pass listed an item whose `updated_at` is newer than the newest delivery (plus a 30 s clock-skew allowance, `webhookActivityClockSkew`, - `src/pkg/github/webhook_mode.go:40`). The reconcile pass reports that + `src/pkg/github/webhook_mode.go:44`). The reconcile pass reports that newest `updated_at` through `RecordGitHubActivity` - (`src/pkg/github/webhook_mode.go:183`). Silence alone — a quiet + (`src/pkg/github/webhook_mode.go:232`). Silence alone — a quiet repository overnight — is **not** degradation and keeps the stretched tick; or - deliveries are failing: the receiver saw a delivery (direct or relayed by the hub) whose signature did not verify or whose payload did not decode, newer than the last good delivery and inside the window - (`RecordWebhookDeliveryFailure`, `src/pkg/github/webhook_mode.go:199`, - called from `src/pkg/dashboard/api_github_webhook.go:58`). A wrong + (`RecordWebhookDeliveryFailure`, `src/pkg/github/webhook_mode.go:251`, + called from `src/pkg/dashboard/api_github_webhook.go:84`). A wrong `GITHUB_WEBHOOK_SECRET` on a spoke therefore shows up as polling mode with `mode_reason: deliveries failing verification`, not as a silent stretch to 900 s. @@ -208,8 +208,12 @@ The dashboard reads the same machine: - `/api/status.webhooks.mode` — `webhook` | `polling`; `/api/status.webhooks.since` — when the current mode began; - `mode_reason` while polling after a degradation; `healthy_windows` while - recovering (`src/pkg/github/webhook_cache.go:307`, `snapshot`). + `mode_reason` while polling after a degradation or while webhooks cannot + work at all; `mode_reason_code` (`no_secret`, `signature_failed`, + `deliveries_rejected`, `no_deliveries`) whenever it is polling for a + reason; `rejected_1h` and `last_rejected_at` for deliveries the receiver + turned away; `healthy_windows` while recovering + (`src/pkg/github/webhook_cache.go:430`, `snapshot`). `webhooks.healthy` stays the raw window (a delivery inside 2 × `eval_interval_s`), so `healthy: false, mode: webhook` reads "quiet but trusted" and `healthy: true, mode: polling` reads "recovering". @@ -228,7 +232,15 @@ Operator checks: `mode: webhook` on a busy repo with the App webhook pointed at the hub or the spoke; `polling` with a `mode_reason` is a configuration fault to fix (secret, App subscription, hub relay target), `polling` without one - means no delivery has ever arrived. + means no delivery has ever arrived. `mode_reason_code` tells the causes + apart: `no_secret` means `GITHUB_WEBHOOK_SECRET` is unset on this spoke + and every delivery, relayed or direct, is answered 503 (`rejected_1h` and + `last_rejected_at` show they are arriving); `signature_failed` means the + secret does not match the sender's; `deliveries_rejected` means signed + payloads did not decode; `no_deliveries` means the secret is set but + nothing verified has arrived, so look at the App webhook URL or the hub + relay. `GET /api/health/deep` carries the same cause as a `webhooks` + warning. 2. `grep 'webhooks degraded' hive.log` — one line per outage with the reason; the matching `webhooks healthy; webhook cadence engaged` closes it. diff --git a/src/pkg/dashboard/api_github_webhook.go b/src/pkg/dashboard/api_github_webhook.go index b8ad183a36..67996df260 100644 --- a/src/pkg/dashboard/api_github_webhook.go +++ b/src/pkg/dashboard/api_github_webhook.go @@ -23,7 +23,7 @@ const ( // GITHUB_WEBHOOK_SECRET and verifies X-Hub-Signature-256 over the raw body. githubWebhookPath = "/api/webhook/github" - githubWebhookSecretEnvVar = "GITHUB_WEBHOOK_SECRET" + githubWebhookSecretEnvVar = github.WebhookSecretEnvVar githubWebhookMaxBodyBytes = 1 << 20 ) @@ -71,12 +71,17 @@ func (s *Server) handleGitHubWebhook(w http.ResponseWriter, r *http.Request) { if !warned { s.webhookLogger().Debug(msg) } + // Counted like any other rejected delivery so /api/status.webhooks + // (mode_reason_code no_secret, rejected_1h, last_rejected_at) and + // /api/health/deep say why the spoke is polling; the log stays at one + // WARN per process (#13666). + github.RecordWebhookDeliveryFailure(github.WebhookReasonNoSecret) jsonError(w, "webhooks disabled: no secret configured", http.StatusServiceUnavailable) return } if !verifyGitHubWebhookSignature(body, r.Header.Get("X-Hub-Signature-256"), secret) { s.webhookLogger().Warn("github webhook signature verification failed", "event", event) - github.RecordWebhookDeliveryFailure() + github.RecordWebhookDeliveryFailure(github.WebhookReasonSignatureFailed) jsonError(w, "invalid signature", http.StatusUnauthorized) return } @@ -97,7 +102,7 @@ func (s *Server) handleGitHubWebhook(w http.ResponseWriter, r *http.Request) { } res, err := s.deps.GHClient.HandleWebhookEvent(event, body) if err != nil { - github.RecordWebhookDeliveryFailure() + github.RecordWebhookDeliveryFailure(github.WebhookReasonDeliveriesRejected) jsonError(w, "invalid webhook payload", http.StatusBadRequest) return } diff --git a/src/pkg/dashboard/api_github_webhook_reason_test.go b/src/pkg/dashboard/api_github_webhook_reason_test.go new file mode 100644 index 0000000000..f198843381 --- /dev/null +++ b/src/pkg/dashboard/api_github_webhook_reason_test.go @@ -0,0 +1,217 @@ +package dashboard + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/github" +) + +// readStatusWebhooks returns /api/status?fields=webhooks as a map. +func readStatusWebhooks(t *testing.T, s *Server) map[string]any { + t.Helper() + s.statusMu.Lock() + if s.status == nil { + s.status = &StatusPayload{HiveID: "webhook-reason-test"} + } + s.statusMu.Unlock() + rec := httptest.NewRecorder() + s.handleStatus(rec, httptest.NewRequest(http.MethodGet, "/api/status?fields=webhooks", nil)) + var status map[string]any + if err := json.Unmarshal(rec.Body.Bytes(), &status); err != nil { + t.Fatal(err) + } + webhooks, ok := status["webhooks"].(map[string]any) + if !ok { + t.Fatalf("/api/status has no webhooks block: %s", rec.Body.String()) + } + return webhooks +} + +// readDeepWebhookCheck returns the body of GET /api/health/deep and its +// `webhooks` check. +func readDeepWebhookCheck(t *testing.T, s *Server) (string, map[string]any) { + t.Helper() + rec := httptest.NewRecorder() + s.handleHealthDeep(rec, httptest.NewRequest(http.MethodGet, "/api/health/deep", nil)) + var out struct { + Status string `json:"status"` + Checks map[string]any `json:"checks"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &out); err != nil { + t.Fatal(err) + } + check, ok := out.Checks[healthCheckWebhooks].(map[string]any) + if !ok { + t.Fatalf("/api/health/deep has no %q check: %s", healthCheckWebhooks, rec.Body.String()) + } + return rec.Body.String(), check +} + +// summaryWebhookCheck returns the webhooks entry of the heartbeat/dashboard +// health summary (the navbar health tooltip reads it). +func summaryWebhookCheck(t *testing.T, s *Server) (status, detail string) { + t.Helper() + raw, err := json.Marshal(s.HealthSummary()) + if err != nil { + t.Fatal(err) + } + var out struct { + Checks []struct { + Name string `json:"name"` + Status string `json:"status"` + Detail string `json:"detail"` + } `json:"checks"` + } + if err := json.Unmarshal(raw, &out); err != nil { + t.Fatal(err) + } + for _, c := range out.Checks { + if c.Name == healthCheckWebhooks { + return c.Status, c.Detail + } + } + t.Fatalf("health summary has no %q check: %s", healthCheckWebhooks, raw) + return "", "" +} + +// A spoke with no GITHUB_WEBHOOK_SECRET answers every relayed delivery 503. +// That used to leave /api/status at `healthy:false, mode:polling` with no +// reason, the same as "GitHub is not sending". The status now names the +// cause and counts the rejections, while the log stays at one WARN. +func TestGitHubWebhookNoSecretReportsReasonAndCountsRejections(t *testing.T) { + t.Cleanup(github.ResetWebhookTrackerForTest()) + t.Setenv(githubWebhookSecretEnvVar, "") + s := newGitHubWebhookTestServer(t) + body := []byte(`{"number":5,"repository":{"full_name":"webhookorg/webhookrepo"}}`) + + for i := 0; i < 3; i++ { + if rec := postGitHubWebhook(s, "pull_request", "sha256=00", body); rec.Code != http.StatusServiceUnavailable { + t.Fatalf("attempt %d: status = %d, want 503", i, rec.Code) + } + } + got := readStatusWebhooks(t, s) + if got["mode"] != string(github.WebhookModePolling) || got["mode_reason_code"] != github.WebhookReasonNoSecret { + t.Fatalf("webhooks = %v, want polling with mode_reason_code %q", got, github.WebhookReasonNoSecret) + } + if reason, _ := got["mode_reason"].(string); !strings.Contains(reason, "no GITHUB_WEBHOOK_SECRET") || !strings.Contains(reason, "503") { + t.Fatalf("mode_reason = %q, want the plain-words no-secret cause", reason) + } + if n, _ := got["rejected_1h"].(float64); n != 3 { + t.Fatalf("rejected_1h = %v, want 3", got["rejected_1h"]) + } + if at, _ := got["last_rejected_at"].(string); at == "" { + t.Fatalf("last_rejected_at = %v, want the time of the last 503", got["last_rejected_at"]) + } else if _, err := time.Parse(time.RFC3339Nano, at); err != nil { + t.Fatalf("last_rejected_at = %q: %v", at, err) + } +} + +// The deep health check (health_deep_read) and the dashboard health summary +// carry the no-secret cause as a warning that never changes the verdict. +func TestHealthDeepWarnsWhenWebhooksRejectedForMissingSecret(t *testing.T) { + t.Cleanup(github.ResetWebhookTrackerForTest()) + t.Setenv(githubWebhookSecretEnvVar, "") + s := newGitHubWebhookTestServer(t) + s.statusMu.Lock() + s.status = &StatusPayload{HiveID: "webhook-health-test"} + s.ready = true + s.statusMu.Unlock() + + // Not configured and nobody sending: polling by design, not a warning. + if _, check := readDeepWebhookCheck(t, s); check["status"] != "pass" || check["mode_reason_code"] != github.WebhookReasonNoSecret { + t.Fatalf("before any delivery webhooks check = %v, want pass with no_secret", check) + } + + if rec := postGitHubWebhook(s, "pull_request", "", []byte(`{}`)); rec.Code != http.StatusServiceUnavailable { + t.Fatalf("status = %d, want 503", rec.Code) + } + body, check := readDeepWebhookCheck(t, s) + detail, _ := check["detail"].(string) + if check["status"] != "warn" || check["mode_reason_code"] != github.WebhookReasonNoSecret || + !strings.Contains(detail, "no GITHUB_WEBHOOK_SECRET on this spoke") || !strings.Contains(detail, "rejected with 503") { + t.Fatalf("webhooks check = %v, want a no-secret warning", check) + } + if !strings.Contains(body, `"status":"ok"`) { + t.Fatalf("a webhook warning must not change the overall verdict: %s", body) + } + if st, d := summaryWebhookCheck(t, s); st != "warn" || !strings.Contains(d, "no GITHUB_WEBHOOK_SECRET") { + t.Fatalf("health summary webhooks = %s %q, want the same warning", st, d) + } + // The summary feeds the header pill and DeepHealth: the warning is listed + // there but does not move its verdict either (#17173 review). + if sum := s.HealthSummary(); sum["warns"] != 0 || sum["status"] != "ok" { + t.Fatalf("health summary verdict = %v (warns %v), want ok with the webhook check not counted", sum["status"], sum["warns"]) + } +} + +// One rejection outside the hourly window must not keep the spoke at warn +// for good: the no-secret warning follows Rejected1h, not LastRejectedAt +// (#17173 review). +func TestWebhookHealthNoSecretWarnsOnlyWithinTheHour(t *testing.T) { + old := time.Now().Add(-26 * time.Hour) + snap := github.WebhookHealth{WebhookModeStatus: github.WebhookModeStatus{Mode: github.WebhookModePolling, ReasonCode: github.WebhookReasonNoSecret}, LastRejectedAt: &old} + if st, d := webhookHealth(snap); st != "pass" { + t.Fatalf("stale rejection: webhookHealth = %s %q, want pass", st, d) + } + snap.Rejected1h = 2 + if st, _ := webhookHealth(snap); st != "warn" { + t.Fatalf("rejections in the last hour: webhookHealth = %s, want warn", st) + } +} + +// With the secret set, the cause is "no deliveries" until a verified delivery +// arrives; then it clears and the check passes. The secret itself never +// appears in the status or health output. +func TestWebhookReasonClearsAfterVerifiedDelivery(t *testing.T) { + const secret = "s3cr3t-never-shown" + t.Cleanup(github.ResetWebhookTrackerForTest()) + t.Setenv(githubWebhookSecretEnvVar, secret) + s := newGitHubWebhookTestServer(t) + s.statusMu.Lock() + s.status = &StatusPayload{HiveID: "webhook-clear-test"} + s.ready = true + s.statusMu.Unlock() + + if got := readStatusWebhooks(t, s); got["mode_reason_code"] != github.WebhookReasonNoDeliveries { + t.Fatalf("before any delivery webhooks = %v, want mode_reason_code %q", got, github.WebhookReasonNoDeliveries) + } + + known := []byte(`{"number":5,"pull_request":{"number":5},"repository":{"full_name":"webhookorg/webhookrepo"}}`) + if rec := postGitHubWebhook(s, "pull_request", "sha256=00", known); rec.Code != http.StatusUnauthorized { + t.Fatalf("bad signature: status = %d, want 401", rec.Code) + } + if got := readStatusWebhooks(t, s); got["mode_reason_code"] != github.WebhookReasonSignatureFailed { + t.Fatalf("after a bad signature webhooks = %v, want mode_reason_code %q", got, github.WebhookReasonSignatureFailed) + } + if _, check := readDeepWebhookCheck(t, s); check["status"] != "warn" { + t.Fatalf("after a bad signature webhooks check = %v, want warn", check) + } + + if rec := postGitHubWebhook(s, "pull_request", signGitHubWebhookForTest(secret, known), known); rec.Code != http.StatusOK { + t.Fatalf("delivery: status = %d (%s)", rec.Code, rec.Body.String()) + } + if got := readStatusWebhooks(t, s); got["mode_reason_code"] != nil || got["healthy"] != true { + t.Fatalf("after a verified delivery webhooks = %v, want healthy with no reason code", got) + } + github.ObserveWebhookMode() + got := readStatusWebhooks(t, s) + if got["mode"] != string(github.WebhookModeWebhook) || got["mode_reason_code"] != nil { + t.Fatalf("after observe webhooks = %v, want webhook mode with no reason", got) + } + body, check := readDeepWebhookCheck(t, s) + if check["status"] != "pass" { + t.Fatalf("webhooks check = %v, want pass", check) + } + raw, _ := json.Marshal(got) + summary, _ := json.Marshal(s.HealthSummary()) + for _, out := range []string{string(raw), body, string(summary)} { + if strings.Contains(out, secret) { + t.Fatalf("webhook secret leaked into output: %s", out) + } + } +} diff --git a/src/pkg/dashboard/health_webhooks.go b/src/pkg/dashboard/health_webhooks.go new file mode 100644 index 0000000000..1c91c3f88f --- /dev/null +++ b/src/pkg/dashboard/health_webhooks.go @@ -0,0 +1,59 @@ +package dashboard + +import ( + "fmt" + + "github.com/hivecommons/hive/pkg/github" +) + +// Webhook receiver health. A hosted spoke that never received +// GITHUB_WEBHOOK_SECRET answers every relayed delivery with 503 and stays on +// the polling cadence; before this check the only trace was one WARN line +// per process, and /api/status read the same as "GitHub is not sending". +// The check names the cause (webhooks.mode_reason_code) as a warning: the +// hive still works on polling, so it is never a failure and never raises the +// hub's health-check-failing alert. +// +// Only whether the secret is set is ever reported, never its value, length +// or a digest of it. + +const healthCheckWebhooks = "webhooks" + +// webhookHealth grades a webhook snapshot: "warn" when deliveries arrive but +// cannot be used, or webhooks degraded while GitHub stayed busy; "pass" +// otherwise, with the mode and cause in the detail. +func webhookHealth(snap github.WebhookHealth) (status, detail string) { + if snap.Mode != github.WebhookModePolling { + return "pass", fmt.Sprintf("webhook mode — %d deliveries in the last hour", snap.Events1h) + } + rejected := fmt.Sprintf("%d rejected in the last hour", snap.Rejected1h) + switch snap.ReasonCode { + case github.WebhookReasonNoSecret: + // Only rejections inside the window count: one delivery days ago must + // not leave the spoke at warn for good. + if snap.Rejected1h > 0 { + return "warn", "polling — no GITHUB_WEBHOOK_SECRET on this spoke; relayed deliveries are rejected with 503 (" + rejected + ")" + } + return "pass", "polling — no GITHUB_WEBHOOK_SECRET on this spoke; webhooks are off and no delivery was rejected in the last hour" + case github.WebhookReasonSignatureFailed: + return "warn", "polling — deliveries fail signature verification: GITHUB_WEBHOOK_SECRET does not match the secret the sender signs with (" + rejected + ")" + case github.WebhookReasonDeliveriesRejected: + return "warn", "polling — deliveries arrive but are rejected as undecodable payloads (" + rejected + ")" + case github.WebhookReasonNoDeliveries: + if snap.Reason != "" { + return "warn", "polling — " + snap.Reason + "; check the GitHub App webhook URL or the hub relay" + } + return "pass", "polling — no verified delivery yet" + } + if snap.HealthyWindows > 0 { + return "pass", fmt.Sprintf("polling — recovering, %d of %d healthy windows", snap.HealthyWindows, github.WebhookRecoveryHealthyWindows) + } + return "pass", "polling — verified deliveries arriving; webhook mode engages at the next eval cycle" +} + +// webhookHealthCheck grades the live webhook tracker. +func webhookHealthCheck() (status, detail string, snap github.WebhookHealth) { + snap = github.WebhookHealthSnapshot() + status, detail = webhookHealth(snap) + return status, detail, snap +} diff --git a/src/pkg/dashboard/server.go b/src/pkg/dashboard/server.go index f09df0f037..20c165bb9d 100644 --- a/src/pkg/dashboard/server.go +++ b/src/pkg/dashboard/server.go @@ -3978,6 +3978,18 @@ func (s *Server) handleHealthDeep(w http.ResponseWriter, r *http.Request) { } } + // Webhook receiver: why the spoke is polling (no secret, deliveries + // failing, none arriving). A warning never changes the overall verdict: + // the hive still works on the polling cadence. + { + st, detail, snap := webhookHealthCheck() + wh := map[string]any{"status": st, "detail": detail, "mode": string(snap.Mode), "rejected_1h": snap.Rejected1h} + if snap.ReasonCode != "" { + wh["mode_reason_code"] = snap.ReasonCode + } + checks[healthCheckWebhooks] = wh + } + // 12. Queue trend (is work being processed?) s.statusMu.RLock() if s.status != nil { @@ -5330,6 +5342,13 @@ func (s *Server) healthSummaryFor(status *StatusPayload, ready bool) map[string] fails++ } + // Webhook receiver cause (see health_webhooks.go): listed, but it does not + // count toward warns, so it never changes the summary verdict that drives + // the header pill and DeepHealth — the hive works on polling. + if st, detail, _ := webhookHealthCheck(); st != "" { + checks = append(checks, check{Name: healthCheckWebhooks, Status: st, Detail: detail}) + } + // 8. Queue if status != nil { total := 0 diff --git a/src/pkg/github/webhook_cache.go b/src/pkg/github/webhook_cache.go index 5835d8137d..e4ce4db285 100644 --- a/src/pkg/github/webhook_cache.go +++ b/src/pkg/github/webhook_cache.go @@ -42,6 +42,11 @@ type WebhookHealth struct { CIEvents1h int `json:"ci_events_1h"` CIEventsApplied1h int `json:"ci_events_applied_1h"` CIRefetches1h int `json:"ci_refetches_1h"` + // Rejected1h counts deliveries the receiver turned away in the last hour + // (no secret configured, signature mismatch, undecodable payload) and + // LastRejectedAt is the newest one; mode_reason_code says which kind. + Rejected1h int `json:"rejected_1h"` + LastRejectedAt *time.Time `json:"last_rejected_at"` // WebhookModeStatus is the reconcile cadence the health drives: // `mode` webhook|polling and `since` (hivecommons/hive#13381). WebhookModeStatus @@ -137,7 +142,42 @@ type webhookTracker struct { // the cadence mode machine (webhook_mode.go). lastActivity time.Time lastFailure time.Time - mode webhookModeMachine + // lastFailureReason is the WebhookReason* code of lastFailure and + // rejected the per-minute count of rejected deliveries (rejected_1h). + lastFailureReason string + rejected webhookMinuteCounter + mode webhookModeMachine +} + +// webhookMinuteCounter counts events over the last hour in sixty one-minute +// buckets: fixed size, so recording a delivery never allocates however many +// arrive. +type webhookMinuteCounter struct { + minute [60]int64 + count [60]uint32 +} + +func (c *webhookMinuteCounter) add(now time.Time) { + m := now.Unix() / 60 + i := int(m % 60) + if c.minute[i] != m { + c.minute[i] = m + c.count[i] = 0 + } + c.count[i]++ +} + +// total is the count over the hour ending at now (the current minute and the +// 59 before it). +func (c *webhookMinuteCounter) total(now time.Time) int { + m := now.Unix() / 60 + n := 0 + for i := range c.minute { + if c.count[i] > 0 && c.minute[i] > m-60 && c.minute[i] <= m { + n += int(c.count[i]) + } + } + return n } func newWebhookTracker() *webhookTracker { @@ -402,7 +442,12 @@ func (t *webhookTracker) snapshot() WebhookHealth { out := WebhookHealth{Healthy: healthy, Events1h: len(t.events), Invalidations1h: len(t.invalidations), TargetedRefreshes1h: len(t.refreshes), TrackerInvalidations1h: len(t.trackerDrops), CIEvents1h: len(t.ciEvents), CIEventsApplied1h: len(t.ciApplied), CIRefetches1h: len(t.ciRefetches), - WebhookModeStatus: t.mode.status(t.window, now)} + Rejected1h: t.rejected.total(now), WebhookModeStatus: t.mode.status(t.window, now)} + t.annotateModeLocked(&out.WebhookModeStatus, now) + if !t.lastFailure.IsZero() { + at := t.lastFailure.UTC() + out.LastRejectedAt = &at + } if !latest.IsZero() { at := latest.UTC() out.LastEventAt = &at diff --git a/src/pkg/github/webhook_mode.go b/src/pkg/github/webhook_mode.go index 3750f8281b..6277b8d130 100644 --- a/src/pkg/github/webhook_mode.go +++ b/src/pkg/github/webhook_mode.go @@ -1,6 +1,10 @@ package github -import "time" +import ( + "os" + "strings" + "time" +) // Webhook-outage fallback to polling (hivecommons/hive#13381, ForgeState // phase 4). @@ -45,13 +49,58 @@ const ( WebhookDegradedReasonFailures = "deliveries failing verification" ) +// Machine-readable causes for polling, reported as webhooks.mode_reason_code +// so an operator (or a tool) can tell "this spoke cannot verify deliveries" +// from "GitHub is not sending any". They are also the reasons +// RecordWebhookDeliveryFailure takes for a rejected delivery. +const ( + // WebhookReasonNoSecret: GITHUB_WEBHOOK_SECRET is unset on this process, + // so every delivery (direct or hub-relayed) is answered 503. + WebhookReasonNoSecret = "no_secret" + // WebhookReasonSignatureFailed: deliveries arrive but their + // X-Hub-Signature-256 does not verify against the configured secret. + WebhookReasonSignatureFailed = "signature_failed" + // WebhookReasonDeliveriesRejected: deliveries arrive signed but are + // rejected for another reason (the payload did not decode). + WebhookReasonDeliveriesRejected = "deliveries_rejected" + // WebhookReasonNoDeliveries: the secret is set but no verified delivery + // is inside the health window (none ever, or none while GitHub was busy). + WebhookReasonNoDeliveries = "no_deliveries" +) + +// webhookReasonNoSecret*Text are the plain-words mode_reason for +// WebhookReasonNoSecret: with deliveries arriving, and with none yet. +const ( + webhookReasonNoSecretRejectedText = "no GITHUB_WEBHOOK_SECRET on this spoke; relayed deliveries are rejected with 503" + webhookReasonNoSecretIdleText = "no GITHUB_WEBHOOK_SECRET on this spoke; webhooks are off" +) + +// WebhookSecretEnvVar names the GitHub App webhook HMAC secret a spoke +// verifies /api/webhook/github deliveries with. +const WebhookSecretEnvVar = "GITHUB_WEBHOOK_SECRET" + +// WebhookSecretConfigured reports whether GITHUB_WEBHOOK_SECRET is set (non-blank) +// in this process. Only the yes/no ever leaves this function: the value, its +// length and any digest of it must never reach status, logs or health output. +func WebhookSecretConfigured() bool { + return strings.TrimSpace(os.Getenv(WebhookSecretEnvVar)) != "" +} + // WebhookModeStatus is the mode block /api/status serves inside `webhooks`. type WebhookModeStatus struct { Mode WebhookMode `json:"mode"` Since *time.Time `json:"since,omitempty"` - // Reason is set while polling after a degradation (empty at boot before - // webhooks were ever healthy, and in webhook mode). + // Reason is the plain-words cause while polling: the degradation reason + // after a degradation, or why webhooks cannot work at all (no secret, + // deliveries failing). Empty in webhook mode and while polling only + // because no delivery has arrived yet. Reason string `json:"mode_reason,omitempty"` + // ReasonCode is the machine-readable cause while polling: one of + // WebhookReasonNoSecret, WebhookReasonSignatureFailed, + // WebhookReasonDeliveriesRejected or WebhookReasonNoDeliveries. Empty in + // webhook mode, and while polling with a verified delivery in the window + // and no newer failure (about to engage webhook mode). + ReasonCode string `json:"mode_reason_code,omitempty"` // HealthyWindows counts the consecutive healthy windows observed while // polling after a degradation; recovery happens at // WebhookRecoveryHealthyWindows. Non-zero means "recovering". @@ -193,16 +242,58 @@ func RecordGitHubActivity(updatedAt time.Time) { } // RecordWebhookDeliveryFailure notes a delivery that reached the receiver -// (directly or via the hub relay) but could not be applied — a signature -// that did not verify or a payload that did not decode. Failures newer than -// the newest good delivery degrade the cadence to polling. -func RecordWebhookDeliveryFailure() { +// (directly or via the hub relay) but could not be applied, with why: +// WebhookReasonNoSecret (no secret to verify with), WebhookReasonSignatureFailed +// or WebhookReasonDeliveriesRejected (payload did not decode). Failures newer +// than the newest good delivery degrade the cadence to polling. Each call +// costs one counter bump in a fixed per-minute ring, never an allocation, so +// a flood of rejected deliveries cannot grow memory. +func RecordWebhookDeliveryFailure(reason string) { t := sharedWebhookTracker() t.mu.Lock() - t.lastFailure = t.now() + now := t.now() + t.lastFailure = now + t.lastFailureReason = reason + t.rejected.add(now) t.mu.Unlock() } +// annotateModeLocked fills the polling cause (mode_reason_code, and a +// plain-words mode_reason where the machine has none) on a mode status. +func (t *webhookTracker) annotateModeLocked(st *WebhookModeStatus, now time.Time) { + if st.Mode != WebhookModePolling { + return + } + healthy, latest := t.healthyLocked(now) + recentFailure := !t.lastFailure.IsZero() && now.Sub(t.lastFailure) < t.window && t.lastFailure.After(latest) + switch { + case !WebhookSecretConfigured(): + st.ReasonCode = WebhookReasonNoSecret + if st.Reason == "" { + st.Reason = webhookReasonNoSecretIdleText + if !t.lastFailure.IsZero() && t.lastFailureReason == WebhookReasonNoSecret { + st.Reason = webhookReasonNoSecretRejectedText + } + } + case recentFailure: + st.ReasonCode = webhookFailureCode(t.lastFailureReason) + if st.Reason == "" { + st.Reason = WebhookDegradedReasonFailures + } + case !healthy: + st.ReasonCode = WebhookReasonNoDeliveries + } +} + +// webhookFailureCode maps a recorded failure reason to the reported code; a +// failure recorded without a known reason reads as a rejected delivery. +func webhookFailureCode(reason string) string { + if reason == WebhookReasonSignatureFailed { + return reason + } + return WebhookReasonDeliveriesRejected +} + func (t *webhookTracker) modeInputLocked(now time.Time) webhookModeInput { healthy, latest := t.healthyLocked(now) in := webhookModeInput{healthy: healthy} @@ -219,13 +310,18 @@ func (t *webhookTracker) observeMode() webhookModeTransition { t.mu.Lock() defer t.mu.Unlock() now := t.now() - return t.mode.observe(t.modeInputLocked(now), t.window, now) + tr := t.mode.observe(t.modeInputLocked(now), t.window, now) + t.annotateModeLocked(&tr.Status, now) + return tr } func (t *webhookTracker) modeSnapshot() WebhookModeStatus { t.mu.Lock() defer t.mu.Unlock() - return t.mode.status(t.window, t.now()) + now := t.now() + st := t.mode.status(t.window, now) + t.annotateModeLocked(&st, now) + return st } // newestEnumeratedUpdate is the newest `updated_at` across one repo's listed diff --git a/src/pkg/github/webhook_mode_test.go b/src/pkg/github/webhook_mode_test.go index ae65829d65..0c60922cbe 100644 --- a/src/pkg/github/webhook_mode_test.go +++ b/src/pkg/github/webhook_mode_test.go @@ -118,7 +118,7 @@ func TestWebhookTrackerModeInputs(t *testing.T) { // A failure newer than the last good delivery degrades; one older than a // later good delivery does not. - RecordWebhookDeliveryFailure() + RecordWebhookDeliveryFailure(WebhookReasonSignatureFailed) if status, degraded, _ = ObserveWebhookMode(); status.Mode != WebhookModePolling || !degraded || status.Reason != WebhookDegradedReasonFailures { t.Fatalf("failure: %+v degraded=%v", status, degraded) } @@ -128,3 +128,58 @@ func TestWebhookTrackerModeInputs(t *testing.T) { t.Fatalf("good delivery after the failure should start the streak: %+v", status) } } + +// TestWebhookModeReasonCodes covers webhooks.mode_reason_code and the +// rejected-delivery counters: no secret wins, a failure newer than the last +// good delivery names its kind, silence is no_deliveries, and a verified +// delivery clears the code. rejected_1h ages out after an hour. +func TestWebhookModeReasonCodes(t *testing.T) { + clock := &webhookTestClock{now: time.Date(2026, 11, 7, 12, 0, 0, 0, time.UTC)} + t.Cleanup(resetWebhookTrackerForTest(clock.Now)) + SetWebhookHealthWindow(10 * time.Minute) + + t.Setenv(WebhookSecretEnvVar, " ") + if snap := WebhookHealthSnapshot(); snap.ReasonCode != WebhookReasonNoSecret || snap.Reason != webhookReasonNoSecretIdleText || snap.LastRejectedAt != nil { + t.Fatalf("blank secret, nothing sent: %+v", snap) + } + for i := 0; i < 4; i++ { + RecordWebhookDeliveryFailure(WebhookReasonNoSecret) + } + snap := WebhookHealthSnapshot() + if snap.ReasonCode != WebhookReasonNoSecret || snap.Reason != webhookReasonNoSecretRejectedText || snap.Rejected1h != 4 || + snap.LastRejectedAt == nil || !snap.LastRejectedAt.Equal(clock.now) { + t.Fatalf("no secret, deliveries rejected: %+v", snap) + } + if got := WebhookModeSnapshot(); got.ReasonCode != WebhookReasonNoSecret { + t.Fatalf("mode snapshot = %+v, want no_secret", got) + } + clock.now = clock.now.Add(59 * time.Minute) + if snap = WebhookHealthSnapshot(); snap.Rejected1h != 4 { + t.Fatalf("rejected_1h inside the hour = %d, want 4", snap.Rejected1h) + } + clock.now = clock.now.Add(time.Minute) + if snap = WebhookHealthSnapshot(); snap.Rejected1h != 0 || snap.LastRejectedAt == nil { + t.Fatalf("after an hour rejected_1h = %d (want 0), last_rejected_at = %v (want kept)", snap.Rejected1h, snap.LastRejectedAt) + } + + t.Setenv(WebhookSecretEnvVar, "set") + if snap = WebhookHealthSnapshot(); snap.ReasonCode != WebhookReasonNoDeliveries || snap.Reason != "" { + t.Fatalf("secret set, silent: %+v", snap) + } + RecordWebhookDeliveryFailure(WebhookReasonSignatureFailed) + if snap = WebhookHealthSnapshot(); snap.ReasonCode != WebhookReasonSignatureFailed || snap.Reason != WebhookDegradedReasonFailures { + t.Fatalf("bad signature: %+v", snap) + } + RecordWebhookDeliveryFailure(WebhookReasonDeliveriesRejected) + if snap = WebhookHealthSnapshot(); snap.ReasonCode != WebhookReasonDeliveriesRejected || snap.Rejected1h != 2 { + t.Fatalf("undecodable payload: %+v", snap) + } + clock.now = clock.now.Add(time.Second) + RecordWebhookEventForTest("org/repo") + if snap = WebhookHealthSnapshot(); snap.ReasonCode != "" || snap.Reason != "" || !snap.Healthy { + t.Fatalf("verified delivery should clear the cause: %+v", snap) + } + if status, _, recovered := ObserveWebhookMode(); status.Mode != WebhookModeWebhook || !recovered || status.ReasonCode != "" { + t.Fatalf("observe: %+v recovered=%v, want webhook mode with no code", status, recovered) + } +}