From 5a7fde8b2aec6cd66f2f6080e1546c34b42f9320 Mon Sep 17 00:00:00 2001 From: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Thu, 29 Oct 2026 00:48:05 -0400 Subject: [PATCH 1/2] =?UTF-8?q?=E2=9C=A8=20feat(webhook):=20react=20to=20f?= =?UTF-8?q?ailed=20CI=20checks=20within=20seconds=20of=20the=20delivery?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A failed check_run, check_suite or status webhook only invalidated the PR cache and scheduled a generic wake, so a red PR waited for the sweep and the next kick's ci-failing.json block - up to the 15 minute webhook-mode tick - before anyone reacted. The webhook now records a CI failure signal per open PR the head resolves to (PR detail cache only, deduplicated per head + check, served in /api/status webhooks.ci_failures) and the governor reacts at once: a targeted FIX-BEFORE-NEW kick to the owning agent of a hive-authored PR, an automerge pass for any other PR. governor.ci_failure_webhook_kick and governor.ci_failure_kick_cooldown_s control it; paused repos, the fleet breaker and paused agents are respected. Refs #15291 Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: clubanderson <407614+clubanderson@users.noreply.github.com> Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- changelog.d/added-ci-failure-webhook-kick.md | 1 + src/cmd/hive/boot.go | 3 + src/cmd/hive/ci_failure_kick.go | 246 ++++++++++++++++++ src/cmd/hive/ci_failure_kick_test.go | 169 ++++++++++++ src/cmd/hive/main.go | 2 + src/docs/webhooks.md | 2 + src/hive.yaml.example | 2 + src/pkg/config/config.go | 34 ++- src/pkg/config/validate.go | 3 + src/pkg/dashboard/api_github_webhook.go | 12 +- .../api_github_webhook_ci_failure_test.go | 69 +++++ src/pkg/dashboard/deps.go | 6 + src/pkg/github/webhook_cache.go | 35 ++- src/pkg/github/webhook_ci_failure.go | 203 +++++++++++++++ src/pkg/github/webhook_ci_failure_test.go | 69 +++++ .../policies/reviewer_lane_ci_poll_test.go | 14 + 16 files changed, 862 insertions(+), 8 deletions(-) create mode 100644 changelog.d/added-ci-failure-webhook-kick.md create mode 100644 src/cmd/hive/ci_failure_kick.go create mode 100644 src/cmd/hive/ci_failure_kick_test.go create mode 100644 src/pkg/dashboard/api_github_webhook_ci_failure_test.go create mode 100644 src/pkg/github/webhook_ci_failure.go create mode 100644 src/pkg/github/webhook_ci_failure_test.go diff --git a/changelog.d/added-ci-failure-webhook-kick.md b/changelog.d/added-ci-failure-webhook-kick.md new file mode 100644 index 0000000000..ee1aaf364f --- /dev/null +++ b/changelog.d/added-ci-failure-webhook-kick.md @@ -0,0 +1 @@ +- **A failed CI check now reaches the right agent within seconds of the webhook, not on the next sweep** ([#15291](https://github.com/hivecommons/hive/issues/15291)). A verified `check_run` / `check_suite` (`failure`, `timed_out`) or `status` (`failure`, `error`) delivery resolves its head SHA to open PRs from the existing PR detail cache (no extra GitHub call), records a CI failure signal deduplicated per head and check (`/api/status` `webhooks.ci_failures`: `count`, `last`), and reacts at once: a hive-authored PR's owning agent gets a targeted FIX-BEFORE-NEW kick naming the check and its log, any other PR gets an automerge pass, and the activity log reads `CI failed on #N (check) - kicked agent X`. New `governor.ci_failure_webhook_kick` (default `true`) and `governor.ci_failure_kick_cooldown_s` (default `600`, one reaction per PR head) control it; paused repos, an engaged fleet breaker and paused agents are respected. diff --git a/src/cmd/hive/boot.go b/src/cmd/hive/boot.go index e8621d5aeb..2da2163679 100644 --- a/src/cmd/hive/boot.go +++ b/src/cmd/hive/boot.go @@ -279,6 +279,9 @@ type boot struct { // sentinelEvents serializes the per-PR sentinel evaluations that // `pull_request` webhooks request (#13379). sentinelEvents *sentinelEventQueue + // ciFailures reacts to the CI failure signals failed check webhooks + // record (hivecommons/hive#15291). + ciFailures *ciFailureReactor // forgeView is the owner/name view over forgeState the automerge lanes // read and sweepTrigger their webhook trigger over it (#13377). forgeView *github.ForgeStateView diff --git a/src/cmd/hive/ci_failure_kick.go b/src/cmd/hive/ci_failure_kick.go new file mode 100644 index 0000000000..95b21297b0 --- /dev/null +++ b/src/cmd/hive/ci_failure_kick.go @@ -0,0 +1,246 @@ +package main + +import ( + "fmt" + "log/slog" + "strings" + "sync" + "time" + + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/escalation" + "github.com/hivecommons/hive/pkg/github" +) + +// Outcomes of one ciFailureReactor.React call, logged with the signal. +const ( + ciFailureKicked = "kicked" + ciFailureAutomergePass = "automerge_pass" + ciFailureDisabled = "disabled" + ciFailureRepoPaused = "repo_paused" + ciFailureCooldown = "cooldown" + ciFailureBreaker = "breaker_engaged" + ciFailureAgentUnavailable = "agent_unavailable" + ciFailureNoReceiver = "no_receiver" + ciFailureKickFailed = "kick_failed" +) + +// ciFailureReactor reacts to a CI failure signal a verified webhook recorded +// (hivecommons/hive#15291) within seconds instead of on the next sweep: +// +// - a hive-authored PR (audit agent_pr_created attribution) gets a +// targeted FIX-BEFORE-NEW kick to its owning agent, naming the failed +// check and its log; +// - any other PR (contributor or human) gets an automerge pass now, the +// same pass that routes a red contributor PR back to its author +// (stage 072) or to the rework lane. +// +// At most one reaction per PR head per governor.ci_failure_kick_cooldown_s. +// governor.ci_failure_webhook_kick=false, a paused repo, an engaged fleet +// breaker or an owner that cannot be kicked (paused, or paused in every +// governor mode) skips the reaction; the sweep and ci-failing.json still +// find the PR red as before. +type ciFailureReactor struct { + mu sync.Mutex + last map[string]time.Time + + now func() time.Time + cfg func() *config.Config + owners func(cfg *config.Config) map[string]string + available func(cfg *config.Config) func(string) bool + breaker func() bool + kicker fixKicker + nudge func(repo string, number int) + note func(repo string, number int, agent, line string) + logger *slog.Logger +} + +// React handles one signal and returns its outcome. +func (r *ciFailureReactor) React(sig github.CIFailureSignal) string { + outcome, agent := r.react(sig) + if r.logger != nil { + r.logger.Info("ci failure webhook reaction", "repo", sig.Repo, "pr", sig.Number, "head_sha", sig.HeadSHA, + "check", sig.Check, "outcome", outcome, "agent", agent) + } + return outcome +} + +func (r *ciFailureReactor) react(sig github.CIFailureSignal) (string, string) { + cfg := r.cfg() + if !cfg.CIFailureWebhookKickEnabled() { + return ciFailureDisabled, "" + } + if cfg.IsRepoPaused(sig.Repo) { + return ciFailureRepoPaused, "" + } + key := escalation.Key(strings.ToLower(sig.Repo), sig.Number) + "@" + sig.HeadSHA + if !r.reserve(key, cfg.CIFailureKickCooldown()) { + return ciFailureCooldown, "" + } + if r.breaker != nil && r.breaker() { + r.release(key) + return ciFailureBreaker, "" + } + owner := "" + if r.owners != nil { + owner = ownerFor(r.owners(cfg), sig.Repo, sig.Number) + } + if owner == "" { + if r.nudge == nil { + r.release(key) + return ciFailureNoReceiver, "" + } + r.nudge(sig.Repo, sig.Number) + r.record(sig, "", fmt.Sprintf("CI failed on #%d (%s) - scheduled an automerge pass", sig.Number, sig.Check)) + return ciFailureAutomergePass, "" + } + if r.available != nil { + if avail := r.available(cfg); avail != nil && !avail(owner) { + r.release(key) + return ciFailureAgentUnavailable, owner + } + } + if r.kicker == nil { + r.release(key) + return ciFailureNoReceiver, owner + } + if err := r.kicker.Kick(owner, ciFailureFixKick(sig, owner)); err != nil { + r.release(key) + if r.logger != nil { + r.logger.Warn("ci failure kick undeliverable", "repo", sig.Repo, "pr", sig.Number, "agent", owner, "error", err) + } + return ciFailureKickFailed, owner + } + r.record(sig, owner, fmt.Sprintf("CI failed on #%d (%s) - kicked agent %s", sig.Number, sig.Check, owner)) + return ciFailureKicked, owner +} + +// reserve claims the cooldown slot for a PR head, so concurrent signals for +// two checks on the same head react once. Expired slots are dropped. +func (r *ciFailureReactor) reserve(key string, cooldown time.Duration) bool { + r.mu.Lock() + defer r.mu.Unlock() + now := r.now() + if r.last == nil { + r.last = map[string]time.Time{} + } + for k, at := range r.last { + if now.Sub(at) >= cooldown { + delete(r.last, k) + } + } + if _, held := r.last[key]; held { + return false + } + r.last[key] = now + return true +} + +// release gives back a slot whose reaction did not happen, so a later +// signal for the same head can still react. +func (r *ciFailureReactor) release(key string) { + r.mu.Lock() + delete(r.last, key) + r.mu.Unlock() +} + +func (r *ciFailureReactor) record(sig github.CIFailureSignal, agent, line string) { + if r.logger != nil { + r.logger.Info(line, "repo", sig.Repo) + } + if r.note != nil { + r.note(sig.Repo, sig.Number, agent, line) + } +} + +// ownerFor looks a PR up in an auditPRAgents map, whose keys spell the repo +// as the audit entry did; GitHub repository names are case-insensitive. +func ownerFor(owners map[string]string, repo string, number int) string { + if a, ok := owners[escalation.Key(repo, number)]; ok { + return a + } + want := escalation.Key(repo, number) + for k, a := range owners { + if strings.EqualFold(k, want) { + return a + } + } + return "" +} + +// ciFailureFixKick is the targeted FIX-BEFORE-NEW kick for a webhook-reported +// failure: the same repair instructions the red-PR re-engagement kick +// carries, naming the failed check and pointing at its log. +func ciFailureFixKick(sig github.CIFailureSignal, owner string) string { + o := escalation.Observation{ + Repo: sig.Repo, + Number: sig.Number, + HeadSHA: sig.HeadSHA, + Agent: owner, + FailingChecks: []string{sig.Check}, + } + if sig.DetailsURL != "" { + o.Excerpt = "log: " + sig.DetailsURL + } + msg := redPRFixKick(o, false) + return strings.Replace(msg, "is red and has not moved", "just failed CI ("+sig.Check+")", 1) +} + +// installCIFailureReactor builds the reactor the dashboard webhook receiver +// hands CI failure signals to. Every dependency is resolved per signal +// because this runs before the agent manager and dashboard server exist. +// Idempotent, like installWebhookWake. +func (b *boot) installCIFailureReactor() { + if b.ciFailures != nil { + return + } + b.ciFailures = &ciFailureReactor{ + now: time.Now, + cfg: func() *config.Config { return b.cfg }, + owners: func(cfg *config.Config) map[string]string { + return auditPRAgents(cfg.Project.Org, time.Now().Add(-auditPRAttributionWindow), "") + }, + available: func(cfg *config.Config) func(string) bool { + if b.agentMgr == nil { + return func(string) bool { return false } + } + return agentAvailability(cfg, b.agentMgr) + }, + breaker: func() bool { + if b.agentMgr == nil { + return false + } + engaged, _ := b.agentMgr.BreakerState() + return engaged + }, + kicker: lazyAgentKicker{b: b}, + nudge: b.sweepNudgeFunc(), + note: func(repo string, number int, agent, line string) { + if b.dashSrv != nil { + b.dashSrv.AuditLogRecord("governor", "ci_failure_kick", line, agent, repo, number) + } + }, + logger: b.logger, + } +} + +// lazyAgentKicker resolves the agent manager at kick time. +type lazyAgentKicker struct{ b *boot } + +func (k lazyAgentKicker) Kick(agent, message string) error { + if k.b.agentMgr == nil { + return fmt.Errorf("agent manager unavailable: cannot kick %s", agent) + } + return agentKicker{mgr: k.b.agentMgr}.Kick(agent, message) +} + +// ciFailureFunc is the dashboard.Dependencies.CIFailure hook. The reaction +// (an audit read and possibly a kick) runs off the webhook handler's +// goroutine so GitHub gets its answer at once. +func (b *boot) ciFailureFunc() func(github.CIFailureSignal) { + if b.ciFailures == nil { + return nil + } + r := b.ciFailures + return func(sig github.CIFailureSignal) { go r.React(sig) } +} diff --git a/src/cmd/hive/ci_failure_kick_test.go b/src/cmd/hive/ci_failure_kick_test.go new file mode 100644 index 0000000000..dd5ebe767b --- /dev/null +++ b/src/cmd/hive/ci_failure_kick_test.go @@ -0,0 +1,169 @@ +package main + +import ( + "errors" + "strconv" + "strings" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/github" +) + +type ciFailureReactorHarness struct { + r *ciFailureReactor + cfg *config.Config + clock time.Time + kicker *fakeKicker + nudges []string + notes []string + paused map[string]bool + breaker bool +} + +func newCIFailureReactorHarness() *ciFailureReactorHarness { + h := &ciFailureReactorHarness{ + cfg: &config.Config{}, + clock: time.Date(2026, 10, 29, 4, 0, 0, 0, time.UTC), + kicker: &fakeKicker{}, + paused: map[string]bool{}, + } + h.cfg.Project.Org = "acme" + h.r = &ciFailureReactor{ + now: func() time.Time { return h.clock }, + cfg: func() *config.Config { return h.cfg }, + owners: func(*config.Config) map[string]string { + return map[string]string{"acme/widgets#7": "scanner"} + }, + available: func(*config.Config) func(string) bool { + return func(agent string) bool { return !h.paused[agent] } + }, + breaker: func() bool { return h.breaker }, + kicker: h.kicker, + nudge: func(repo string, number int) { + h.nudges = append(h.nudges, repo+"#"+strconv.Itoa(number)) + }, + note: func(_ string, _ int, _ string, line string) { h.notes = append(h.notes, line) }, + } + return h +} + +func ciSignal(number int, head, check string) github.CIFailureSignal { + return github.CIFailureSignal{Repo: "acme/widgets", Number: number, HeadSHA: head, Check: check, + DetailsURL: "https://ci.example/run/1", At: time.Date(2026, 10, 29, 4, 0, 0, 0, time.UTC)} +} + +// A failed check on a hive-authored PR kicks its owner at once with the +// FIX-BEFORE-NEW repair instructions, the check name and the log pointer, +// logs one activity line, and does not react again to the same head inside +// governor.ci_failure_kick_cooldown_s (hivecommons/hive#15291). +func TestCIFailureReactorKicksOwnerOncePerHeadWithinCooldown(t *testing.T) { + h := newCIFailureReactorHarness() + + if got := h.r.React(ciSignal(7, "sha-a", "unit")); got != ciFailureKicked { + t.Fatalf("first signal outcome = %q, want %q", got, ciFailureKicked) + } + if len(h.kicker.kicks) != 1 || h.kicker.kicks[0].agent != "scanner" { + t.Fatalf("kicks = %+v, want one to scanner", h.kicker.kicks) + } + msg := h.kicker.kicks[0].message + for _, want := range []string{"FIX-BEFORE-NEW", "acme/widgets#7 just failed CI (unit)", "failing: unit", "log: https://ci.example/run/1", "head: sha-a", "do not watch, poll, or sleep on CI"} { + if !strings.Contains(msg, want) { + t.Errorf("kick lacks %q:\n%s", want, msg) + } + } + if len(h.notes) != 1 || h.notes[0] != "CI failed on #7 (unit) - kicked agent scanner" { + t.Fatalf("activity lines = %q", h.notes) + } + + h.clock = h.clock.Add(time.Minute) + if got := h.r.React(ciSignal(7, "sha-a", "lint")); got != ciFailureCooldown { + t.Fatalf("second check on the same head = %q, want %q", got, ciFailureCooldown) + } + if got := h.r.React(ciSignal(7, "sha-b", "unit")); got != ciFailureKicked { + t.Fatalf("a new head = %q, want %q", got, ciFailureKicked) + } + h.clock = h.clock.Add(config.DefaultCIFailureKickCooldownS * time.Second) + if got := h.r.React(ciSignal(7, "sha-a", "lint")); got != ciFailureKicked { + t.Fatalf("after the cooldown = %q, want %q", got, ciFailureKicked) + } + if len(h.kicker.kicks) != 3 { + t.Fatalf("kicks = %d, want 3", len(h.kicker.kicks)) + } + + h.cfg.Governor.CIFailureKickCooldownS = 30 + h.clock = h.clock.Add(31 * time.Second) + if got := h.r.React(ciSignal(7, "sha-a", "e2e")); got != ciFailureKicked { + t.Fatalf("a configured 30s cooldown = %q, want %q", got, ciFailureKicked) + } +} + +// A failed check on a PR no hive agent opened schedules an automerge pass +// (the contributor-lane / stage 072 path) once per head per cooldown. +func TestCIFailureReactorSchedulesAutomergePassForContributorPR(t *testing.T) { + h := newCIFailureReactorHarness() + if got := h.r.React(ciSignal(12, "sha-c", "unit")); got != ciFailureAutomergePass { + t.Fatalf("outcome = %q, want %q", got, ciFailureAutomergePass) + } + if got := h.r.React(ciSignal(12, "sha-c", "lint")); got != ciFailureCooldown { + t.Fatalf("second check = %q, want %q", got, ciFailureCooldown) + } + if len(h.nudges) != 1 || h.nudges[0] != "acme/widgets#12" || len(h.kicker.kicks) != 0 { + t.Fatalf("nudges = %v kicks = %d, want one pass and no kick", h.nudges, len(h.kicker.kicks)) + } +} + +// Every guard skips the reaction without spending the head's cooldown slot, +// so the reaction happens once the guard lifts. +func TestCIFailureReactorGuards(t *testing.T) { + h := newCIFailureReactorHarness() + + off := false + h.cfg.Governor.CIFailureWebhookKick = &off + if got := h.r.React(ciSignal(7, "sha-a", "unit")); got != ciFailureDisabled { + t.Fatalf("disabled = %q", got) + } + h.cfg.Governor.CIFailureWebhookKick = nil + + h.cfg.Project.PausedRepos = []config.RepoPause{{Repo: "widgets"}} + if got := h.r.React(ciSignal(7, "sha-a", "unit")); got != ciFailureRepoPaused { + t.Fatalf("paused repo = %q", got) + } + if got := h.r.React(ciSignal(12, "sha-c", "unit")); got != ciFailureRepoPaused { + t.Fatalf("paused repo contributor PR = %q", got) + } + h.cfg.Project.PausedRepos = nil + + h.breaker = true + if got := h.r.React(ciSignal(7, "sha-a", "unit")); got != ciFailureBreaker { + t.Fatalf("breaker = %q", got) + } + h.breaker = false + + h.paused["scanner"] = true + if got := h.r.React(ciSignal(7, "sha-a", "unit")); got != ciFailureAgentUnavailable { + t.Fatalf("paused owner = %q", got) + } + h.paused["scanner"] = false + + h.kicker.err = errors.New("pane gone") + if got := h.r.React(ciSignal(7, "sha-a", "unit")); got != ciFailureKickFailed { + t.Fatalf("undeliverable = %q", got) + } + h.kicker.err = nil + + if len(h.kicker.kicks) != 0 || len(h.nudges) != 0 || len(h.notes) != 0 { + t.Fatalf("guards must not react: kicks=%d nudges=%v notes=%q", len(h.kicker.kicks), h.nudges, h.notes) + } + if got := h.r.React(ciSignal(7, "sha-a", "unit")); got != ciFailureKicked { + t.Fatalf("once every guard lifts = %q, want %q", got, ciFailureKicked) + } +} + +func TestCIFailureConfigDefaults(t *testing.T) { + var cfg *config.Config + if !cfg.CIFailureWebhookKickEnabled() || cfg.CIFailureKickCooldown() != 600*time.Second { + t.Fatalf("defaults: enabled=%v cooldown=%v, want true and 600s", cfg.CIFailureWebhookKickEnabled(), cfg.CIFailureKickCooldown()) + } +} diff --git a/src/cmd/hive/main.go b/src/cmd/hive/main.go index 5af466ce29..8d253c107c 100644 --- a/src/cmd/hive/main.go +++ b/src/cmd/hive/main.go @@ -1254,6 +1254,7 @@ func (b *boot) wireBootClosures() { b.installReviewEvents() b.installWebhookWake() b.installSentinelEvents() + b.installCIFailureReactor() b.heartbeatFleetStats = func() (*int, *int, *int, string) { var prsMerged, prsRejected, cvesClosed *int collectedAt := "" @@ -1478,6 +1479,7 @@ func (b *boot) wireBootClosures() { SweepNudge: b.sweepNudgeFunc(), ApplyWebhookItems: b.applyWebhookItemsFunc(), WebhookDelivery: b.webhookDeliveryFunc(), + CIFailure: b.ciFailureFunc(), EvalLoop: b.evalLoopStatusFunc(), SentinelPREvent: b.sentinelPREventFunc(), ProjectsV2ItemEvent: b.projectsV2ItemEventFunc(), diff --git a/src/docs/webhooks.md b/src/docs/webhooks.md index 338fb0eab6..5cd6033bf6 100644 --- a/src/docs/webhooks.md +++ b/src/docs/webhooks.md @@ -113,6 +113,8 @@ These are the acceptance criteria of #13373 and the tests that hold them. | `governor.eval_interval_max_s` | `1800` | cap on either tick after the API-budget stretch | | `governor.webhook_wake_debounce_s` | `10` | a delivery and its check fan-out coalesce into one wake cycle | | `governor.webhook_wake_min_gap_s` | `30` | minimum spacing between webhook-triggered cycles, and between the cycles recorded review verdicts trigger | +| `governor.ci_failure_webhook_kick` | `true` | a failed `check_run` / `check_suite` (`failure`, `timed_out`) or `status` (`failure`, `error`) on an open PR's head records a CI failure signal (`/api/status` `webhooks.ci_failures`: `count`, `last`) and reacts at once: a targeted FIX-BEFORE-NEW kick to the owning agent of a hive-authored PR, an automerge pass for any other PR. Heads are resolved from the PR detail cache only; a paused repo, an engaged fleet breaker or a paused owner skip the reaction and the sweep finds the PR red as before | +| `governor.ci_failure_kick_cooldown_s` | `600` | at most one such reaction per PR head in this window | | `forgestate.reconcile_interval_s` | `900` | ForgeState's own full-list interval per repo, shortened while drift is high | | `forgestate.freshness_bound_s` | `60` | maximum item age before merge/close/claim/kick re-reads it | | `forgestate.drift_threshold` | `0.05` | fraction of items a reconcile may find wrong before the interval shortens | diff --git a/src/hive.yaml.example b/src/hive.yaml.example index eeb0235f28..f9f0dc4cab 100644 --- a/src/hive.yaml.example +++ b/src/hive.yaml.example @@ -418,6 +418,8 @@ governor: eval_interval_webhook_s: 900 # Base interval while GitHub webhooks are healthy (see /api/webhook/github) webhook_wake_debounce_s: 10 # Quiet period after an invalidating webhook before the early eval cycle webhook_wake_min_gap_s: 30 # Minimum gap between webhook-triggered eval cycles + ci_failure_webhook_kick: true # Kick the owning agent (or run an automerge pass) when a webhook reports a failed check + ci_failure_kick_cooldown_s: 600 # At most one failed-check reaction per PR head in this window conserve_interval_multiplier: 2 # Eval interval multiplier in API budget conserve mode optional_sweep_every_n_cycles: 1 # Run optional sweeps every Nth normal cycle (1 = every cycle) diff --git a/src/pkg/config/config.go b/src/pkg/config/config.go index 0b5a521600..9268109a5e 100644 --- a/src/pkg/config/config.go +++ b/src/pkg/config/config.go @@ -1563,10 +1563,18 @@ type GovernorConfig struct { // cycle. WebhookWakeMinGapS is the minimum gap between two // webhook-triggered cycles. Both default at Load; see // docs/operator-reference.md "Webhook-driven PR cache invalidation". - WebhookWakeDebounceS int `yaml:"webhook_wake_debounce_s,omitempty"` - WebhookWakeMinGapS int `yaml:"webhook_wake_min_gap_s,omitempty"` - ConserveIntervalMultiplier int `yaml:"conserve_interval_multiplier,omitempty"` - OptionalSweepEveryNCycles int `yaml:"optional_sweep_every_n_cycles,omitempty"` + WebhookWakeDebounceS int `yaml:"webhook_wake_debounce_s,omitempty"` + WebhookWakeMinGapS int `yaml:"webhook_wake_min_gap_s,omitempty"` + // CIFailureWebhookKick turns on the immediate reaction to a failed + // check_run / check_suite / status webhook (hivecommons/hive#15291): a + // targeted fix kick to the owning agent of a hive-authored PR, an + // automerge pass for any other PR. nil (unset) means on. + // CIFailureKickCooldownS is the minimum gap between two such reactions + // for the same PR head (default DefaultCIFailureKickCooldownS). + CIFailureWebhookKick *bool `yaml:"ci_failure_webhook_kick,omitempty" json:"ci_failure_webhook_kick,omitempty"` + CIFailureKickCooldownS int `yaml:"ci_failure_kick_cooldown_s,omitempty" json:"ci_failure_kick_cooldown_s,omitempty"` + ConserveIntervalMultiplier int `yaml:"conserve_interval_multiplier,omitempty"` + OptionalSweepEveryNCycles int `yaml:"optional_sweep_every_n_cycles,omitempty"` // ExplainMode is the hive-wide default explain mode for agents that leave // their own explain_mode unset. "" means "no hive default configured", in // which case ExplainModeEnvVar is consulted and then off — see @@ -4895,6 +4903,23 @@ func (c *Config) WebhookWakeMinGap() time.Duration { return DefaultWebhookWakeMinGapS * time.Second } +// CIFailureWebhookKickEnabled reports governor.ci_failure_webhook_kick, +// which defaults to on when unset. +func (c *Config) CIFailureWebhookKickEnabled() bool { + if c == nil || c.Governor.CIFailureWebhookKick == nil { + return true + } + return *c.Governor.CIFailureWebhookKick +} + +// CIFailureKickCooldown is the effective governor.ci_failure_kick_cooldown_s. +func (c *Config) CIFailureKickCooldown() time.Duration { + if c != nil && c.Governor.CIFailureKickCooldownS > 0 { + return time.Duration(c.Governor.CIFailureKickCooldownS) * time.Second + } + return DefaultCIFailureKickCooldownS * time.Second +} + type DashboardIssueBandsConfig struct { WaitingLabels []string `yaml:"waiting_labels,omitempty" json:"waiting_labels,omitempty"` DoneLabels []string `yaml:"done_labels,omitempty" json:"done_labels,omitempty"` @@ -5822,6 +5847,7 @@ const ( DefaultEvalIntervalWebhookS = 900 DefaultWebhookWakeDebounceS = 10 DefaultWebhookWakeMinGapS = 30 + DefaultCIFailureKickCooldownS = 600 DefaultConserveIntervalMultiplier = 2 DefaultOptionalSweepEveryNCycles = 1 ) diff --git a/src/pkg/config/validate.go b/src/pkg/config/validate.go index 43315ecc59..0c251f7d1d 100644 --- a/src/pkg/config/validate.go +++ b/src/pkg/config/validate.go @@ -277,6 +277,9 @@ func (c *Config) ValidateWithOptions(opts ValidateOptions) error { if c.Governor.WebhookWakeMinGapS < 0 { return fmt.Errorf("governor.webhook_wake_min_gap_s must be at least 0") } + if c.Governor.CIFailureKickCooldownS < 0 { + return fmt.Errorf("governor.ci_failure_kick_cooldown_s must be at least 0") + } if c.Governor.ConserveIntervalMultiplier < 0 { return fmt.Errorf("governor.conserve_interval_multiplier must be at least 1") } diff --git a/src/pkg/dashboard/api_github_webhook.go b/src/pkg/dashboard/api_github_webhook.go index d42b61764a..80205d958d 100644 --- a/src/pkg/dashboard/api_github_webhook.go +++ b/src/pkg/dashboard/api_github_webhook.go @@ -42,7 +42,9 @@ func (s *Server) registerGitHubWebhookRoutes() { // duplicate-PR claim ledger on the event (hivecommons/hive#14814), so the // contribute lane stops offering them without waiting for the reconcile // pass; opened/synchronize/reopened also hands the PR to the sentinel -// (#13379), and the +// (#13379). A failed check_run, check_suite or status records a CI failure +// signal per open PR its head resolves to and hands each new one to the +// governor (hivecommons/hive#15291), and the // organization-scoped `projects_v2_item` delivery patches the GitHub // Projects work source cache. Deliveries for repositories this hive does not // manage are ignored. @@ -122,6 +124,14 @@ func (s *Server) handleGitHubWebhook(w http.ResponseWriter, r *http.Request) { if moved := s.noteForgeOutcome(event, body); moved { resp["outcome_counted"] = true } + if len(res.CIFailures) > 0 { + resp["ci_failures"] = len(res.CIFailures) + if s.deps.CIFailure != nil { + for _, sig := range res.CIFailures { + s.deps.CIFailure(sig) + } + } + } if reason, ok := webhookWakeReason(event, res); ok && s.deps.WebhookWake != nil { s.deps.WebhookWake(reason) resp["wake"] = reason diff --git a/src/pkg/dashboard/api_github_webhook_ci_failure_test.go b/src/pkg/dashboard/api_github_webhook_ci_failure_test.go new file mode 100644 index 0000000000..16b8917b20 --- /dev/null +++ b/src/pkg/dashboard/api_github_webhook_ci_failure_test.go @@ -0,0 +1,69 @@ +package dashboard + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/github" +) + +// A failed check_run delivery records one CI failure signal per PR, hands it +// to the governor hook once (a redelivery is deduplicated per head + check), +// and /api/status serves the count and the newest signal +// (hivecommons/hive#15291). +func TestGitHubWebhookFailedCheckRecordsCIFailureSignal(t *testing.T) { + now := time.Date(2026, 10, 29, 4, 0, 0, 0, time.UTC) + t.Cleanup(github.ResetWebhookTrackerForTestWithClock(func() time.Time { return now })) + const secret = "s3cr3t" + t.Setenv(githubWebhookSecretEnvVar, secret) + s := newGitHubWebhookTestServer(t) + s.status = &StatusPayload{HiveID: "webhook-test"} + var got []github.CIFailureSignal + s.deps.CIFailure = func(sig github.CIFailureSignal) { got = append(got, sig) } + + failed := []byte(`{"action":"completed","check_run":{"name":"unit","conclusion":"failure","head_sha":"abc123","details_url":"https://ci.example/run/7","pull_requests":[{"number":41}]},"repository":{"full_name":"webhookorg/webhookrepo"}}`) + for i := 0; i < 2; i++ { + rec := postGitHubWebhook(s, "check_run", signGitHubWebhookForTest(secret, failed), failed) + if rec.Code != http.StatusOK { + t.Fatalf("delivery %d: %d %s", i, rec.Code, rec.Body.String()) + } + if want := i == 0; strings.Contains(rec.Body.String(), `"ci_failures":1`) != want { + t.Fatalf("delivery %d: ci_failures reported = %v, want %v: %s", i, !want, want, rec.Body.String()) + } + } + if len(got) != 1 { + t.Fatalf("hook got %d signals, want 1 (redelivery deduplicated): %+v", len(got), got) + } + sig := got[0] + if sig.Repo != "webhookorg/webhookrepo" || sig.Number != 41 || sig.HeadSHA != "abc123" || sig.Check != "unit" || sig.DetailsURL != "https://ci.example/run/7" || !sig.At.Equal(now) { + t.Fatalf("signal = %+v", sig) + } + + passed := []byte(`{"action":"completed","check_run":{"name":"lint","conclusion":"success","head_sha":"abc123","pull_requests":[{"number":41}]},"repository":{"full_name":"webhookorg/webhookrepo"}}`) + if rec := postGitHubWebhook(s, "check_run", signGitHubWebhookForTest(secret, passed), passed); strings.Contains(rec.Body.String(), "ci_failures") { + t.Fatalf("a passing check must not record a signal: %s", rec.Body.String()) + } + status := []byte(`{"sha":"abc123","state":"error","context":"ci/legacy","target_url":"https://ci.example/legacy","repository":{"full_name":"webhookorg/webhookrepo"}}`) + if rec := postGitHubWebhook(s, "status", signGitHubWebhookForTest(secret, status), status); strings.Contains(rec.Body.String(), "ci_failures") { + t.Fatalf("a status whose head no cached PR holds is left to the sweep: %s", rec.Body.String()) + } + + statusRec := httptest.NewRecorder() + s.handleStatus(statusRec, httptest.NewRequest(http.MethodGet, "/api/status?fields=webhooks", nil)) + var payload struct { + Webhooks struct { + CIFailures github.CIFailureStatus `json:"ci_failures"` + } `json:"webhooks"` + } + if err := json.Unmarshal(statusRec.Body.Bytes(), &payload); err != nil { + t.Fatalf("decode /api/status: %v: %s", err, statusRec.Body.String()) + } + cf := payload.Webhooks.CIFailures + if cf.Count != 1 || cf.Last == nil || cf.Last.Number != 41 || cf.Last.Check != "unit" || cf.Last.HeadSHA != "abc123" { + t.Fatalf("/api/status webhooks.ci_failures = %+v, want count 1 and last #41 unit", cf) + } +} diff --git a/src/pkg/dashboard/deps.go b/src/pkg/dashboard/deps.go index 0b65cec71c..3445cf256c 100644 --- a/src/pkg/dashboard/deps.go +++ b/src/pkg/dashboard/deps.go @@ -293,6 +293,12 @@ type Dependencies struct { // named item and the automerge lanes schedule a pass for the affected // PRs (hivecommons/hive#13377). nil installs no receiver. WebhookDelivery func(event, deliveryID string, payload []byte, res ghpkg.WebhookEventResult) + // CIFailure receives each CI failure signal a verified failed + // check_run / check_suite / status delivery newly recorded + // (hivecommons/hive#15291), so the governor can kick the owning agent + // or schedule an automerge pass now. It must not block. nil installs + // no receiver; the signal still shows in /api/status. + CIFailure func(sig ghpkg.CIFailureSignal) // EvalLoop reports the governor loop's live pacing (poll vs webhook mode // and the last early wake) for /api/status. Nil omits the fields. EvalLoop func() EvalLoopStatus diff --git a/src/pkg/github/webhook_cache.go b/src/pkg/github/webhook_cache.go index fdd8ad785a..a9a31c6cb8 100644 --- a/src/pkg/github/webhook_cache.go +++ b/src/pkg/github/webhook_cache.go @@ -31,6 +31,9 @@ type WebhookHealth struct { // WebhookModeStatus is the reconcile cadence the health drives: // `mode` webhook|polling and `since` (hivecommons/hive#13381). WebhookModeStatus + // CIFailures is the CI failure signal store's count and newest signal + // (hivecommons/hive#15291). + CIFailures CIFailureStatus `json:"ci_failures"` } // WebhookEventResult reports what one delivered GitHub webhook did. @@ -61,6 +64,11 @@ type WebhookEventResult struct { // (closing keyword, branch name or reference) in its own repository, // as recorded into the claim ledger on the event (hivecommons/hive#14814). ClaimedIssues []int `json:"claimed_issues,omitempty"` + // CIFailures lists the CI failure signals a failed check_run, + // check_suite or status delivery newly recorded, one per open PR the + // head resolves to; a repeat of an already recorded head + check adds + // none (hivecommons/hive#15291). + CIFailures []CIFailureSignal `json:"ci_failures,omitempty"` // Item is the payload's own view of the issue or pull request the // delivery named (every `issues`, `issue_comment` and `pull_request*` // payload embeds the full object with its state), nil for deliveries @@ -109,6 +117,8 @@ type webhookTracker struct { lastActivity time.Time lastFailure time.Time mode webhookModeMachine + // ciFailures is the CI failure signal store (webhook_ci_failure.go). + ciFailures ciFailureStore } func newWebhookTracker() *webhookTracker { @@ -349,7 +359,7 @@ func (t *webhookTracker) snapshot() WebhookHealth { healthy, latest := t.healthyLocked(now) out := WebhookHealth{Healthy: healthy, Events1h: len(t.events), Invalidations1h: len(t.invalidations), TargetedRefreshes1h: len(t.refreshes), TrackerInvalidations1h: len(t.trackerDrops), - WebhookModeStatus: t.mode.status(t.window, now)} + WebhookModeStatus: t.mode.status(t.window, now), CIFailures: t.ciFailures.statusLocked(now)} if !latest.IsZero() { at := latest.UTC() out.LastEventAt = &at @@ -441,13 +451,26 @@ type webhookEventPayload struct { CheckSuite *struct { HeadSHA string `json:"head_sha"` PullRequests []webhookPRRef `json:"pull_requests"` + Conclusion string `json:"conclusion"` + App *struct { + Name string `json:"name"` + } `json:"app"` } `json:"check_suite"` CheckRun *struct { HeadSHA string `json:"head_sha"` PullRequests []webhookPRRef `json:"pull_requests"` + Name string `json:"name"` + Conclusion string `json:"conclusion"` + DetailsURL string `json:"details_url"` + HTMLURL string `json:"html_url"` } `json:"check_run"` - SHA string `json:"sha"` - Branches []struct { + SHA string `json:"sha"` + // State, Context and TargetURL are a `status` delivery's outcome, the + // status context and its log link. + State string `json:"state"` + Context string `json:"context"` + TargetURL string `json:"target_url"` + Branches []struct { Name string `json:"name"` } `json:"branches"` Ref string `json:"ref"` @@ -514,6 +537,12 @@ func (c *Client) HandleWebhookEvent(event string, payload []byte) (WebhookEventR c.RecordReviewEvidenceRevert(merged) } res.HeadSHA = webhookHeadSHA(event, p) + if check, detailsURL, ok := webhookCIFailure(event, p); ok && res.HeadSHA != "" { + // Resolved before the invalidation below drops the cached entries + // the head SHA is matched against. + prs := webhookCIFailurePRs(event, p, aliases, res.HeadSHA) + res.CIFailures = sharedWebhookTracker.recordCIFailures(p.Repository.FullName, prs, res.HeadSHA, check, detailsURL) + } numbers := webhookInvalidationTargets(event, p, aliases) for _, n := range numbers { c.invalidateWebhookPR(aliases, n) diff --git a/src/pkg/github/webhook_ci_failure.go b/src/pkg/github/webhook_ci_failure.go new file mode 100644 index 0000000000..ce698fab83 --- /dev/null +++ b/src/pkg/github/webhook_ci_failure.go @@ -0,0 +1,203 @@ +package github + +import ( + "sort" + "strconv" + "strings" + "time" +) + +// CIFailureSignal is one failed check a verified webhook delivery reported +// on an open PR's head (hivecommons/hive#15291): which PR, which head, which +// check, where its log is and when the delivery arrived. The governor reacts +// to it within seconds instead of waiting for the next sweep to find the PR +// red. +type CIFailureSignal struct { + Repo string `json:"repo"` + Number int `json:"number"` + HeadSHA string `json:"head_sha"` + Check string `json:"check"` + DetailsURL string `json:"details_url,omitempty"` + At time.Time `json:"at"` +} + +// CIFailureStatus is the `webhooks.ci_failures` block of /api/status: how +// many distinct (PR, head, check) failure signals are held and the newest. +type CIFailureStatus struct { + Count int `json:"count"` + Last *CIFailureSignal `json:"last,omitempty"` +} + +const ( + // ciFailureRetention is how long a signal is held. A head that is + // still red a day later is the sweep's (ci-failing.json) business. + ciFailureRetention = 24 * time.Hour + // ciFailureMaxSignals bounds the store against a check fan-out storm. + ciFailureMaxSignals = 512 +) + +// ciFailureStore is the small in-memory CI failure signal store, keyed by +// repo, PR, head SHA and check so a redelivery or a second failing +// check_suite/check_run for the same check on the same head records once. +// It lives on the webhook tracker and shares its lock and clock. +type ciFailureStore struct { + signals map[string]CIFailureSignal + last *CIFailureSignal +} + +func ciFailureKey(repo string, number int, headSHA, check string) string { + return strings.ToLower(repo) + "#" + strconv.Itoa(number) + "@" + headSHA + "|" + strings.ToLower(check) +} + +func (s *ciFailureStore) pruneLocked(now time.Time) { + for k, sig := range s.signals { + if now.Sub(sig.At) >= ciFailureRetention { + delete(s.signals, k) + } + } + if len(s.signals) <= ciFailureMaxSignals { + return + } + keys := make([]string, 0, len(s.signals)) + for k := range s.signals { + keys = append(keys, k) + } + sort.Slice(keys, func(i, j int) bool { return s.signals[keys[i]].At.Before(s.signals[keys[j]].At) }) + for _, k := range keys[:len(keys)-ciFailureMaxSignals] { + delete(s.signals, k) + } +} + +func (s *ciFailureStore) statusLocked(now time.Time) CIFailureStatus { + s.pruneLocked(now) + out := CIFailureStatus{Count: len(s.signals)} + if s.last != nil { + last := *s.last + out.Last = &last + } + return out +} + +// recordCIFailures records one signal per PR for a failed check on headSHA +// and returns the ones not already held. +func (t *webhookTracker) recordCIFailures(repo string, numbers []int, headSHA, check, detailsURL string) []CIFailureSignal { + if len(numbers) == 0 || headSHA == "" { + return nil + } + t.mu.Lock() + defer t.mu.Unlock() + now := t.now() + s := &t.ciFailures + if s.signals == nil { + s.signals = map[string]CIFailureSignal{} + } + s.pruneLocked(now) + var out []CIFailureSignal + for _, n := range numbers { + key := ciFailureKey(repo, n, headSHA, check) + if _, seen := s.signals[key]; seen { + continue + } + sig := CIFailureSignal{Repo: repo, Number: n, HeadSHA: headSHA, Check: check, DetailsURL: detailsURL, At: now.UTC()} + s.signals[key] = sig + last := sig + s.last = &last + out = append(out, sig) + } + s.pruneLocked(now) + return out +} + +// CIFailureSignalsSnapshot returns the CI failure signal store's status +// block (also served as /api/status webhooks.ci_failures). +func CIFailureSignalsSnapshot() CIFailureStatus { + t := sharedWebhookTracker + t.mu.Lock() + defer t.mu.Unlock() + return t.ciFailures.statusLocked(t.now()) +} + +// webhookCIFailure reports whether a delivery is a failed check and names +// the check and its log link: a completed check_run concluding failure or +// timed_out, a check_suite concluding failure or timed_out, or a status in +// state failure or error. Anything else (success, neutral, cancelled, +// pending, in-progress) is not a failure signal. +func webhookCIFailure(event string, p webhookEventPayload) (check, detailsURL string, ok bool) { + switch event { + case "check_run": + if p.CheckRun == nil || p.Action != "completed" || !ciFailedConclusion(p.CheckRun.Conclusion) { + return "", "", false + } + url := strings.TrimSpace(p.CheckRun.DetailsURL) + if url == "" { + url = strings.TrimSpace(p.CheckRun.HTMLURL) + } + return ciFailureCheckName(p.CheckRun.Name, event), url, true + case "check_suite": + if p.CheckSuite == nil || !ciFailedConclusion(p.CheckSuite.Conclusion) { + return "", "", false + } + name := "" + if p.CheckSuite.App != nil { + name = p.CheckSuite.App.Name + } + return ciFailureCheckName(name, event), "", true + case "status": + switch strings.ToLower(strings.TrimSpace(p.State)) { + case "failure", "error": + return ciFailureCheckName(p.Context, event), strings.TrimSpace(p.TargetURL), true + } + } + return "", "", false +} + +func ciFailedConclusion(c string) bool { + switch strings.ToLower(strings.TrimSpace(c)) { + case "failure", "timed_out": + return true + } + return false +} + +func ciFailureCheckName(name, event string) string { + if name = strings.TrimSpace(name); name != "" { + return name + } + return event +} + +// webhookCIFailurePRs resolves a failed head to open PRs from what the +// delivery and the PR detail cache already hold: the payload's own +// pull_requests list and every cached PR whose head is headSHA. No GitHub +// call is made; a head neither names is left to the sweep, which the +// invalidation's dirty mark already schedules. +func webhookCIFailurePRs(event string, p webhookEventPayload, aliases []string, headSHA string) []int { + set := map[int]bool{} + var refs []webhookPRRef + switch event { + case "check_run": + if p.CheckRun != nil { + refs = p.CheckRun.PullRequests + } + case "check_suite": + if p.CheckSuite != nil { + refs = p.CheckSuite.PullRequests + } + } + for _, r := range refs { + if r.Number > 0 { + set[r.Number] = true + } + } + for _, n := range sharedPRDetailCache.numbersMatching(aliases, func(sha, _, _ string) bool { return sha == headSHA }) { + if n > 0 { + set[n] = true + } + } + out := make([]int, 0, len(set)) + for n := range set { + out = append(out, n) + } + sort.Ints(out) + return out +} diff --git a/src/pkg/github/webhook_ci_failure_test.go b/src/pkg/github/webhook_ci_failure_test.go new file mode 100644 index 0000000000..2f30eeb129 --- /dev/null +++ b/src/pkg/github/webhook_ci_failure_test.go @@ -0,0 +1,69 @@ +package github + +import ( + "reflect" + "testing" + "time" +) + +// Failed check deliveries resolve their head SHA to open PRs from the PR +// detail cache (no GitHub call: setupWebhookCacheTest fails on any) and +// record one deduplicated signal per PR, head and check +// (hivecommons/hive#15291). +func TestHandleWebhookEventRecordsCIFailureSignals(t *testing.T) { + c, clock := setupWebhookCacheTest(t) + seedWebhookTestPRs(c, clock.now.Add(-time.Minute)) + + deliver := func(event, payload string) WebhookEventResult { + t.Helper() + res, err := c.HandleWebhookEvent(event, []byte(payload)) + if err != nil { + t.Fatalf("%s: %v", event, err) + } + return res + } + checks := func(res WebhookEventResult) []string { + var out []string + for _, s := range res.CIFailures { + out = append(out, s.Check) + } + return out + } + + // A fork PR's check_run carries no pull_requests: the cached head wins. + res := deliver("check_run", `{"action":"completed","check_run":{"name":"unit","conclusion":"timed_out","head_sha":"sha-2","html_url":"https://gh.example/runs/9","pull_requests":[]},"repository":{"full_name":"org/repo"}}`) + want := []CIFailureSignal{{Repo: "org/repo", Number: 2, HeadSHA: "sha-2", Check: "unit", DetailsURL: "https://gh.example/runs/9", At: clock.now}} + if !reflect.DeepEqual(res.CIFailures, want) { + t.Fatalf("check_run signals = %+v, want %+v", res.CIFailures, want) + } + if again := deliver("check_run", `{"action":"completed","check_run":{"name":"unit","conclusion":"failure","head_sha":"sha-2","pull_requests":[{"number":2}]},"repository":{"full_name":"org/repo"}}`); len(again.CIFailures) != 0 { + t.Fatalf("same head + check must record once, got %+v", again.CIFailures) + } + if got := checks(deliver("check_suite", `{"action":"completed","check_suite":{"conclusion":"failure","head_sha":"sha-1","app":{"name":"GitHub Actions"}},"repository":{"full_name":"org/repo"}}`)); !reflect.DeepEqual(got, []string{"GitHub Actions"}) { + t.Fatalf("check_suite checks = %v", got) + } + if got := checks(deliver("status", `{"sha":"sha-3","state":"failure","context":"ci/dco","target_url":"https://ci.example/dco","repository":{"full_name":"org/repo"}}`)); !reflect.DeepEqual(got, []string{"ci/dco"}) { + t.Fatalf("status checks = %v", got) + } + + for _, ignored := range []struct{ event, payload string }{ + {"check_run", `{"action":"created","check_run":{"name":"lint","conclusion":"failure","head_sha":"sha-1","pull_requests":[{"number":1}]},"repository":{"full_name":"org/repo"}}`}, + {"check_run", `{"action":"completed","check_run":{"name":"lint","conclusion":"cancelled","head_sha":"sha-1","pull_requests":[{"number":1}]},"repository":{"full_name":"org/repo"}}`}, + {"check_suite", `{"action":"completed","check_suite":{"conclusion":"success","head_sha":"sha-1","pull_requests":[{"number":1}]},"repository":{"full_name":"org/repo"}}`}, + {"status", `{"sha":"sha-1","state":"pending","context":"ci/x","repository":{"full_name":"org/repo"}}`}, + {"status", `{"sha":"unknown-head","state":"failure","context":"ci/x","repository":{"full_name":"org/repo"}}`}, + } { + if res := deliver(ignored.event, ignored.payload); len(res.CIFailures) != 0 { + t.Fatalf("%s %s recorded %+v, want none", ignored.event, ignored.payload, res.CIFailures) + } + } + + snap := WebhookHealthSnapshot().CIFailures + if snap.Count != 3 || snap.Last == nil || snap.Last.Check != "ci/dco" || snap.Last.Number != 3 { + t.Fatalf("ci_failures = %+v, want count 3, last #3 ci/dco", snap) + } + clock.now = clock.now.Add(ciFailureRetention) + if got := CIFailureSignalsSnapshot().Count; got != 0 { + t.Fatalf("signals older than the retention must be pruned, count = %d", got) + } +} diff --git a/src/pkg/policies/reviewer_lane_ci_poll_test.go b/src/pkg/policies/reviewer_lane_ci_poll_test.go index d39bae77a4..7534c7e2c7 100644 --- a/src/pkg/policies/reviewer_lane_ci_poll_test.go +++ b/src/pkg/policies/reviewer_lane_ci_poll_test.go @@ -20,3 +20,17 @@ func TestReviewerLaneForbidsCIPolling(t *testing.T) { } } } + +// Agents never watch CI: failures reach them out of band, from the webhook +// (hivecommons/hive#15291) or the sweep. The reviewer lane keeps its +// mandatory "Do Not Wait for CI" section, heading included. +func TestReviewerLaneKeepsDoNotWaitForCISection(t *testing.T) { + data, err := DefaultPolicies.ReadFile("defaults/reviewer-lane.md") + if err != nil { + t.Fatalf("read embedded reviewer-lane policy: %v", err) + } + const heading = "## After You Push: Do Not Wait for CI (MANDATORY)" + if !strings.Contains(string(data), heading) { + t.Fatalf("reviewer-lane policy lacks %q (#15291)", heading) + } +} From 644ecec69e4cc43f19bd82bc356610a0c9259602 Mon Sep 17 00:00:00 2001 From: ci-maintainer <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Thu, 29 Oct 2026 01:04:08 -0400 Subject: [PATCH 2/2] fix(webhook): allowlist ci_failure governor leaves; drop stale last on prune Add governor.ci_failure_webhook_kick and governor.ci_failure_kick_cooldown_s to settings_parity_allowlist.txt so TestSettingsDialogReferencesEveryHiveYAMLLeaf passes, and clear ciFailureStore.last once it ages past the retention so /api/status webhooks.ci_failures never reports count 0 with a stale last. Hive-Fix-Attempt: 1/6 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: ci-maintainer <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- src/pkg/dashboard/settings_parity_allowlist.txt | 2 ++ src/pkg/github/webhook_ci_failure.go | 3 +++ src/pkg/github/webhook_ci_failure_test.go | 4 ++-- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/pkg/dashboard/settings_parity_allowlist.txt b/src/pkg/dashboard/settings_parity_allowlist.txt index 5f02eb06aa..242896562c 100644 --- a/src/pkg/dashboard/settings_parity_allowlist.txt +++ b/src/pkg/dashboard/settings_parity_allowlist.txt @@ -199,6 +199,8 @@ governor.claims.contributor_flap_window_s # contributor flap cool-off (#13579); governor.claims.contributor_flap_cooloff_s # contributor flap cool-off (#13579); yaml-only governor.claims.contributor_starvation_s # contributor starvation guard (#13579); yaml-only governor.auto_close_superseded # superseded hive-PR auto-close; yaml-only +governor.ci_failure_webhook_kick # failed-check webhook reaction (#15291); yaml-only +governor.ci_failure_kick_cooldown_s # failed-check webhook reaction (#15291); yaml-only governor.cadence_owners. # TODO #11854 governor.gateways[].api_key_env # TODO #11854 governor.gateways[].api_key_file # TODO #11854 diff --git a/src/pkg/github/webhook_ci_failure.go b/src/pkg/github/webhook_ci_failure.go index ce698fab83..355fafe3bb 100644 --- a/src/pkg/github/webhook_ci_failure.go +++ b/src/pkg/github/webhook_ci_failure.go @@ -55,6 +55,9 @@ func (s *ciFailureStore) pruneLocked(now time.Time) { delete(s.signals, k) } } + if s.last != nil && now.Sub(s.last.At) >= ciFailureRetention { + s.last = nil + } if len(s.signals) <= ciFailureMaxSignals { return } diff --git a/src/pkg/github/webhook_ci_failure_test.go b/src/pkg/github/webhook_ci_failure_test.go index 2f30eeb129..5410c36120 100644 --- a/src/pkg/github/webhook_ci_failure_test.go +++ b/src/pkg/github/webhook_ci_failure_test.go @@ -63,7 +63,7 @@ func TestHandleWebhookEventRecordsCIFailureSignals(t *testing.T) { t.Fatalf("ci_failures = %+v, want count 3, last #3 ci/dco", snap) } clock.now = clock.now.Add(ciFailureRetention) - if got := CIFailureSignalsSnapshot().Count; got != 0 { - t.Fatalf("signals older than the retention must be pruned, count = %d", got) + if got := CIFailureSignalsSnapshot(); got.Count != 0 || got.Last != nil { + t.Fatalf("signals older than the retention must be pruned, got %+v", got) } }