diff --git a/changelog.d/added-ci-failure-webhook-kick.md b/changelog.d/added-ci-failure-webhook-kick.md new file mode 100644 index 0000000000..56b5fe8644 --- /dev/null +++ b/changelog.d/added-ci-failure-webhook-kick.md @@ -0,0 +1 @@ +- A failed `check_run`, `check_suite` or `status` webhook now reacts within seconds instead of waiting for the next governor sweep: the head SHA is resolved to its open PR from the PR detail cache, recorded as a CI failure signal (`/api/status.ci_failure_signals`), and a hive-authored PR's owning agent is kicked at once with the check name and log URL while a contributor PR gets a targeted automerge/contributor-lane pass — gated by `governor.ci_failure_webhook_kick` (default on), repo/agent pauses and a per-head cooldown (`governor.ci_failure_kick_cooldown_s`, default 600). Agents still never watch or poll CI themselves. diff --git a/src/cmd/hive/ci_failure_kick.go b/src/cmd/hive/ci_failure_kick.go new file mode 100644 index 0000000000..1830316665 --- /dev/null +++ b/src/cmd/hive/ci_failure_kick.go @@ -0,0 +1,167 @@ +package main + +import ( + "context" + "errors" + "log/slog" + "strings" + "time" + + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/dashboard" + "github.com/hivecommons/hive/pkg/github" + "github.com/hivecommons/hive/pkg/governor" +) + +// ciFailureKickDefaultAgent is the agent an unattributed hive-authored PR is +// routed to, matching the fix-before-new builder's default (scanner is the +// fleet's primary PR creator). +const ciFailureKickDefaultAgent = "scanner" + +// ciFailureKickSendTimeout bounds one targeted kick delivery; a stuck pane +// must not pin the worker. +const ciFailureKickSendTimeout = 2 * time.Minute + +var errNoAgentRuntime = errors.New("agent runtime not available") + +// ciFailureKicker turns a webhook CI-failure signal on a hive-authored PR +// into an immediate targeted kick of the owning agent. Gating mirrors the +// CEL trigger path (celTriggerKickAgents): config flag, agent enabled and +// not paused in config, runtime pause (which the fleet breaker also sets), +// governor eligibility (cadence pause, on-demand, budget), repo pause, and +// the per-PR-head cooldown kept by the signal store. The kick itself runs +// off the webhook goroutine so GitHub gets its 200 at once. +type ciFailureKicker struct { + cfg func() *config.Config + gov *governor.Governor + isPaused func(agent string) bool + message func(agent string, sig github.CIFailureSignal) string + attribute func(repo string, number int) string + send func(ctx context.Context, agent, message string) error + logger *slog.Logger + ctx context.Context + // spawn runs the kick delivery; tests swap in a synchronous runner. + spawn func(func()) +} + +func (b *boot) newCIFailureKicker() *ciFailureKicker { + k := &ciFailureKicker{ + cfg: func() *config.Config { return b.cfg }, + gov: b.gov, + attribute: github.AuthorAgentForPR, + logger: b.logger, + ctx: b.ctx, + spawn: func(fn func()) { go fn() }, + } + // Resolve the scheduler and runtime per call: the dependencies closure + // may be built before boot finishes wiring them. + k.message = func(agent string, sig github.CIFailureSignal) string { + if b.sched == nil { + return "" + } + return b.sched.CIFailureKickMessage(agent, sig) + } + k.isPaused = func(agent string) bool { return b.agentMgr != nil && b.agentMgr.IsPaused(agent) } + k.send = func(_ context.Context, agent, message string) error { + if b.agentMgr == nil { + return errNoAgentRuntime + } + return b.agentMgr.SendKick(agent, message) + } + return k +} + +// ciFailureKickFunc is the dashboard.Dependencies.CIFailureKick hook. +func (b *boot) ciFailureKickFunc() func(sig github.CIFailureSignal) dashboard.CIFailureKickOutcome { + k := b.newCIFailureKicker() + return k.React +} + +// React decides and, when every gate passes, dispatches the kick. +func (k *ciFailureKicker) React(sig github.CIFailureSignal) dashboard.CIFailureKickOutcome { + if k == nil { + return dashboard.CIFailureKickOutcome{Reason: "no kicker"} + } + logger := k.logger + if logger == nil { + logger = slog.Default() + } + cfg := k.cfg() + if cfg == nil { + return dashboard.CIFailureKickOutcome{Reason: "no config"} + } + if !cfg.CIFailureWebhookKick() { + return dashboard.CIFailureKickOutcome{Reason: "disabled"} + } + agent := "" + if k.attribute != nil { + agent = strings.TrimSpace(k.attribute(sig.Repo, sig.Number)) + } + if agent == "" { + agent = ciFailureKickDefaultAgent + } + out := dashboard.CIFailureKickOutcome{Agent: agent} + ac, ok := cfg.Agents[agent] + if !ok { + out.Reason = "unknown agent" + return out + } + if !ac.Enabled || ac.Paused { + out.Reason = "agent paused" + return out + } + if k.isPaused != nil && k.isPaused(agent) { + out.Reason = "agent paused" + return out + } + if cfg.IsRepoPaused(sig.Repo) { + out.Reason = "repo paused" + return out + } + if k.gov != nil && !k.gov.AgentEligibleForCELKick(agent) { + out.Reason = "governor gated" + return out + } + // Attribute before rendering so the fix-before-new builder lists the + // signal under this agent. + github.CIFailureSignals().SetAgent(sig.Repo, sig.Number, agent) + sig.Agent = agent + message := "" + if k.message != nil { + message = k.message(agent, sig) + } + if strings.TrimSpace(message) == "" { + out.Reason = "nothing to repair" + return out + } + if !github.CIFailureSignals().AllowKick(sig.Repo, sig.Number, sig.HeadSHA, cfg.CIFailureKickCooldown()) { + out.Reason = "cooldown" + return out + } + if k.send == nil { + out.Reason = "no kick sender" + return out + } + out.Kicked = true + spawn := k.spawn + if spawn == nil { + spawn = func(fn func()) { go fn() } + } + spawn(func() { + parent := k.ctx + if parent == nil { + parent = context.Background() + } + ctx, cancel := context.WithTimeout(parent, ciFailureKickSendTimeout) + defer cancel() + if err := k.send(ctx, agent, message); err != nil { + logger.Warn("ci failure webhook kick failed", "agent", agent, "repo", sig.Repo, "number", sig.Number, "check", sig.Check, "error", err) + return + } + if k.gov != nil { + k.gov.RecordKickForRepo(agent, sig.Repo) + } + logger.Info("ci failure webhook kick delivered", "agent", agent, "repo", sig.Repo, "number", sig.Number, "check", sig.Check) + }) + return out +} diff --git a/src/cmd/hive/ci_failure_kick_test.go b/src/cmd/hive/ci_failure_kick_test.go new file mode 100644 index 0000000000..01106228f0 --- /dev/null +++ b/src/cmd/hive/ci_failure_kick_test.go @@ -0,0 +1,149 @@ +package main + +import ( + "context" + "errors" + "log/slog" + "os" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/github" + "github.com/hivecommons/hive/pkg/governor" +) + +type ciKickRecord struct { + agent, message string +} + +func newCIFailureKickerForTest(t *testing.T, cfg *config.Config) (*ciFailureKicker, *[]ciKickRecord) { + t.Helper() + var sent []ciKickRecord + logger := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelError})) + k := &ciFailureKicker{ + cfg: func() *config.Config { return cfg }, + gov: governor.New(cfg.Governor, cfg.EnabledAgents(), logger), + attribute: func(repo string, number int) string { return "quality" }, + message: func(agent string, sig github.CIFailureSignal) string { + return "CI failed on " + sig.Repo + " for " + agent + }, + send: func(_ context.Context, agent, message string) error { + sent = append(sent, ciKickRecord{agent: agent, message: message}) + return nil + }, + logger: logger, + ctx: context.Background(), + spawn: func(fn func()) { fn() }, + } + return k, &sent +} + +func ciKickTestConfig() *config.Config { + return &config.Config{ + Project: config.ProjectConfig{Org: "acme", Repos: []string{"widgets"}}, + AgentRuntimeConfig: config.AgentRuntimeConfig{ + Agents: map[string]config.AgentConfig{ + "quality": {Enabled: true, Mode: "ISSUES_AND_PRS"}, + "scanner": {Enabled: true, Mode: "ISSUES_AND_PRS"}, + }, + }, + } +} + +// The webhook kick honours every gate the CEL trigger path honours — flag, +// config pause, runtime pause, repo pause, governor eligibility — plus the +// per-head cooldown, and otherwise delivers one targeted kick to the agent +// the audit log attributes the PR to (scanner when unattributed). +func TestCIFailureKicker_GatesAndKicks(t *testing.T) { + now := time.Date(2026, 10, 21, 12, 0, 0, 0, time.UTC) + clock := func() time.Time { return now } + t.Cleanup(github.ResetCIFailureSignalsForTest(clock)) + cfg := ciKickTestConfig() + k, sent := newCIFailureKickerForTest(t, cfg) + sig := github.CIFailureSignal{Repo: "acme/widgets", Number: 7, HeadSHA: "abc", Check: "build"} + + out := k.React(sig) + if !out.Kicked || out.Agent != "quality" || len(*sent) != 1 || (*sent)[0].agent != "quality" { + t.Fatalf("first kick: %+v, sent = %+v", out, *sent) + } + if got := github.CIFailureSignals().Snapshot().Kicks; got != 1 { + t.Fatalf("kicks recorded = %d", got) + } + + // Same head again inside the cooldown: refused. + if out := k.React(sig); out.Kicked || out.Reason != "cooldown" { + t.Fatalf("cooldown: %+v", out) + } + // New head: a fresh kick. + sig2 := sig + sig2.HeadSHA = "def" + if out := k.React(sig2); !out.Kicked || len(*sent) != 2 { + t.Fatalf("new head: %+v, sent = %d", out, len(*sent)) + } + // Past the cooldown the original head may kick again. + now = now.Add(cfg.CIFailureKickCooldown() + time.Second) + if out := k.React(sig); !out.Kicked || len(*sent) != 3 { + t.Fatalf("after cooldown: %+v, sent = %d", out, len(*sent)) + } + + sig3 := sig + sig3.HeadSHA = "ghi" + ac := cfg.Agents["quality"] + ac.Paused = true + cfg.Agents["quality"] = ac + if out := k.React(sig3); out.Kicked || out.Reason != "agent paused" { + t.Fatalf("config pause: %+v", out) + } + ac.Paused = false + cfg.Agents["quality"] = ac + + k.isPaused = func(string) bool { return true } + if out := k.React(sig3); out.Kicked || out.Reason != "agent paused" { + t.Fatalf("runtime pause: %+v", out) + } + k.isPaused = nil + + cfg.Project.PausedRepos = []config.RepoPause{{Repo: "widgets"}} + if out := k.React(sig3); out.Kicked || out.Reason != "repo paused" { + t.Fatalf("repo pause: %+v", out) + } + cfg.Project.PausedRepos = nil + + off := false + cfg.Governor.CIFailureWebhookKick = &off + if out := k.React(sig3); out.Kicked || out.Reason != "disabled" { + t.Fatalf("flag off: %+v", out) + } + cfg.Governor.CIFailureWebhookKick = nil + + k.message = func(string, github.CIFailureSignal) string { return "" } + if out := k.React(sig3); out.Kicked || out.Reason != "nothing to repair" { + t.Fatalf("empty message: %+v", out) + } + k.message = func(agent string, _ github.CIFailureSignal) string { return "m" } + + // Unattributed PR routes to scanner; an unknown agent is not kicked. + k.attribute = func(string, int) string { return "" } + if out := k.React(sig3); !out.Kicked || out.Agent != ciFailureKickDefaultAgent { + t.Fatalf("default agent: %+v", out) + } + k.attribute = func(string, int) string { return "ghost" } + sig4 := sig + sig4.HeadSHA = "jkl" + if out := k.React(sig4); out.Kicked || out.Reason != "unknown agent" { + t.Fatalf("unknown agent: %+v", out) + } + + // A delivery failure is logged, not surfaced as a kick. + k.attribute = func(string, int) string { return "quality" } + k.send = func(context.Context, string, string) error { return errors.New("pane gone") } + sig5 := sig + sig5.HeadSHA = "mno" + if out := k.React(sig5); !out.Kicked { + t.Fatalf("send failure must still report dispatch: %+v", out) + } + if len(*sent) != 4 { + t.Fatalf("sent = %d, want 4", len(*sent)) + } +} diff --git a/src/cmd/hive/forgestate_wire.go b/src/cmd/hive/forgestate_wire.go index 057388247b..531f089341 100644 --- a/src/cmd/hive/forgestate_wire.go +++ b/src/cmd/hive/forgestate_wire.go @@ -25,6 +25,12 @@ func (b *boot) forgeStateOpenPRs() *github.ForgeStateView { } view := b.ghClient.ForgeStateView() if view == nil { + // No cache: the lanes run their listing ticker, but a trigger is + // still wired so a webhook CI-failure nudge runs a targeted pass + // at once (the legacy loop selects on it too). + if b.sweepTrigger == nil { + b.sweepTrigger = automerge.NewSweepTrigger(nil) + } return nil } b.forgeView = view @@ -50,8 +56,8 @@ func (b *boot) webhookDeliveryFunc() func(event, deliveryID string, payload []by // sweepNudgeFunc is the dashboard.Dependencies.SweepNudge hook: schedule // an automerge pass for one PR. The trigger is resolved per call because -// it is wired before the App client exists; without the forgestate cache -// the governor loop is woken instead, whose eval cycle runs the lanes. +// it is wired before the App client exists; before it is, the governor +// loop is woken instead, whose eval cycle runs the lanes. func (b *boot) sweepNudgeFunc() func(repo string, number int) { return func(repo string, number int) { if trigger := b.sweepTrigger; trigger != nil { diff --git a/src/cmd/hive/main.go b/src/cmd/hive/main.go index de6d1cfeb0..48741da2c7 100644 --- a/src/cmd/hive/main.go +++ b/src/cmd/hive/main.go @@ -1481,6 +1481,7 @@ func (b *boot) wireBootClosures() { WebhookDelivery: b.webhookDeliveryFunc(), EvalLoop: b.evalLoopStatusFunc(), SentinelPREvent: b.sentinelPREventFunc(), + CIFailureKick: b.ciFailureKickFunc(), ProjectsV2ItemEvent: b.projectsV2ItemEventFunc(), GHAppAuth: b.appAuth, GHTokenScopes: b.ghAuth.TokenScopes, diff --git a/src/docs/api-reference.md b/src/docs/api-reference.md index a2b6f292a4..b4cfe73489 100644 --- a/src/docs/api-reference.md +++ b/src/docs/api-reference.md @@ -17,7 +17,7 @@ History point limits, timestamp cutoffs, incremental cost reads and ETag/304 beh | `GET` | `/api/health/deep` | Public | Deep health probe | `pkg/dashboard/server.go:1594` | | `GET` | `/api/livez` | Public | Kubernetes liveness probe | `pkg/dashboard/server.go:1595` | | `GET` | `/metrics` | Registered only when `HIVE_METRICS_ENABLED`; requires `Authorization: ****** (403 if the token is unset) | Prometheus metrics | `pkg/dashboard/server.go:1601` | -| `GET` | `/api/status` | Dashboard auth/session | Dashboard aggregate status; `labelPresets[]` (`{key,label,description,labels[],states[]}`) lists the built-in label-filter presets for the Projects cards, led by `needs-human` whose `labels[]` are the effective hard-suppress labels plus `needs-signal`, the hold labels, `stage/071-needs-author` and the reporter-trust awaiting label (#12172); accepts `?fields=a,b` for top-level selection and `?omit=repos` to drop heavy sections, and honors `Accept-Encoding: gzip`; `agents[]` includes busy-pane visibility fields (`kicksUndeliverable`, `busySince`, `lastTranscriptActivity`, `condition`, `conditionMessage`) when a Working pane cannot accept kicks; a working agent also carries `agents[].run` — the live per-run line (#14576): `{since, elapsedS, costUSD, tokens, priced, tests, filesChanged}`, the estimated spend/tokens since the kick was delivered (`lastKickAt`, falling back to `busySince`) (delta of the token collector's per-agent cumulative figure; `priced:false` when the agent has none yet), the last visible test-runner result (`pass`/`fail`/empty) and the last `N files changed` git summary (`-1` when none) parsed from the captured pane; top-level `fleetRun` `{running, hourUSD, hourPriced}` is the governor strip's `running N · $X this hour` figure; `sla` (fleet) and `repos[].sla` (per repo) carry the Delivery SLA block derived from the lifecycle timeline with no extra GitHub calls: `windows.{7d,30d}.{open_to_triage,triage_to_start,start_to_pr,pr_to_merge,open_to_merge}` as `{p50_h,p95_h,n}` (hours, nearest-rank), `history[]` of up to 30 daily buckets `{day,start_to_pr_p50,pr_to_merge_p50,open_to_merge_p50,n_started,n_pr,n_merged}` persisted under `/data/sla-history.json`, `stale_in_progress[]` (`{number,repo,started_at,age_h,agent}` — issues carrying `hive/in-progress` longer than the agent claim TTL with no linked PR), `stale_count`, `merged_7d`, `claim_ttl_s` and, on the fleet block, the effective `targets` (`start_to_pr_h`, `pr_to_merge_h`, `open_to_merge_h`); `staleSince` and `repos[].countsStale` / `countsStaleSince` mark totals served from last-good figures — the previous enumeration (#13382) or the ForgeState cache's last-good counts (#13376) | `pkg/dashboard/server.go:1606` | +| `GET` | `/api/status` | Dashboard auth/session | Dashboard aggregate status; `labelPresets[]` (`{key,label,description,labels[],states[]}`) lists the built-in label-filter presets for the Projects cards, led by `needs-human` whose `labels[]` are the effective hard-suppress labels plus `needs-signal`, the hold labels, `stage/071-needs-author` and the reporter-trust awaiting label (#12172); accepts `?fields=a,b` for top-level selection and `?omit=repos` to drop heavy sections, and honors `Accept-Encoding: gzip`; `agents[]` includes busy-pane visibility fields (`kicksUndeliverable`, `busySince`, `lastTranscriptActivity`, `condition`, `conditionMessage`) when a Working pane cannot accept kicks; a working agent also carries `agents[].run` — the live per-run line (#14576): `{since, elapsedS, costUSD, tokens, priced, tests, filesChanged}`, the estimated spend/tokens since the kick was delivered (`lastKickAt`, falling back to `busySince`) (delta of the token collector's per-agent cumulative figure; `priced:false` when the agent has none yet), the last visible test-runner result (`pass`/`fail`/empty) and the last `N files changed` git summary (`-1` when none) parsed from the captured pane; top-level `fleetRun` `{running, hourUSD, hourPriced}` is the governor strip's `running N · $X this hour` figure; `sla` (fleet) and `repos[].sla` (per repo) carry the Delivery SLA block derived from the lifecycle timeline with no extra GitHub calls: `windows.{7d,30d}.{open_to_triage,triage_to_start,start_to_pr,pr_to_merge,open_to_merge}` as `{p50_h,p95_h,n}` (hours, nearest-rank), `history[]` of up to 30 daily buckets `{day,start_to_pr_p50,pr_to_merge_p50,open_to_merge_p50,n_started,n_pr,n_merged}` persisted under `/data/sla-history.json`, `stale_in_progress[]` (`{number,repo,started_at,age_h,agent}` — issues carrying `hive/in-progress` longer than the agent claim TTL with no linked PR), `stale_count`, `merged_7d`, `claim_ttl_s` and, on the fleet block, the effective `targets` (`start_to_pr_h`, `pr_to_merge_h`, `open_to_merge_h`); `staleSince` and `repos[].countsStale` / `countsStaleSince` mark totals served from last-good figures — the previous enumeration (#13382) or the ForgeState cache's last-good counts (#13376); `ci_failure_signals` `{count, last, kicks, unmatched}` is the webhook CI-failure signal store — distinct red (repo, head, check) signals retained, the most recent one `{repo, number, head_sha, check, conclusion, details_url, agent, at}`, webhook-triggered kicks/nudges since boot, and failure deliveries whose head matched no open PR | `pkg/dashboard/server.go:1606` | | `GET` | `/api/status/summary` | Dashboard auth/session | Compact agent/governor/budget status summary for pollers; honors `Accept-Encoding: gzip` | `pkg/dashboard/server.go:1607` | | `GET` | `/api/overview/issues.csv` | Dashboard auth/session | Cached Overview issue-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1608` | | `GET` | `/api/overview/issues.json` | Dashboard auth/session | Cached Overview issue-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1609` | diff --git a/src/docs/operator-reference.md b/src/docs/operator-reference.md index 4934b25d63..fc8ab8b31b 100644 --- a/src/docs/operator-reference.md +++ b/src/docs/operator-reference.md @@ -1600,6 +1600,31 @@ v5 and v6 spokes) tracking one repo all stay on webhook mode. `review_requested` deliveries also queue an early review dispatch when `review.event_driven` is on; see [Event-driven dispatch](review-swarm.md#event-driven-dispatch). +- **A failed check reacts within seconds, not at the next sweep.** A + `check_run` (`completed`, conclusion `failure`/`timed_out`), `check_suite` + (`failure`/`timed_out`) or `status` (`failure`/`error`) delivery is + resolved to the open PR(s) at that head from the PR detail cache — one + cached lookup, no extra `GET /pulls` fan-out; a head no cached PR matches + is counted under `/api/status.ci_failure_signals.unmatched` and left to + the sweep's dirty mark. Each match becomes a **CI failure signal** + `{repo, number, head, check, details_url, at}`, deduplicated per head + + check and kept for two hours. A **hive-authored** PR is attributed to the + agent whose relay opened it (`agent_pr_created` in the audit log; scanner + when unattributed) and that agent is kicked at once with its + FIX-BEFORE-NEW section naming the check and the log URL — the same + section every regular kick carries, into which the signal is also folded + until `ci-failing.json` catches up, so nothing is lost if the kick is + gated. A **contributor** PR nudges the automerge lanes for a targeted pass + over that PR now (the contributor lane / `stage/072` path), with or + without the ForgeState cache. Guards, in this order: `governor.ci_failure_webhook_kick` + (default `true`; `false` records signals only), the repo pause + (`project.paused_repos`), the agent's config/runtime pause (the fleet + breaker pauses every agent), governor kick eligibility, and one + webhook-triggered kick or nudge per PR head per + `governor.ci_failure_kick_cooldown_s` (default `600`). The activity log + records `ci_failure_kick` — "CI failed on #N (check) — kicked agent X" / + "nudged contributor lane". **Agents never watch or poll CI themselves**: + detection is out-of-band via this webhook path, or the sweep as fallback. - **A recorded review verdict also wakes the governor loop.** When the relay records a verdict bound to a review dispatch, the head that wrote it is free and the PR may be ready for its next head or its decision kick, so diff --git a/src/docs/webhooks.md b/src/docs/webhooks.md index 338fb0eab6..965747ffe6 100644 --- a/src/docs/webhooks.md +++ b/src/docs/webhooks.md @@ -113,6 +113,8 @@ These are the acceptance criteria of #13373 and the tests that hold them. | `governor.eval_interval_max_s` | `1800` | cap on either tick after the API-budget stretch | | `governor.webhook_wake_debounce_s` | `10` | a delivery and its check fan-out coalesce into one wake cycle | | `governor.webhook_wake_min_gap_s` | `30` | minimum spacing between webhook-triggered cycles, and between the cycles recorded review verdicts trigger | +| `governor.ci_failure_webhook_kick` | `true` | a failed `check_run`/`check_suite`/`status` delivery kicks the owning agent (hive-authored PR) or runs a targeted contributor-lane pass (contributor PR) at once; `false` records the signal for the sweep only | +| `governor.ci_failure_kick_cooldown_s` | `600` | at most one webhook-triggered CI-failure kick or nudge per PR head in this window | | `forgestate.reconcile_interval_s` | `900` | ForgeState's own full-list interval per repo, shortened while drift is high | | `forgestate.freshness_bound_s` | `60` | maximum item age before merge/close/claim/kick re-reads it | | `forgestate.drift_threshold` | `0.05` | fraction of items a reconcile may find wrong before the interval shortens | diff --git a/src/pkg/config/config.go b/src/pkg/config/config.go index 0b5a521600..ae3c2cacad 100644 --- a/src/pkg/config/config.go +++ b/src/pkg/config/config.go @@ -1563,10 +1563,18 @@ type GovernorConfig struct { // cycle. WebhookWakeMinGapS is the minimum gap between two // webhook-triggered cycles. Both default at Load; see // docs/operator-reference.md "Webhook-driven PR cache invalidation". - WebhookWakeDebounceS int `yaml:"webhook_wake_debounce_s,omitempty"` - WebhookWakeMinGapS int `yaml:"webhook_wake_min_gap_s,omitempty"` - ConserveIntervalMultiplier int `yaml:"conserve_interval_multiplier,omitempty"` - OptionalSweepEveryNCycles int `yaml:"optional_sweep_every_n_cycles,omitempty"` + WebhookWakeDebounceS int `yaml:"webhook_wake_debounce_s,omitempty"` + WebhookWakeMinGapS int `yaml:"webhook_wake_min_gap_s,omitempty"` + // CIFailureWebhookKick (default true) lets a check_run / check_suite / + // status failure delivery kick the owning agent (or nudge the + // contributor lane) at once instead of waiting for the next sweep. + // CIFailureKickCooldownS (default 600) bounds webhook-triggered kicks + // to one per PR head in that window. See docs/operator-reference.md + // "Webhook-driven CI failure reaction". + CIFailureWebhookKick *bool `yaml:"ci_failure_webhook_kick,omitempty"` + CIFailureKickCooldownS int `yaml:"ci_failure_kick_cooldown_s,omitempty"` + ConserveIntervalMultiplier int `yaml:"conserve_interval_multiplier,omitempty"` + OptionalSweepEveryNCycles int `yaml:"optional_sweep_every_n_cycles,omitempty"` // ExplainMode is the hive-wide default explain mode for agents that leave // their own explain_mode unset. "" means "no hive default configured", in // which case ExplainModeEnvVar is consulted and then off — see @@ -4895,6 +4903,24 @@ func (c *Config) WebhookWakeMinGap() time.Duration { return DefaultWebhookWakeMinGapS * time.Second } +// CIFailureWebhookKick is the effective governor.ci_failure_webhook_kick +// (default true). +func (c *Config) CIFailureWebhookKick() bool { + if c == nil || c.Governor.CIFailureWebhookKick == nil { + return true + } + return *c.Governor.CIFailureWebhookKick +} + +// CIFailureKickCooldown is the effective governor.ci_failure_kick_cooldown_s +// as a duration (default 10 minutes). +func (c *Config) CIFailureKickCooldown() time.Duration { + if c != nil && c.Governor.CIFailureKickCooldownS > 0 { + return time.Duration(c.Governor.CIFailureKickCooldownS) * time.Second + } + return DefaultCIFailureKickCooldownS * time.Second +} + type DashboardIssueBandsConfig struct { WaitingLabels []string `yaml:"waiting_labels,omitempty" json:"waiting_labels,omitempty"` DoneLabels []string `yaml:"done_labels,omitempty" json:"done_labels,omitempty"` @@ -5822,6 +5848,7 @@ const ( DefaultEvalIntervalWebhookS = 900 DefaultWebhookWakeDebounceS = 10 DefaultWebhookWakeMinGapS = 30 + DefaultCIFailureKickCooldownS = 600 DefaultConserveIntervalMultiplier = 2 DefaultOptionalSweepEveryNCycles = 1 ) diff --git a/src/pkg/config/testdata/agent_runtime_golden.json b/src/pkg/config/testdata/agent_runtime_golden.json index b12f4ee235..2ed2482d70 100644 --- a/src/pkg/config/testdata/agent_runtime_golden.json +++ b/src/pkg/config/testdata/agent_runtime_golden.json @@ -171,6 +171,8 @@ "EvalIntervalWebhookS": 900, "WebhookWakeDebounceS": 10, "WebhookWakeMinGapS": 30, + "CIFailureWebhookKick": null, + "CIFailureKickCooldownS": 0, "ConserveIntervalMultiplier": 2, "OptionalSweepEveryNCycles": 1, "ExplainMode": "", diff --git a/src/pkg/config/validate.go b/src/pkg/config/validate.go index 43315ecc59..0c251f7d1d 100644 --- a/src/pkg/config/validate.go +++ b/src/pkg/config/validate.go @@ -277,6 +277,9 @@ func (c *Config) ValidateWithOptions(opts ValidateOptions) error { if c.Governor.WebhookWakeMinGapS < 0 { return fmt.Errorf("governor.webhook_wake_min_gap_s must be at least 0") } + if c.Governor.CIFailureKickCooldownS < 0 { + return fmt.Errorf("governor.ci_failure_kick_cooldown_s must be at least 0") + } if c.Governor.ConserveIntervalMultiplier < 0 { return fmt.Errorf("governor.conserve_interval_multiplier must be at least 1") } diff --git a/src/pkg/dashboard/api_github_webhook.go b/src/pkg/dashboard/api_github_webhook.go index d42b61764a..3653077486 100644 --- a/src/pkg/dashboard/api_github_webhook.go +++ b/src/pkg/dashboard/api_github_webhook.go @@ -126,6 +126,9 @@ func (s *Server) handleGitHubWebhook(w http.ResponseWriter, r *http.Request) { s.deps.WebhookWake(reason) resp["wake"] = reason } + if reactions := s.reactToCIFailure(res); len(reactions) > 0 { + resp["ci_failure"] = reactions + } if n, ok := sentinelPREventTarget(event, body); ok && s.deps.SentinelPREvent != nil { s.deps.SentinelPREvent(res.ConfiguredRepo, n) resp["sentinel"] = n diff --git a/src/pkg/dashboard/api_github_webhook_ci_failure.go b/src/pkg/dashboard/api_github_webhook_ci_failure.go new file mode 100644 index 0000000000..acaadf002b --- /dev/null +++ b/src/pkg/dashboard/api_github_webhook_ci_failure.go @@ -0,0 +1,133 @@ +package dashboard + +import ( + "fmt" + "time" + + "github.com/hivecommons/hive/pkg/github" +) + +// auditActionCIFailureKick is the activity-log action for a webhook +// CI-failure signal that kicked an agent or nudged the contributor lane. +const auditActionCIFailureKick = "ci_failure_kick" + +// CIFailureKickOutcome is what the governor-side CIFailureKick hook reports +// for one hive-authored PR signal. +type CIFailureKickOutcome struct { + // Agent is the agent the PR was attributed to ("" when unattributed). + Agent string `json:"agent,omitempty"` + // Kicked is true when a targeted kick was dispatched. + Kicked bool `json:"kicked"` + // Reason names why no kick went out (paused, cooldown, …). + Reason string `json:"reason,omitempty"` +} + +// ciFailureReaction is the per-PR record the webhook response carries. +type ciFailureReaction struct { + Number int `json:"number"` + Check string `json:"check"` + Route string `json:"route"` + Agent string `json:"agent,omitempty"` + Outcome string `json:"outcome"` +} + +// reactToCIFailure turns a failed-check delivery into "ci failure signals" +// and reacts to each one now rather than on the next sweep: a hive-authored +// PR goes to the CIFailureKick hook (targeted kick of the owning agent); a +// contributor PR goes to SweepNudge (the contributor lane / stage 072 path). +// Agents never watch or poll CI themselves — this out-of-band signal, or +// the sweep as fallback, is the only detection. Guards: config flag, repo +// pause, dedup per head+check, and the per-head kick cooldown. +func (s *Server) reactToCIFailure(res github.WebhookEventResult) []ciFailureReaction { + f := res.CIFailure + if f == nil || s == nil || s.deps == nil { + return nil + } + store := github.CIFailureSignals() + if len(f.PRs) == 0 { + store.RecordUnmatched() + return nil + } + // owner/name as the payload reports it: the sweep trigger, the repo + // pause predicate and the signal store all accept that form. + repo := res.Repo + if repo == "" { + repo = res.ConfiguredRepo + } + cfg := s.deps.Config + enabled := cfg == nil || cfg.CIFailureWebhookKick() + repoPaused := cfg != nil && cfg.IsRepoPaused(repo) + cooldown := cfg.CIFailureKickCooldown() + now := time.Now().UTC() + + var out []ciFailureReaction + for _, pr := range f.PRs { + sig := github.CIFailureSignal{ + Repo: repo, Number: pr.Number, HeadSHA: f.HeadSHA, Check: f.Check, + Conclusion: f.Conclusion, DetailsURL: f.DetailsURL, Title: pr.Title, + Author: pr.Author, HiveAuthored: pr.HiveAuthored, At: now, + } + r := ciFailureReaction{Number: pr.Number, Check: f.Check, Route: "contributor"} + if pr.HiveAuthored { + r.Route = "agent" + } + fresh := store.Record(sig) + switch { + case !fresh: + r.Outcome = "duplicate" + case !enabled: + r.Outcome = "disabled" + case repoPaused: + r.Outcome = "repo paused" + case pr.HiveAuthored: + r.Agent, r.Outcome = s.kickForCIFailure(sig) + default: + r.Outcome = s.nudgeContributorForCIFailure(sig, cooldown) + } + out = append(out, r) + } + return out +} + +func (s *Server) kickForCIFailure(sig github.CIFailureSignal) (agent, outcome string) { + if s.deps.CIFailureKick == nil { + return "", "no kick hook" + } + res := s.deps.CIFailureKick(sig) + if res.Agent != "" { + github.CIFailureSignals().SetAgent(sig.Repo, sig.Number, res.Agent) + } + if !res.Kicked { + if res.Reason == "" { + res.Reason = "not kicked" + } + return res.Agent, res.Reason + } + s.logCIFailureReaction(sig, fmt.Sprintf("kicked agent %s", res.Agent), res.Agent) + return res.Agent, "kicked" +} + +func (s *Server) nudgeContributorForCIFailure(sig github.CIFailureSignal, cooldown time.Duration) string { + if s.deps.SweepNudge == nil { + return "no sweep nudge" + } + if !github.CIFailureSignals().AllowKick(sig.Repo, sig.Number, sig.HeadSHA, cooldown) { + return "cooldown" + } + s.deps.SweepNudge(sig.Repo, sig.Number) + s.logCIFailureReaction(sig, "nudged contributor lane", "") + return "nudged" +} + +// logCIFailureReaction writes the one-line activity entry +// "CI failed on #N (check) — kicked agent X". +func (s *Server) logCIFailureReaction(sig github.CIFailureSignal, what, agent string) { + if s.audit == nil { + return + } + line := fmt.Sprintf("CI failed on #%d (%s) — %s", sig.Number, sig.Check, what) + if s.logger != nil { + s.logger.Info("github webhook: "+line, "repo", sig.Repo, "head", shortSHA(sig.HeadSHA), "details", sig.DetailsURL) + } + s.audit.LogRecord("system", auditActionCIFailureKick, line, agent, sig.Repo, sig.Number) +} diff --git a/src/pkg/dashboard/api_github_webhook_ci_failure_test.go b/src/pkg/dashboard/api_github_webhook_ci_failure_test.go new file mode 100644 index 0000000000..8d6aad8f96 --- /dev/null +++ b/src/pkg/dashboard/api_github_webhook_ci_failure_test.go @@ -0,0 +1,171 @@ +package dashboard + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "strconv" + "strings" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/github" +) + +func seedCIFailureWebhookServer(t *testing.T) (*Server, *[]github.CIFailureSignal, *[]string) { + t.Helper() + s := newGitHubWebhookTestServer(t) + s.deps.GHClient.SetAppBotLogin("hive-bot[bot]") + s.deps.Config = &config.Config{Project: config.ProjectConfig{Org: "webhookorg", Repos: []string{"webhookrepo"}}} + github.SeedPRDetailCacheForTest("webhookorg/webhookrepo", 41, "hive-bot[bot]", "agent PR", "feedface", "main") + github.SeedPRDetailCacheForTest("webhookorg/webhookrepo", 42, "octocat", "human PR", "c0ffee", "main") + + var kicks []github.CIFailureSignal + var nudges []string + s.deps.CIFailureKick = func(sig github.CIFailureSignal) CIFailureKickOutcome { + kicks = append(kicks, sig) + return CIFailureKickOutcome{Agent: "scanner", Kicked: true} + } + s.deps.SweepNudge = func(repo string, number int) { nudges = append(nudges, repo+"#"+strconv.Itoa(number)) } + return s, &kicks, &nudges +} + +func checkRunFailureBody(sha, name string) []byte { + return []byte(`{"action":"completed","repository":{"full_name":"webhookorg/webhookrepo"},"check_run":{"head_sha":"` + sha + `","name":"` + name + `","conclusion":"failure","details_url":"https://ci/` + name + `","pull_requests":[]}}`) +} + +func postCIFailure(t *testing.T, s *Server, event string, body []byte) []ciFailureReaction { + t.Helper() + const secret = "s3cr3t" + rec := postGitHubWebhook(s, event, signGitHubWebhookForTest(secret, body), body) + if rec.Code != http.StatusOK { + t.Fatalf("status = %d: %s", rec.Code, rec.Body.String()) + } + var got struct { + CIFailure []ciFailureReaction `json:"ci_failure"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &got); err != nil { + t.Fatal(err) + } + return got.CIFailure +} + +// A failed check on a hive-authored PR head kicks the owning agent through +// the hook and writes the activity line; the same head+check a second time +// is a duplicate and does nothing; a contributor PR head nudges the sweep +// (contributor lane / stage 072) instead, once per head within the cooldown. +func TestGitHubWebhookCIFailure_RoutesAndDedups(t *testing.T) { + t.Setenv(githubWebhookSecretEnvVar, "s3cr3t") + now := time.Date(2026, 10, 21, 12, 0, 0, 0, time.UTC) + clock := func() time.Time { return now } + t.Cleanup(github.ResetCIFailureSignalsForTest(clock)) + t.Cleanup(github.ResetWebhookTrackerForTest()) + s, kicks, nudges := seedCIFailureWebhookServer(t) + + got := postCIFailure(t, s, "check_run", checkRunFailureBody("feedface", "build-gate")) + if len(got) != 1 || got[0].Number != 41 || got[0].Route != "agent" || got[0].Outcome != "kicked" || got[0].Agent != "scanner" { + t.Fatalf("agent reaction = %+v", got) + } + if len(*kicks) != 1 || (*kicks)[0].Check != "build-gate" || (*kicks)[0].DetailsURL != "https://ci/build-gate" || !(*kicks)[0].HiveAuthored { + t.Fatalf("kick hook got %+v", *kicks) + } + var sawLine bool + for _, e := range s.audit.Recent(50) { + if e.Action == auditActionCIFailureKick && strings.Contains(e.Detail, "CI failed on #41 (build-gate) — kicked agent scanner") { + sawLine = true + } + } + if !sawLine { + t.Fatalf("activity log missing the CI failure line: %+v", s.audit.Recent(50)) + } + + // Same head + check again (GitHub redelivery / check_suite fan-out): + // deduplicated, no second kick. The cache entry is gone after the first + // delivery, so this exercises the number-only path too. + github.SeedPRDetailCacheForTest("webhookorg/webhookrepo", 41, "hive-bot[bot]", "agent PR", "feedface", "main") + got = postCIFailure(t, s, "check_run", checkRunFailureBody("feedface", "build-gate")) + if len(got) != 1 || got[0].Outcome != "duplicate" || len(*kicks) != 1 { + t.Fatalf("duplicate reaction = %+v, kicks = %d", got, len(*kicks)) + } + + // Contributor PR: nudge the sweep, never the kick hook. + status := []byte(`{"sha":"c0ffee","state":"failure","context":"ci/test","target_url":"https://ci/3","repository":{"full_name":"webhookorg/webhookrepo"}}`) + got = postCIFailure(t, s, "status", status) + if len(got) != 1 || got[0].Number != 42 || got[0].Route != "contributor" || got[0].Outcome != "nudged" { + t.Fatalf("contributor reaction = %+v", got) + } + if len(*nudges) != 1 || !strings.HasPrefix((*nudges)[0], "webhookorg/webhookrepo#") || len(*kicks) != 1 { + t.Fatalf("nudges = %v, kicks = %d", *nudges, len(*kicks)) + } + // A different check on the same contributor head within the cooldown is + // a new signal but no second nudge. + github.SeedPRDetailCacheForTest("webhookorg/webhookrepo", 42, "octocat", "human PR", "c0ffee", "main") + got = postCIFailure(t, s, "check_run", checkRunFailureBody("c0ffee", "lint")) + if len(got) != 1 || got[0].Outcome != "cooldown" || len(*nudges) != 1 { + t.Fatalf("cooldown reaction = %+v, nudges = %v", got, *nudges) + } + now = now.Add(config.DefaultCIFailureKickCooldownS*time.Second + time.Second) + github.SeedPRDetailCacheForTest("webhookorg/webhookrepo", 42, "octocat", "human PR", "c0ffee", "main") + got = postCIFailure(t, s, "check_run", checkRunFailureBody("c0ffee", "e2e")) + if len(got) != 1 || got[0].Outcome != "nudged" || len(*nudges) != 2 { + t.Fatalf("post-cooldown reaction = %+v, nudges = %v", got, *nudges) + } + + // /api/status exposes the store. + s.statusMu.Lock() + s.status = &StatusPayload{HiveID: "webhook-test"} + s.statusMu.Unlock() + rec := httptest.NewRecorder() + s.handleStatus(rec, httptest.NewRequest(http.MethodGet, "/api/status?fields=ci_failure_signals", nil)) + var st struct { + Signals github.CIFailureSignalsStatus `json:"ci_failure_signals"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &st); err != nil { + t.Fatal(err) + } + if st.Signals.Count != 4 || st.Signals.Last == nil || st.Signals.Last.Check != "e2e" || st.Signals.Kicks != 2 { + t.Fatalf("/api/status ci_failure_signals = %+v", st.Signals) + } +} + +// Repo pause and the config flag stop the reaction (the signal is still +// recorded for the sweep); a head no open PR matches counts as unmatched. +func TestGitHubWebhookCIFailure_PauseFlagAndUnmatched(t *testing.T) { + t.Setenv(githubWebhookSecretEnvVar, "s3cr3t") + t.Cleanup(github.ResetCIFailureSignalsForTest(nil)) + t.Cleanup(github.ResetWebhookTrackerForTest()) + s, kicks, nudges := seedCIFailureWebhookServer(t) + + s.deps.Config.Project.PausedRepos = []config.RepoPause{{Repo: "webhookrepo"}} + got := postCIFailure(t, s, "check_run", checkRunFailureBody("feedface", "build-gate")) + if len(got) != 1 || got[0].Outcome != "repo paused" || len(*kicks) != 0 { + t.Fatalf("paused repo reaction = %+v, kicks = %d", got, len(*kicks)) + } + s.deps.Config.Project.PausedRepos = nil + + off := false + s.deps.Config.Governor.CIFailureWebhookKick = &off + github.SeedPRDetailCacheForTest("webhookorg/webhookrepo", 42, "octocat", "human PR", "c0ffee", "main") + got = postCIFailure(t, s, "check_run", checkRunFailureBody("c0ffee", "lint")) + if len(got) != 1 || got[0].Outcome != "disabled" || len(*nudges) != 0 { + t.Fatalf("disabled reaction = %+v, nudges = %v", got, *nudges) + } + if snap := github.CIFailureSignals().Snapshot(); snap.Count != 2 { + t.Fatalf("signals must still be recorded for the sweep: %+v", snap) + } + s.deps.Config.Governor.CIFailureWebhookKick = nil + + got = postCIFailure(t, s, "status", []byte(`{"sha":"deadbeef","state":"error","context":"ci","repository":{"full_name":"webhookorg/webhookrepo"}}`)) + if len(got) != 0 { + t.Fatalf("unmatched head must produce no reaction: %+v", got) + } + if snap := github.CIFailureSignals().Snapshot(); snap.Unmatched != 1 { + t.Fatalf("unmatched = %d, want 1", snap.Unmatched) + } + // A green check never reacts. + green := []byte(`{"action":"completed","repository":{"full_name":"webhookorg/webhookrepo"},"check_run":{"head_sha":"feedface","name":"build-gate","conclusion":"success","pull_requests":[{"number":41}]}}`) + if got := postCIFailure(t, s, "check_run", green); len(got) != 0 { + t.Fatalf("success must not react: %+v", got) + } +} diff --git a/src/pkg/dashboard/deps.go b/src/pkg/dashboard/deps.go index 0b65cec71c..926854f2c4 100644 --- a/src/pkg/dashboard/deps.go +++ b/src/pkg/dashboard/deps.go @@ -301,6 +301,12 @@ type Dependencies struct { // repo is the configured slug. It must not block the webhook response. // Nil leaves PRs to the sentinel's reconcile cadence. SentinelPREvent func(repo string, number int) + // CIFailureKick reacts to one webhook CI-failure signal on a + // hive-authored PR: attributes it to the owning agent and, when the + // agent is enabled, unpaused, governor-eligible and outside the per-head + // cooldown, kicks it now with the fix-before-new section. It must not + // block the webhook response. Nil leaves the signal to the sweep. + CIFailureKick func(sig ghpkg.CIFailureSignal) CIFailureKickOutcome // ProjectsV2ItemEvent applies a verified `projects_v2_item` delivery to // the GitHub Projects work source cache as a targeted patch (#13379). // These deliveries are organization-scoped and carry no repository. Nil diff --git a/src/pkg/dashboard/server.go b/src/pkg/dashboard/server.go index 8829ee503d..6cd8516923 100644 --- a/src/pkg/dashboard/server.go +++ b/src/pkg/dashboard/server.go @@ -3663,7 +3663,7 @@ func (s *Server) handleStatus(w http.ResponseWriter, r *http.Request) { return } now := time.Now().UTC() - payload := statusWithWebhooks{StatusPayload: s.statusWithOverviewBands(status, now), Webhooks: github.WebhookHealthSnapshot()} + payload := statusWithWebhooks{StatusPayload: s.statusWithOverviewBands(status, now), Webhooks: github.WebhookHealthSnapshot(), CIFailureSignals: github.CIFailureSignals().Snapshot()} if s.deps != nil && s.deps.EvalLoop != nil { loop := s.deps.EvalLoop() payload.EvalLoopStatus = &loop @@ -3679,6 +3679,9 @@ func (s *Server) handleStatus(w http.ResponseWriter, r *http.Request) { type statusWithWebhooks struct { *StatusPayload Webhooks github.WebhookHealth `json:"webhooks"` + // CIFailureSignals is the webhook CI-failure signal store: retained + // signals, the last one, webhook-triggered kicks and unmatched heads. + CIFailureSignals github.CIFailureSignalsStatus `json:"ci_failure_signals"` *EvalLoopStatus // UpgradeProgress is the spoke's view of the scheduled-upgrade state // machine (scheduled / draining / pulling / … / missed) with the drain diff --git a/src/pkg/dashboard/settings_parity_allowlist.txt b/src/pkg/dashboard/settings_parity_allowlist.txt index 5f02eb06aa..e137214e25 100644 --- a/src/pkg/dashboard/settings_parity_allowlist.txt +++ b/src/pkg/dashboard/settings_parity_allowlist.txt @@ -194,6 +194,8 @@ governor.budget.coins..budget # TODO #11854 governor.budget.coins..label # TODO #11854 governor.budget.coins..tokens_per_coin # TODO #11854 governor.budget.coins..usd_per_coin # TODO #11854 +governor.ci_failure_kick_cooldown_s # webhook CI-failure kick cooldown (#15291); yaml-only +governor.ci_failure_webhook_kick # webhook CI-failure kick flag (#15291); yaml-only governor.claims.contributor_flap_threshold # contributor flap cool-off (#13579); yaml-only governor.claims.contributor_flap_window_s # contributor flap cool-off (#13579); yaml-only governor.claims.contributor_flap_cooloff_s # contributor flap cool-off (#13579); yaml-only diff --git a/src/pkg/github/automerge/automerge_sweep.go b/src/pkg/github/automerge/automerge_sweep.go index e6ae827ba5..ccf3b813bd 100644 --- a/src/pkg/github/automerge/automerge_sweep.go +++ b/src/pkg/github/automerge/automerge_sweep.go @@ -381,7 +381,12 @@ func New(transport Transport, opts Options) *Engine { if e.headFirstSeen == nil { e.headFirstSeen = sharedHeadFirstSeen } - if e.sweepTrigger == nil && e.openPRs != nil { + if e.sweepTrigger == nil { + // Always present: with the forgestate cache the event-driven loop + // drains it; without, the legacy ticker loop still selects on it so + // a webhook CI-failure nudge (SweepNudge) runs a targeted pass now + // instead of on the next tick. A nil source only disables + // NotifyHead's SHA lookup. e.sweepTrigger = NewSweepTrigger(e.openPRs) } @@ -1531,6 +1536,11 @@ func (c *Engine) StartSelfAuthoredAutoMergeSweep(ctx context.Context, maxMerges select { case <-ctx.Done(): return + case <-c.sweepTrigger.C(): + // A webhook nudge (CI failure on a PR, operator row action) + // names the PRs; evaluate only those now. A full request + // (nil targets) is a regular pass brought forward. + c.sweepPass(ctx, maxMerges, acmmAllowed, c.sweepTrigger.take(), "event") case <-t.C: gate := c.sweepTickGate(interval, cautionTick, hgithub.ShedLowPriorityTick) if gate.reset { diff --git a/src/pkg/github/ci_failure_signal.go b/src/pkg/github/ci_failure_signal.go new file mode 100644 index 0000000000..718b7dfb3e --- /dev/null +++ b/src/pkg/github/ci_failure_signal.go @@ -0,0 +1,351 @@ +package github + +import ( + "sort" + "strconv" + "strings" + "sync" + "time" + + gh "github.com/google/go-github/v72/github" +) + +// CIFailurePR is a PR a check_run / check_suite / status failure delivery +// resolved to, as the PR detail cache knew it at the moment of the event +// (captured before the delivery invalidates the entry). +type CIFailurePR struct { + Number int `json:"number"` + Author string `json:"author,omitempty"` + Title string `json:"title,omitempty"` + URL string `json:"url,omitempty"` + BaseRef string `json:"base_ref,omitempty"` + HiveAuthored bool `json:"hive_authored"` +} + +// WebhookCIFailure is the failed-check content of one check_run / +// check_suite / status delivery, carried on WebhookEventResult.CIFailure so +// the receiver can react now rather than on the next sweep. +type WebhookCIFailure struct { + HeadSHA string `json:"head_sha"` + Check string `json:"check"` + Conclusion string `json:"conclusion"` + DetailsURL string `json:"details_url,omitempty"` + PRs []CIFailurePR `json:"prs,omitempty"` +} + +// CIFailureSignal is one recorded "this check went red on this PR head" +// observation. Agent is the hive agent the PR is attributed to (empty for a +// contributor PR or an unattributed one). +type CIFailureSignal struct { + Repo string `json:"repo"` + Number int `json:"number"` + HeadSHA string `json:"head_sha"` + Check string `json:"check"` + Conclusion string `json:"conclusion,omitempty"` + DetailsURL string `json:"details_url,omitempty"` + Title string `json:"title,omitempty"` + Author string `json:"author,omitempty"` + HiveAuthored bool `json:"hive_authored"` + Agent string `json:"agent,omitempty"` + At time.Time `json:"at"` +} + +// CIFailureSignalsStatus is the /api/status view of the store. +type CIFailureSignalsStatus struct { + // Count is the number of distinct (repo, head, check) signals retained. + Count int `json:"count"` + // Last is the most recently recorded signal, nil when none. + Last *CIFailureSignal `json:"last,omitempty"` + // Kicks counts webhook-triggered agent kicks since boot. + Kicks int `json:"kicks"` + // Unmatched counts failure deliveries whose head SHA resolved to no open + // PR; the sweep handles them from the dirty mark. + Unmatched int `json:"unmatched"` +} + +// ciFailureSignalTTL bounds how long a signal stays in the store after its +// last observation; a PR that stays red longer is re-listed from +// ci-failing.json by the sweep anyway. +const ciFailureSignalTTL = 2 * time.Hour + +type ciFailureSignalKey struct { + repo, headSHA, check string +} + +// CIFailureSignalStore is the in-memory "ci failure signal" store: dedup per +// head SHA + check, per-PR-head kick cooldown, and a bounded TTL. +type CIFailureSignalStore struct { + mu sync.Mutex + now func() time.Time + signals map[ciFailureSignalKey]CIFailureSignal + lastKick map[string]time.Time + last *CIFailureSignal + kicks int + unmatched int +} + +func newCIFailureSignalStore() *CIFailureSignalStore { + return &CIFailureSignalStore{ + now: time.Now, + signals: map[ciFailureSignalKey]CIFailureSignal{}, + lastKick: map[string]time.Time{}, + } +} + +var sharedCIFailureSignals = newCIFailureSignalStore() + +// CIFailureSignals is the process-wide store the webhook receiver records +// into and the kick builder / sweep read from. +func CIFailureSignals() *CIFailureSignalStore { return sharedCIFailureSignals } + +// ResetCIFailureSignalsForTest swaps in a fresh store on the given clock and +// returns the restore func. +func ResetCIFailureSignalsForTest(now func() time.Time) func() { + old := sharedCIFailureSignals + s := newCIFailureSignalStore() + if now != nil { + s.now = now + } + sharedCIFailureSignals = s + return func() { sharedCIFailureSignals = old } +} + +func ciFailureKey(sig CIFailureSignal) ciFailureSignalKey { + return ciFailureSignalKey{ + repo: canonicalPRDetailRepo(sig.Repo), + headSHA: strings.ToLower(strings.TrimSpace(sig.HeadSHA)), + check: strings.TrimSpace(sig.Check), + } +} + +func ciFailureKickKey(repo string, number int, headSHA string) string { + return canonicalPRDetailRepo(repo) + "#" + strconv.Itoa(number) + "@" + strings.ToLower(strings.TrimSpace(headSHA)) +} + +// Record stores sig and reports whether it is new for its (repo, head, +// check); a repeat observation refreshes the entry (and fills in a missing +// agent attribution) but returns false so callers do not re-kick. +func (s *CIFailureSignalStore) Record(sig CIFailureSignal) bool { + if s == nil || sig.Repo == "" || sig.Number <= 0 || strings.TrimSpace(sig.HeadSHA) == "" { + return false + } + s.mu.Lock() + defer s.mu.Unlock() + now := s.now() + s.pruneLocked(now) + if sig.At.IsZero() { + sig.At = now + } + key := ciFailureKey(sig) + prev, dup := s.signals[key] + if dup { + if prev.Agent != "" && sig.Agent == "" { + sig.Agent = prev.Agent + } + if sig.Title == "" { + sig.Title = prev.Title + } + } + s.signals[key] = sig + s.last = &sig + return !dup +} + +// SetAgent attributes every stored signal for repo#number to agent. +func (s *CIFailureSignalStore) SetAgent(repo string, number int, agent string) { + if s == nil { + return + } + s.mu.Lock() + defer s.mu.Unlock() + canon := canonicalPRDetailRepo(repo) + for k, sig := range s.signals { + if k.repo == canon && sig.Number == number { + sig.Agent = agent + s.signals[k] = sig + } + } +} + +// RecordUnmatched counts a failure delivery no open PR matched. +func (s *CIFailureSignalStore) RecordUnmatched() { + if s == nil { + return + } + s.mu.Lock() + s.unmatched++ + s.mu.Unlock() +} + +// AllowKick reports whether a webhook-triggered kick for repo#number at +// headSHA is outside the cooldown, and records it when so. +func (s *CIFailureSignalStore) AllowKick(repo string, number int, headSHA string, cooldown time.Duration) bool { + if s == nil { + return false + } + s.mu.Lock() + defer s.mu.Unlock() + now := s.now() + key := ciFailureKickKey(repo, number, headSHA) + if last, ok := s.lastKick[key]; ok && cooldown > 0 && now.Sub(last) < cooldown { + return false + } + s.lastKick[key] = now + s.kicks++ + return true +} + +// ForAgent lists the retained signals attributed to agent, oldest first. +func (s *CIFailureSignalStore) ForAgent(agent string) []CIFailureSignal { + return s.filter(func(sig CIFailureSignal) bool { return agent != "" && sig.Agent == agent }) +} + +// ForPR lists the retained signals for repo#number, oldest first. +func (s *CIFailureSignalStore) ForPR(repo string, number int) []CIFailureSignal { + canon := canonicalPRDetailRepo(repo) + return s.filter(func(sig CIFailureSignal) bool { + return sig.Number == number && canonicalPRDetailRepo(sig.Repo) == canon + }) +} + +func (s *CIFailureSignalStore) filter(keep func(CIFailureSignal) bool) []CIFailureSignal { + if s == nil { + return nil + } + s.mu.Lock() + defer s.mu.Unlock() + s.pruneLocked(s.now()) + var out []CIFailureSignal + for _, sig := range s.signals { + if keep(sig) { + out = append(out, sig) + } + } + sort.Slice(out, func(i, j int) bool { + if !out[i].At.Equal(out[j].At) { + return out[i].At.Before(out[j].At) + } + return out[i].Check < out[j].Check + }) + return out +} + +// Forget drops every signal for repo#number (the PR merged, closed, or its +// head moved on to a new SHA that is not this one). +func (s *CIFailureSignalStore) Forget(repo string, number int) { + if s == nil { + return + } + s.mu.Lock() + defer s.mu.Unlock() + canon := canonicalPRDetailRepo(repo) + for k, sig := range s.signals { + if k.repo == canon && sig.Number == number { + delete(s.signals, k) + } + } +} + +// Snapshot is the /api/status view. +func (s *CIFailureSignalStore) Snapshot() CIFailureSignalsStatus { + if s == nil { + return CIFailureSignalsStatus{} + } + s.mu.Lock() + defer s.mu.Unlock() + s.pruneLocked(s.now()) + out := CIFailureSignalsStatus{Count: len(s.signals), Kicks: s.kicks, Unmatched: s.unmatched} + if s.last != nil { + last := *s.last + out.Last = &last + } + return out +} + +func (s *CIFailureSignalStore) pruneLocked(now time.Time) { + for k, sig := range s.signals { + if now.Sub(sig.At) > ciFailureSignalTTL { + delete(s.signals, k) + } + } + for k, at := range s.lastKick { + if now.Sub(at) > ciFailureSignalTTL { + delete(s.lastKick, k) + } + } +} + +// AuthorAgentForPR is the hive agent whose relay request opened repo#number +// per the audit log (agent_pr_created), or "" when none did. +func AuthorAgentForPR(repo string, number int) string { + return auditAuthorAgentForPR(repo, number, reviewVerdictAuditPath, time.Now().UTC().Add(-reviewVerdictAuditLookback)) +} + +// webhookCIFailure extracts the failed-check content of a check_run / +// check_suite / status delivery, or nil when the delivery is not a failure +// the hive should react to (pending, success, neutral, skipped, cancelled). +func webhookCIFailure(event string, p webhookEventPayload) *WebhookCIFailure { + switch event { + case "check_run": + if p.CheckRun == nil || p.Action != "completed" { + return nil + } + switch p.CheckRun.Conclusion { + case "failure", "timed_out": + default: + return nil + } + return &WebhookCIFailure{ + HeadSHA: strings.TrimSpace(p.CheckRun.HeadSHA), + Check: strings.TrimSpace(p.CheckRun.Name), + Conclusion: p.CheckRun.Conclusion, + DetailsURL: firstNonEmpty(p.CheckRun.DetailsURL, p.CheckRun.HTMLURL), + } + case "check_suite": + if p.CheckSuite == nil || p.Action != "completed" { + return nil + } + switch p.CheckSuite.Conclusion { + case "failure", "timed_out": + default: + return nil + } + name := "check suite" + if app := strings.TrimSpace(p.CheckSuite.App.Name); app != "" { + name = app + " check suite" + } + return &WebhookCIFailure{ + HeadSHA: strings.TrimSpace(p.CheckSuite.HeadSHA), + Check: name, + Conclusion: p.CheckSuite.Conclusion, + DetailsURL: strings.TrimSpace(p.CheckSuite.URL), + } + case "status": + switch p.State { + case "failure", "error": + default: + return nil + } + return &WebhookCIFailure{ + HeadSHA: strings.TrimSpace(p.SHA), + Check: strings.TrimSpace(p.Context), + Conclusion: p.State, + DetailsURL: strings.TrimSpace(p.TargetURL), + } + } + return nil +} + +// SeedPRDetailCacheForTest places a minimal open PR in the shared detail +// cache so webhook tests can resolve a head SHA without a GitHub call. +func SeedPRDetailCacheForTest(repo string, number int, author, title, headSHA, baseRef string) { + sharedPRDetailCache.put(repo, number, &gh.PullRequest{ + Number: gh.Ptr(number), + State: gh.Ptr("open"), + Title: gh.Ptr(title), + HTMLURL: gh.Ptr("https://github.com/" + repo + "/pull/" + strconv.Itoa(number)), + User: &gh.User{Login: gh.Ptr(author)}, + Head: &gh.PullRequestBranch{SHA: gh.Ptr(headSHA)}, + Base: &gh.PullRequestBranch{Ref: gh.Ptr(baseRef)}, + }) +} diff --git a/src/pkg/github/ci_failure_signal_test.go b/src/pkg/github/ci_failure_signal_test.go new file mode 100644 index 0000000000..9e5de4a4a8 --- /dev/null +++ b/src/pkg/github/ci_failure_signal_test.go @@ -0,0 +1,242 @@ +package github + +import ( + "log/slog" + "net/http" + "net/http/httptest" + "os" + "testing" + "time" + + gh "github.com/google/go-github/v72/github" +) + +func TestWebhookCIFailure_EventsAndConclusions(t *testing.T) { + cases := []struct { + name string + event string + payload webhookEventPayload + want *WebhookCIFailure + }{ + {name: "check_run failure", event: "check_run", payload: func() webhookEventPayload { + var p webhookEventPayload + p.Action = "completed" + p.CheckRun = ptrCheckRun("abc", "build-gate", "failure", "https://ci/run/1") + return p + }(), want: &WebhookCIFailure{HeadSHA: "abc", Check: "build-gate", Conclusion: "failure", DetailsURL: "https://ci/run/1"}}, + {name: "check_run timed_out", event: "check_run", payload: func() webhookEventPayload { + var p webhookEventPayload + p.Action = "completed" + p.CheckRun = ptrCheckRun("abc", "e2e", "timed_out", "") + return p + }(), want: &WebhookCIFailure{HeadSHA: "abc", Check: "e2e", Conclusion: "timed_out"}}, + {name: "check_run cancelled is not a failure", event: "check_run", payload: func() webhookEventPayload { + var p webhookEventPayload + p.Action = "completed" + p.CheckRun = ptrCheckRun("abc", "e2e", "cancelled", "") + return p + }()}, + {name: "check_run created is not a failure", event: "check_run", payload: func() webhookEventPayload { + var p webhookEventPayload + p.Action = "created" + p.CheckRun = ptrCheckRun("abc", "e2e", "failure", "") + return p + }()}, + {name: "status error", event: "status", payload: webhookEventPayload{SHA: "def", State: "error", Context: "ci/lint", TargetURL: "https://ci/2"}, + want: &WebhookCIFailure{HeadSHA: "def", Check: "ci/lint", Conclusion: "error", DetailsURL: "https://ci/2"}}, + {name: "status pending", event: "status", payload: webhookEventPayload{SHA: "def", State: "pending", Context: "ci/lint"}}, + {name: "check_suite success", event: "check_suite", payload: func() webhookEventPayload { + var p webhookEventPayload + p.Action = "completed" + p.CheckSuite = ptrCheckSuite("abc", "success") + return p + }()}, + {name: "check_suite failure", event: "check_suite", payload: func() webhookEventPayload { + var p webhookEventPayload + p.Action = "completed" + p.CheckSuite = ptrCheckSuite("abc", "failure") + return p + }(), want: &WebhookCIFailure{HeadSHA: "abc", Check: "GitHub Actions check suite", Conclusion: "failure"}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := webhookCIFailure(tc.event, tc.payload) + if (got == nil) != (tc.want == nil) { + t.Fatalf("got %+v, want %+v", got, tc.want) + } + if got == nil { + return + } + if got.HeadSHA != tc.want.HeadSHA || got.Check != tc.want.Check || got.Conclusion != tc.want.Conclusion || got.DetailsURL != tc.want.DetailsURL { + t.Fatalf("got %+v, want %+v", got, tc.want) + } + }) + } +} + +type checkRunPayload = struct { + HeadSHA string `json:"head_sha"` + Name string `json:"name"` + Conclusion string `json:"conclusion"` + DetailsURL string `json:"details_url"` + HTMLURL string `json:"html_url"` + PullRequests []webhookPRRef `json:"pull_requests"` +} + +type checkSuitePayload = struct { + HeadSHA string `json:"head_sha"` + Conclusion string `json:"conclusion"` + URL string `json:"url"` + PullRequests []webhookPRRef `json:"pull_requests"` + App struct { + Name string `json:"name"` + } `json:"app"` +} + +func ptrCheckRun(sha, name, conclusion, url string) *checkRunPayload { + return &checkRunPayload{HeadSHA: sha, Name: name, Conclusion: conclusion, DetailsURL: url} +} + +func ptrCheckSuite(sha, conclusion string) *checkSuitePayload { + p := &checkSuitePayload{HeadSHA: sha, Conclusion: conclusion} + p.App.Name = "GitHub Actions" + return p +} + +// A failed check_run on a cached PR head resolves to the PR with its author +// and hive-authorship captured BEFORE the delivery invalidates the entry; a +// second delivery for the same head finds the cache empty and reports the +// PR by number only (the sweep fills in the rest). +func TestHandleWebhookEvent_CIFailureResolvesCachedPR(t *testing.T) { + t.Cleanup(resetPRDetailCacheForTest(nil, 0)) + t.Cleanup(ResetWebhookTrackerForTest()) + logger := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelError})) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + t.Errorf("webhook must not call GitHub: %s", r.URL.Path) + })) + t.Cleanup(srv.Close) + c := NewClientForTest(srv.URL, "acme", []string{"widgets"}, logger) + c.SetAppBotLogin("hive-bot[bot]") + + sharedPRDetailCache.put("acme/widgets", 41, &gh.PullRequest{ + Number: gh.Ptr(41), State: gh.Ptr("open"), Title: gh.Ptr("agent fix"), + User: &gh.User{Login: gh.Ptr("hive-bot[bot]")}, + Head: &gh.PullRequestBranch{SHA: gh.Ptr("feedface"), Ref: gh.Ptr("hive/fix")}, + Base: &gh.PullRequestBranch{Ref: gh.Ptr("main")}, + }) + sharedPRDetailCache.put("acme/widgets", 42, &gh.PullRequest{ + Number: gh.Ptr(42), State: gh.Ptr("open"), Title: gh.Ptr("human PR"), + User: &gh.User{Login: gh.Ptr("octocat")}, + Head: &gh.PullRequestBranch{SHA: gh.Ptr("c0ffee"), Ref: gh.Ptr("feat")}, + Base: &gh.PullRequestBranch{Ref: gh.Ptr("main")}, + }) + + body := []byte(`{"action":"completed","repository":{"full_name":"acme/widgets"}, + "check_run":{"head_sha":"feedface","name":"build-gate","conclusion":"failure","details_url":"https://ci/1","pull_requests":[]}}`) + res, err := c.HandleWebhookEvent("check_run", body) + if err != nil { + t.Fatal(err) + } + if res.CIFailure == nil || len(res.CIFailure.PRs) != 1 { + t.Fatalf("CIFailure = %+v, want one PR", res.CIFailure) + } + pr := res.CIFailure.PRs[0] + if pr.Number != 41 || !pr.HiveAuthored || pr.Author != "hive-bot[bot]" || pr.Title != "agent fix" || pr.BaseRef != "main" { + t.Fatalf("resolved PR = %+v", pr) + } + if res.CIFailure.Check != "build-gate" || res.CIFailure.DetailsURL != "https://ci/1" { + t.Fatalf("CIFailure = %+v", res.CIFailure) + } + + // Contributor PR: resolved, not hive-authored. + body = []byte(`{"sha":"c0ffee","state":"failure","context":"ci/test","target_url":"https://ci/2","repository":{"full_name":"acme/widgets"}}`) + res, err = c.HandleWebhookEvent("status", body) + if err != nil { + t.Fatal(err) + } + if res.CIFailure == nil || len(res.CIFailure.PRs) != 1 || res.CIFailure.PRs[0].HiveAuthored || res.CIFailure.PRs[0].Number != 42 { + t.Fatalf("contributor CIFailure = %+v", res.CIFailure) + } + + // Success deliveries carry no failure. + body = []byte(`{"action":"completed","repository":{"full_name":"acme/widgets"}, + "check_run":{"head_sha":"feedface","name":"build-gate","conclusion":"success","pull_requests":[{"number":41}]}}`) + res, _ = c.HandleWebhookEvent("check_run", body) + if res.CIFailure != nil { + t.Fatalf("success must not produce a CI failure: %+v", res.CIFailure) + } + + // Unknown head (cache already invalidated): the delivery still names the + // PR when GitHub lists it, by number only. + body = []byte(`{"action":"completed","repository":{"full_name":"acme/widgets"}, + "check_run":{"head_sha":"feedface","name":"lint","conclusion":"failure","pull_requests":[{"number":41}]}}`) + res, _ = c.HandleWebhookEvent("check_run", body) + if res.CIFailure == nil || len(res.CIFailure.PRs) != 1 || res.CIFailure.PRs[0].Author != "" { + t.Fatalf("number-only CIFailure = %+v", res.CIFailure) + } + // And a head no PR matches is an unmatched failure. + body = []byte(`{"sha":"deadbeef","state":"failure","context":"ci/test","repository":{"full_name":"acme/widgets"}}`) + res, _ = c.HandleWebhookEvent("status", body) + if res.CIFailure == nil || len(res.CIFailure.PRs) != 0 { + t.Fatalf("unmatched CIFailure = %+v", res.CIFailure) + } +} + +func TestCIFailureSignalStore_DedupCooldownAndSnapshot(t *testing.T) { + now := time.Date(2026, 10, 21, 12, 0, 0, 0, time.UTC) + clock := func() time.Time { return now } + t.Cleanup(ResetCIFailureSignalsForTest(clock)) + s := CIFailureSignals() + + sig := CIFailureSignal{Repo: "acme/widgets", Number: 7, HeadSHA: "ABC", Check: "build"} + if !s.Record(sig) { + t.Fatal("first signal must be fresh") + } + sig.HeadSHA = "abc" // same head, case-insensitive + if s.Record(sig) { + t.Fatal("same head+check must dedup") + } + sig.Check = "lint" + if !s.Record(sig) { + t.Fatal("different check on the same head is a new signal") + } + if got := s.Snapshot(); got.Count != 2 || got.Last == nil || got.Last.Check != "lint" { + t.Fatalf("snapshot = %+v", got) + } + + if !s.AllowKick("acme/widgets", 7, "abc", 10*time.Minute) { + t.Fatal("first kick must be allowed") + } + now = now.Add(5 * time.Minute) + if s.AllowKick("acme/widgets", 7, "abc", 10*time.Minute) { + t.Fatal("kick inside the cooldown must be refused") + } + if !s.AllowKick("acme/widgets", 7, "newhead", 10*time.Minute) { + t.Fatal("a new head is outside the cooldown") + } + now = now.Add(6 * time.Minute) + if !s.AllowKick("acme/widgets", 7, "abc", 10*time.Minute) { + t.Fatal("kick after the cooldown must be allowed") + } + if got := s.Snapshot().Kicks; got != 3 { + t.Fatalf("kicks = %d, want 3", got) + } + + s.SetAgent("acme/widgets", 7, "scanner") + if got := s.ForAgent("scanner"); len(got) != 2 || got[0].Agent != "scanner" { + t.Fatalf("ForAgent = %+v", got) + } + if got := s.ForPR("ACME/widgets", 7); len(got) != 2 { + t.Fatalf("ForPR = %+v", got) + } + s.Forget("acme/widgets", 7) + if got := s.Snapshot().Count; got != 0 { + t.Fatalf("count after Forget = %d", got) + } + + s.Record(CIFailureSignal{Repo: "acme/widgets", Number: 8, HeadSHA: "x", Check: "build"}) + now = now.Add(ciFailureSignalTTL + time.Minute) + if got := s.Snapshot().Count; got != 0 { + t.Fatalf("signal must expire after the TTL, count = %d", got) + } +} diff --git a/src/pkg/github/prdetailcache.go b/src/pkg/github/prdetailcache.go index 568865690c..3811bd8cf4 100644 --- a/src/pkg/github/prdetailcache.go +++ b/src/pkg/github/prdetailcache.go @@ -144,6 +144,23 @@ func (c *prDetailCache) invalidate(repo string, number int) bool { return ok } +// peek returns the cached PR for (repo, number) regardless of TTL or +// mergeable state, for readers that only need the PR's identity (author, +// title, head) at the moment of a webhook delivery. +func (c *prDetailCache) peek(repo string, number int) (*gh.PullRequest, string, bool) { + if c == nil || number <= 0 { + return nil, "", false + } + key := prDetailCacheKey{repo: canonicalPRDetailRepo(repo), number: number} + c.mu.Lock() + defer c.mu.Unlock() + e, ok := c.entries[key] + if !ok || e.pr == nil { + return nil, "", false + } + return e.pr, e.headSHA, true +} + // numbersMatching returns the PR numbers cached for any of repos whose head // SHA, head ref or base ref satisfy match. func (c *prDetailCache) numbersMatching(repos []string, match func(headSHA, headRef, baseRef string) bool) []int { diff --git a/src/pkg/github/webhook_cache.go b/src/pkg/github/webhook_cache.go index 23db230b4a..9fbc0c94aa 100644 --- a/src/pkg/github/webhook_cache.go +++ b/src/pkg/github/webhook_cache.go @@ -74,6 +74,10 @@ type WebhookEventResult struct { // so a stage tracker whose confirming GET is refused or lost cannot keep // showing a closed item as open (hivecommons/hive#14159, #14064). Item *WebhookItemHint `json:"item,omitempty"` + // CIFailure is set when a check_run, check_suite or status delivery + // reported a failed check (failure / timed_out / error) on a head SHA, + // with the open PRs that head resolved to. + CIFailure *WebhookCIFailure `json:"ci_failure,omitempty"` } // WebhookItemHint is the state a webhook payload reports for the item it @@ -452,14 +456,26 @@ type webhookEventPayload struct { } `json:"comment"` CheckSuite *struct { HeadSHA string `json:"head_sha"` + Conclusion string `json:"conclusion"` + URL string `json:"url"` PullRequests []webhookPRRef `json:"pull_requests"` + App struct { + Name string `json:"name"` + } `json:"app"` } `json:"check_suite"` CheckRun *struct { HeadSHA string `json:"head_sha"` + Name string `json:"name"` + Conclusion string `json:"conclusion"` + DetailsURL string `json:"details_url"` + HTMLURL string `json:"html_url"` PullRequests []webhookPRRef `json:"pull_requests"` } `json:"check_run"` - SHA string `json:"sha"` - Branches []struct { + SHA string `json:"sha"` + State string `json:"state"` + Context string `json:"context"` + TargetURL string `json:"target_url"` + Branches []struct { Name string `json:"name"` } `json:"branches"` Ref string `json:"ref"` @@ -536,6 +552,10 @@ func (c *Client) HandleWebhookEvent(event string, payload []byte) (WebhookEventR } res.HeadSHA = webhookHeadSHA(event, p) numbers := webhookInvalidationTargets(event, p, aliases) + // Capture the failed-check identity and the PRs it resolved to BEFORE + // invalidation drops their cache entries: the receiver reacts to a red + // check on the event rather than on the next sweep. + res.CIFailure = c.webhookCIFailureResult(event, p, aliases, numbers) for _, n := range numbers { c.invalidateWebhookPR(aliases, n) } @@ -754,3 +774,41 @@ func (c *Client) invalidateWebhookPR(aliases []string, number int) { } sharedWebhookTracker.markDirty(aliases, number) } + +// webhookCIFailureResult pairs a failed-check delivery with the PRs it +// resolved to, read from the detail cache before the delivery invalidates +// them. A PR the cache does not hold is reported by number only. +func (c *Client) webhookCIFailureResult(event string, p webhookEventPayload, aliases []string, numbers []int) *WebhookCIFailure { + f := webhookCIFailure(event, p) + if f == nil || f.HeadSHA == "" { + return nil + } + for _, n := range numbers { + ref := CIFailurePR{Number: n} + for _, repo := range aliases { + pr, headSHA, ok := sharedPRDetailCache.peek(repo, n) + if !ok { + continue + } + if pr.GetState() != "" && pr.GetState() != "open" { + ref.Number = 0 + break + } + if headSHA != "" && !strings.EqualFold(headSHA, f.HeadSHA) { + // The cache knows a newer head: this failure is stale. + ref.Number = 0 + break + } + ref.Author = pr.GetUser().GetLogin() + ref.Title = pr.GetTitle() + ref.URL = pr.GetHTMLURL() + ref.BaseRef = pr.GetBase().GetRef() + ref.HiveAuthored = c.IsHiveAuthorLogin(ref.Author) + break + } + if ref.Number > 0 { + f.PRs = append(f.PRs, ref) + } + } + return f +} diff --git a/src/pkg/scheduler/ci_failure_signal_test.go b/src/pkg/scheduler/ci_failure_signal_test.go new file mode 100644 index 0000000000..c1e36063d1 --- /dev/null +++ b/src/pkg/scheduler/ci_failure_signal_test.go @@ -0,0 +1,125 @@ +package scheduler + +import ( + "log/slog" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/github" +) + +// A webhook CI-failure signal reaches the agent's fix-before-new section +// before ci-failing.json knows the PR, carrying the check name and the log +// pointer; a signal for a PR the file already lists adds to that row; a +// signal whose check is red on the base branch too is dropped. +func TestFormatRedPRFixDataWithSignals_MergesWebhookSignals(t *testing.T) { + at := time.Date(2026, 10, 21, 12, 0, 0, 0, time.UTC) + signals := []github.CIFailureSignal{ + {Repo: "test-org/console", Number: 22815, HeadSHA: "abc", Check: "lint", Conclusion: "failure", DetailsURL: "https://ci/lint/1", Agent: "scanner", At: at}, + {Repo: "test-org/console", Number: 500, HeadSHA: "def", Check: "unit", Conclusion: "timed_out", DetailsURL: "https://ci/unit/2", Title: "fresh red PR", Agent: "scanner", At: at}, + {Repo: "test-org/console", Number: 600, HeadSHA: "fed", Check: "build-gate", Agent: "scanner", At: at}, + } + data := []byte(`{"ci_failing":[ + {"number":22815,"repo":"test-org/console","title":"[Refactor] split tests","agent":"scanner","failing_checks":["build-gate"],"excerpt":"old evidence"}, + {"number":600,"repo":"test-org/console","title":"base is red","agent":"scanner","failing_checks":["build-gate"],"base_red":true,"base_red_checks":["build-gate"]} + ]}`) + out := formatRedPRFixDataWithSignals(data, "scanner", signals) + for _, want := range []string{ + "failing CI or merge conflicts (2)", + "#22815 test-org/console", + "failing: build-gate, lint", + "old evidence", + "webhook: lint failed — log: https://ci/lint/1", + "#500 test-org/console — fresh red PR", + "failing: unit", + "webhook: unit failed (timed_out) — log: https://ci/unit/2", + "do NOT watch, poll, or sleep on CI", + } { + if !strings.Contains(out, want) { + t.Errorf("section missing %q:\n%s", want, out) + } + } + if strings.Contains(out, "#600 test-org/console — base is red") { + t.Errorf("base-red check must stay excluded even when the webhook re-reports it:\n%s", out) + } + if got := formatRedPRFixDataWithSignals(nil, "scanner", signals[1:2]); !strings.Contains(got, "#500 ") { + t.Errorf("signals alone (no ci-failing.json yet) must still render:\n%s", got) + } + if got := formatRedPRFixDataWithSignals(nil, "quality", signals); got != "" { + t.Errorf("another agent's signals must not render, got:\n%s", got) + } +} + +// CIFailureKickMessage is the targeted kick: a one-line trigger header that +// restates the no-polling rule, then the same fix-before-new section a +// regular kick carries, sourced from the signal store when the file is +// absent. Advisory agents get nothing. +func TestCIFailureKickMessage(t *testing.T) { + t.Cleanup(github.ResetCIFailureSignalsForTest(nil)) + orig := ciFailingPath + ciFailingPath = filepath.Join(t.TempDir(), "missing.json") + t.Cleanup(func() { ciFailingPath = orig }) + + cfg := &config.Config{ + Project: config.ProjectConfig{Org: "test-org", Repos: []string{"test-org/console"}}, + AgentRuntimeConfig: config.AgentRuntimeConfig{ + Agents: map[string]config.AgentConfig{ + "scanner": {Mode: "ISSUES_AND_PRS"}, + "advisor": {Mode: "ADVISORY"}, + }, + }, + } + logger := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelError})) + s := New(cfg, logger) + + sig := github.CIFailureSignal{Repo: "test-org/console", Number: 77, HeadSHA: "abc", Check: "build-gate", DetailsURL: "https://ci/77", Title: "scanner PR", Agent: "scanner"} + if got := s.CIFailureKickMessage("scanner", sig); got != "" { + t.Fatalf("no recorded signal and no file: expected empty message, got:\n%s", got) + } + github.CIFailureSignals().Record(sig) + got := s.CIFailureKickMessage("scanner", sig) + for _, want := range []string{ + "CI failed on test-org/console#77 (build-gate)", + "do not watch or poll CI yourself", + "FIX-BEFORE-NEW", + "#77 test-org/console — scanner PR", + "log: https://ci/77", + } { + if !strings.Contains(got, want) { + t.Errorf("kick missing %q:\n%s", want, got) + } + } + if got := s.CIFailureKickMessage("advisor", sig); got != "" { + t.Errorf("advisory agent must not get a CI kick, got:\n%s", got) + } + // The regular kick path carries the same signal until ci-failing.json + // catches up, so nothing is lost between the webhook and the sweep. + if msg := s.addRedPRFixFirst("scanner", "[agent:scanner]\nWORK\n"); !strings.Contains(msg, "#77 test-org/console") { + t.Errorf("regular kick must fold the webhook signal in:\n%s", msg) + } +} + +// Agents never watch or poll CI themselves: detection is out-of-band +// (webhook, or the sweep as fallback). The reviewer lane's prompt is the +// contract that says so, and must keep saying so. +func TestReviewerLanePromptKeepsDoNotWaitForCI(t *testing.T) { + s := reviewerTestScheduler(t, reviewerCloseACMMLevel, reviewerFixture) + workList := s.buildReviewerWorkList() + if workList == "" { + t.Fatal("fixture produced no work list") + } + for _, msg := range []string{ + s.buildReviewerMessageHardcoded("adjudicator", reviewerCloseACMMLevel, workList), + s.renderReviewerLaneTemplate("adjudicator", &github.ActionableResult{}, reviewerCloseACMMLevel, workList), + } { + for _, want := range []string{"Do Not Wait for CI", "Never watch, poll, or sleep on\nCI"} { + if !strings.Contains(msg, want) { + t.Errorf("reviewer prompt lost %q:\n%s", want, msg) + } + } + } +} diff --git a/src/pkg/scheduler/policy_overlays.go b/src/pkg/scheduler/policy_overlays.go index 4fde5f7697..33d2f54fe6 100644 --- a/src/pkg/scheduler/policy_overlays.go +++ b/src/pkg/scheduler/policy_overlays.go @@ -4,6 +4,7 @@ import ( "encoding/json" "fmt" "os" + "slices" "strings" "github.com/hivecommons/hive/pkg/github" @@ -186,15 +187,43 @@ func (s *Scheduler) addRedPRFixFirst(agentName string, message string) string { if message == "" || !s.isPRCapableAgent(agentName) { return message } + section := s.redPRFixSection(agentName) + if section == "" { + return message + } + return appendKickSection(message, section) +} + +// redPRFixSection renders the agent's fix-before-new section from +// ci-failing.json merged with the webhook CI-failure signals attributed to +// it (github.CIFailureSignals), so a check that went red seconds ago is +// listed — with its name and log pointer — before the next eval tick +// rewrites the file. Empty when the agent has nothing to repair. +func (s *Scheduler) redPRFixSection(agentName string) string { + base := s.cfg.BaseAgentName(agentName) data, err := os.ReadFile(ciFailingPath) if err != nil { - return message + data = nil } - section := formatRedPRFixData(data, s.cfg.BaseAgentName(agentName)) + return formatRedPRFixDataWithSignals(data, base, github.CIFailureSignals().ForAgent(base)) +} + +// CIFailureKickMessage is the targeted kick a webhook CI-failure signal +// sends to the owning agent: the same fix-before-new section a regular kick +// would carry, under a one-line header naming the trigger. Empty when the +// agent is not PR-capable or has no red PR to repair (e.g. every failing +// check is red on the base branch too). +func (s *Scheduler) CIFailureKickMessage(agentName string, sig github.CIFailureSignal) string { + if s == nil || !s.isPRCapableAgent(agentName) { + return "" + } + section := s.redPRFixSection(agentName) if section == "" { - return message + return "" } - return appendKickSection(message, section) + header := fmt.Sprintf("CI failed on %s#%d (%s) — repair it now. Detection was out-of-band (webhook); do not watch or poll CI yourself.\n", + sig.Repo, sig.Number, sig.Check) + return header + section } // heldRedPRNote is appended to a held PR's entry in the fix-before-new @@ -220,6 +249,30 @@ const heldRedPRExemptAgent = "outreach" // it is named in one line without repair instructions, and when every red PR // the agent owns is deferred there is no section — and no banner — at all. func formatRedPRFixData(data []byte, agent string) string { + return formatRedPRFixDataWithSignals(data, agent, nil) +} + +// ciFailureSignalEvidence is the evidence line a webhook signal contributes: +// the check name plus the log pointer, so the agent can open the run +// without listing checks itself. +func ciFailureSignalEvidence(sig github.CIFailureSignal) string { + line := "webhook: " + sig.Check + " failed" + if sig.Conclusion != "" && sig.Conclusion != "failure" { + line += " (" + sig.Conclusion + ")" + } + if sig.DetailsURL != "" { + line += " — log: " + sig.DetailsURL + } + return line +} + +// formatRedPRFixDataWithSignals is formatRedPRFixData with the agent's +// webhook CI-failure signals folded in: a signal for a PR the file already +// lists adds its check and log pointer to that row; a signal for a PR the +// file does not know yet (the file is rewritten per eval tick) becomes its +// own row. A signal whose check the file reports red on the base branch is +// dropped like any other base-red failure. +func formatRedPRFixDataWithSignals(data []byte, agent string, signals []github.CIFailureSignal) string { type ciFailingRow struct { Number int `json:"number"` Repo string `json:"repo"` @@ -248,9 +301,47 @@ func formatRedPRFixData(data []byte, agent string) string { var payload struct { Items []ciFailingRow `json:"ci_failing"` } - if json.Unmarshal(data, &payload) != nil { + if len(data) > 0 && json.Unmarshal(data, &payload) != nil { return "" } + if len(signals) > 0 { + known := map[string]int{} + for i, r := range payload.Items { + known[fmt.Sprintf("%s#%d", strings.ToLower(r.Repo), r.Number)] = i + } + for _, sig := range signals { + key := fmt.Sprintf("%s#%d", strings.ToLower(sig.Repo), sig.Number) + if i, ok := known[key]; ok { + r := &payload.Items[i] + if slices.Contains(r.BaseRedChecks, sig.Check) { + continue + } + if r.BaseRed { + // The file says every failing check is base-red; a new + // check the file has not seen yet changes that. + r.BaseRed = false + } + if !slices.Contains(r.FailingChecks, sig.Check) { + r.FailingChecks = append(r.FailingChecks, sig.Check) + } + r.Excerpt = strings.TrimSpace(r.Excerpt + "\n" + ciFailureSignalEvidence(sig)) + continue + } + row := ciFailingRow{ + Number: sig.Number, + Repo: sig.Repo, + Title: sig.Title, + Agent: sig.Agent, + FailingChecks: []string{sig.Check}, + Excerpt: ciFailureSignalEvidence(sig), + } + if row.Title == "" { + row.Title = "(title pending)" + } + payload.Items = append(payload.Items, row) + known[key] = len(payload.Items) - 1 + } + } var mine, deferred, baseRed []ciFailingRow forks := 0 for _, pr := range payload.Items {