Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/added-ci-failure-webhook-kick.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- A failed `check_run`, `check_suite` or `status` webhook now reacts within seconds instead of waiting for the next governor sweep: the head SHA is resolved to its open PR from the PR detail cache, recorded as a CI failure signal (`/api/status.ci_failure_signals`), and a hive-authored PR's owning agent is kicked at once with the check name and log URL while a contributor PR gets a targeted automerge/contributor-lane pass — gated by `governor.ci_failure_webhook_kick` (default on), repo/agent pauses and a per-head cooldown (`governor.ci_failure_kick_cooldown_s`, default 600). Agents still never watch or poll CI themselves.
167 changes: 167 additions & 0 deletions src/cmd/hive/ci_failure_kick.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,167 @@
package main

import (
"context"
"errors"
"log/slog"
"strings"
"time"

"github.com/hivecommons/hive/pkg/config"
"github.com/hivecommons/hive/pkg/dashboard"
"github.com/hivecommons/hive/pkg/github"
"github.com/hivecommons/hive/pkg/governor"
)

// ciFailureKickDefaultAgent is the agent an unattributed hive-authored PR is
// routed to, matching the fix-before-new builder's default (scanner is the
// fleet's primary PR creator).
const ciFailureKickDefaultAgent = "scanner"

// ciFailureKickSendTimeout bounds one targeted kick delivery; a stuck pane
// must not pin the worker.
const ciFailureKickSendTimeout = 2 * time.Minute

var errNoAgentRuntime = errors.New("agent runtime not available")

// ciFailureKicker turns a webhook CI-failure signal on a hive-authored PR
// into an immediate targeted kick of the owning agent. Gating mirrors the
// CEL trigger path (celTriggerKickAgents): config flag, agent enabled and
// not paused in config, runtime pause (which the fleet breaker also sets),
// governor eligibility (cadence pause, on-demand, budget), repo pause, and
// the per-PR-head cooldown kept by the signal store. The kick itself runs
// off the webhook goroutine so GitHub gets its 200 at once.
type ciFailureKicker struct {
cfg func() *config.Config
gov *governor.Governor
isPaused func(agent string) bool
message func(agent string, sig github.CIFailureSignal) string
attribute func(repo string, number int) string
send func(ctx context.Context, agent, message string) error
logger *slog.Logger
ctx context.Context
// spawn runs the kick delivery; tests swap in a synchronous runner.
spawn func(func())
}

func (b *boot) newCIFailureKicker() *ciFailureKicker {
k := &ciFailureKicker{
cfg: func() *config.Config { return b.cfg },
gov: b.gov,
attribute: github.AuthorAgentForPR,
logger: b.logger,
ctx: b.ctx,
spawn: func(fn func()) { go fn() },
}
// Resolve the scheduler and runtime per call: the dependencies closure
// may be built before boot finishes wiring them.
k.message = func(agent string, sig github.CIFailureSignal) string {
if b.sched == nil {
return ""
}
return b.sched.CIFailureKickMessage(agent, sig)
}
k.isPaused = func(agent string) bool { return b.agentMgr != nil && b.agentMgr.IsPaused(agent) }
k.send = func(_ context.Context, agent, message string) error {
if b.agentMgr == nil {
return errNoAgentRuntime
}
return b.agentMgr.SendKick(agent, message)
}
return k
}

// ciFailureKickFunc is the dashboard.Dependencies.CIFailureKick hook.
func (b *boot) ciFailureKickFunc() func(sig github.CIFailureSignal) dashboard.CIFailureKickOutcome {
k := b.newCIFailureKicker()
return k.React
}

// React decides and, when every gate passes, dispatches the kick.
func (k *ciFailureKicker) React(sig github.CIFailureSignal) dashboard.CIFailureKickOutcome {
if k == nil {
return dashboard.CIFailureKickOutcome{Reason: "no kicker"}
}
logger := k.logger
if logger == nil {
logger = slog.Default()
}
cfg := k.cfg()
if cfg == nil {
return dashboard.CIFailureKickOutcome{Reason: "no config"}
}
if !cfg.CIFailureWebhookKick() {
return dashboard.CIFailureKickOutcome{Reason: "disabled"}
}
agent := ""
if k.attribute != nil {
agent = strings.TrimSpace(k.attribute(sig.Repo, sig.Number))
}
if agent == "" {
agent = ciFailureKickDefaultAgent
}
out := dashboard.CIFailureKickOutcome{Agent: agent}
ac, ok := cfg.Agents[agent]
if !ok {
out.Reason = "unknown agent"
return out
}
if !ac.Enabled || ac.Paused {
out.Reason = "agent paused"
return out
}
if k.isPaused != nil && k.isPaused(agent) {
out.Reason = "agent paused"
return out
}
if cfg.IsRepoPaused(sig.Repo) {
out.Reason = "repo paused"
return out
}
if k.gov != nil && !k.gov.AgentEligibleForCELKick(agent) {
out.Reason = "governor gated"
return out
}
// Attribute before rendering so the fix-before-new builder lists the
// signal under this agent.
github.CIFailureSignals().SetAgent(sig.Repo, sig.Number, agent)
sig.Agent = agent
message := ""
if k.message != nil {
message = k.message(agent, sig)
}
if strings.TrimSpace(message) == "" {
out.Reason = "nothing to repair"
return out
}
if !github.CIFailureSignals().AllowKick(sig.Repo, sig.Number, sig.HeadSHA, cfg.CIFailureKickCooldown()) {
out.Reason = "cooldown"
return out
}
if k.send == nil {
out.Reason = "no kick sender"
return out
}
out.Kicked = true
spawn := k.spawn
if spawn == nil {
spawn = func(fn func()) { go fn() }
}
spawn(func() {
parent := k.ctx
if parent == nil {
parent = context.Background()
}
ctx, cancel := context.WithTimeout(parent, ciFailureKickSendTimeout)
defer cancel()
if err := k.send(ctx, agent, message); err != nil {
logger.Warn("ci failure webhook kick failed", "agent", agent, "repo", sig.Repo, "number", sig.Number, "check", sig.Check, "error", err)
return
}
if k.gov != nil {
k.gov.RecordKickForRepo(agent, sig.Repo)
}
logger.Info("ci failure webhook kick delivered", "agent", agent, "repo", sig.Repo, "number", sig.Number, "check", sig.Check)
})
return out
}
149 changes: 149 additions & 0 deletions src/cmd/hive/ci_failure_kick_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,149 @@
package main

import (
"context"
"errors"
"log/slog"
"os"
"testing"
"time"

"github.com/hivecommons/hive/pkg/config"
"github.com/hivecommons/hive/pkg/github"
"github.com/hivecommons/hive/pkg/governor"
)

type ciKickRecord struct {
agent, message string
}

func newCIFailureKickerForTest(t *testing.T, cfg *config.Config) (*ciFailureKicker, *[]ciKickRecord) {
t.Helper()
var sent []ciKickRecord
logger := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelError}))
k := &ciFailureKicker{
cfg: func() *config.Config { return cfg },
gov: governor.New(cfg.Governor, cfg.EnabledAgents(), logger),
attribute: func(repo string, number int) string { return "quality" },
message: func(agent string, sig github.CIFailureSignal) string {
return "CI failed on " + sig.Repo + " for " + agent
},
send: func(_ context.Context, agent, message string) error {
sent = append(sent, ciKickRecord{agent: agent, message: message})
return nil
},
logger: logger,
ctx: context.Background(),
spawn: func(fn func()) { fn() },
}
return k, &sent
}

func ciKickTestConfig() *config.Config {
return &config.Config{
Project: config.ProjectConfig{Org: "acme", Repos: []string{"widgets"}},
AgentRuntimeConfig: config.AgentRuntimeConfig{
Agents: map[string]config.AgentConfig{
"quality": {Enabled: true, Mode: "ISSUES_AND_PRS"},
"scanner": {Enabled: true, Mode: "ISSUES_AND_PRS"},
},
},
}
}

// The webhook kick honours every gate the CEL trigger path honours — flag,
// config pause, runtime pause, repo pause, governor eligibility — plus the
// per-head cooldown, and otherwise delivers one targeted kick to the agent
// the audit log attributes the PR to (scanner when unattributed).
func TestCIFailureKicker_GatesAndKicks(t *testing.T) {
now := time.Date(2026, 10, 21, 12, 0, 0, 0, time.UTC)
clock := func() time.Time { return now }
t.Cleanup(github.ResetCIFailureSignalsForTest(clock))
cfg := ciKickTestConfig()
k, sent := newCIFailureKickerForTest(t, cfg)
sig := github.CIFailureSignal{Repo: "acme/widgets", Number: 7, HeadSHA: "abc", Check: "build"}

out := k.React(sig)
if !out.Kicked || out.Agent != "quality" || len(*sent) != 1 || (*sent)[0].agent != "quality" {
t.Fatalf("first kick: %+v, sent = %+v", out, *sent)
}
if got := github.CIFailureSignals().Snapshot().Kicks; got != 1 {
t.Fatalf("kicks recorded = %d", got)
}

// Same head again inside the cooldown: refused.
if out := k.React(sig); out.Kicked || out.Reason != "cooldown" {
t.Fatalf("cooldown: %+v", out)
}
// New head: a fresh kick.
sig2 := sig
sig2.HeadSHA = "def"
if out := k.React(sig2); !out.Kicked || len(*sent) != 2 {
t.Fatalf("new head: %+v, sent = %d", out, len(*sent))
}
// Past the cooldown the original head may kick again.
now = now.Add(cfg.CIFailureKickCooldown() + time.Second)
if out := k.React(sig); !out.Kicked || len(*sent) != 3 {
t.Fatalf("after cooldown: %+v, sent = %d", out, len(*sent))
}

sig3 := sig
sig3.HeadSHA = "ghi"
ac := cfg.Agents["quality"]
ac.Paused = true
cfg.Agents["quality"] = ac
if out := k.React(sig3); out.Kicked || out.Reason != "agent paused" {
t.Fatalf("config pause: %+v", out)
}
ac.Paused = false
cfg.Agents["quality"] = ac

k.isPaused = func(string) bool { return true }
if out := k.React(sig3); out.Kicked || out.Reason != "agent paused" {
t.Fatalf("runtime pause: %+v", out)
}
k.isPaused = nil

cfg.Project.PausedRepos = []config.RepoPause{{Repo: "widgets"}}
if out := k.React(sig3); out.Kicked || out.Reason != "repo paused" {
t.Fatalf("repo pause: %+v", out)
}
cfg.Project.PausedRepos = nil

off := false
cfg.Governor.CIFailureWebhookKick = &off
if out := k.React(sig3); out.Kicked || out.Reason != "disabled" {
t.Fatalf("flag off: %+v", out)
}
cfg.Governor.CIFailureWebhookKick = nil

k.message = func(string, github.CIFailureSignal) string { return "" }
if out := k.React(sig3); out.Kicked || out.Reason != "nothing to repair" {
t.Fatalf("empty message: %+v", out)
}
k.message = func(agent string, _ github.CIFailureSignal) string { return "m" }

// Unattributed PR routes to scanner; an unknown agent is not kicked.
k.attribute = func(string, int) string { return "" }
if out := k.React(sig3); !out.Kicked || out.Agent != ciFailureKickDefaultAgent {
t.Fatalf("default agent: %+v", out)
}
k.attribute = func(string, int) string { return "ghost" }
sig4 := sig
sig4.HeadSHA = "jkl"
if out := k.React(sig4); out.Kicked || out.Reason != "unknown agent" {
t.Fatalf("unknown agent: %+v", out)
}

// A delivery failure is logged, not surfaced as a kick.
k.attribute = func(string, int) string { return "quality" }
k.send = func(context.Context, string, string) error { return errors.New("pane gone") }
sig5 := sig
sig5.HeadSHA = "mno"
if out := k.React(sig5); !out.Kicked {
t.Fatalf("send failure must still report dispatch: %+v", out)
}
if len(*sent) != 4 {
t.Fatalf("sent = %d, want 4", len(*sent))
}
}
10 changes: 8 additions & 2 deletions src/cmd/hive/forgestate_wire.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,12 @@ func (b *boot) forgeStateOpenPRs() *github.ForgeStateView {
}
view := b.ghClient.ForgeStateView()
if view == nil {
// No cache: the lanes run their listing ticker, but a trigger is
// still wired so a webhook CI-failure nudge runs a targeted pass
// at once (the legacy loop selects on it too).
if b.sweepTrigger == nil {
b.sweepTrigger = automerge.NewSweepTrigger(nil)
}
return nil
}
b.forgeView = view
Expand All @@ -50,8 +56,8 @@ func (b *boot) webhookDeliveryFunc() func(event, deliveryID string, payload []by

// sweepNudgeFunc is the dashboard.Dependencies.SweepNudge hook: schedule
// an automerge pass for one PR. The trigger is resolved per call because
// it is wired before the App client exists; without the forgestate cache
// the governor loop is woken instead, whose eval cycle runs the lanes.
// it is wired before the App client exists; before it is, the governor
// loop is woken instead, whose eval cycle runs the lanes.
func (b *boot) sweepNudgeFunc() func(repo string, number int) {
return func(repo string, number int) {
if trigger := b.sweepTrigger; trigger != nil {
Expand Down
1 change: 1 addition & 0 deletions src/cmd/hive/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -1481,6 +1481,7 @@ func (b *boot) wireBootClosures() {
WebhookDelivery: b.webhookDeliveryFunc(),
EvalLoop: b.evalLoopStatusFunc(),
SentinelPREvent: b.sentinelPREventFunc(),
CIFailureKick: b.ciFailureKickFunc(),
ProjectsV2ItemEvent: b.projectsV2ItemEventFunc(),
GHAppAuth: b.appAuth,
GHTokenScopes: b.ghAuth.TokenScopes,
Expand Down
2 changes: 1 addition & 1 deletion src/docs/api-reference.md
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ History point limits, timestamp cutoffs, incremental cost reads and ETag/304 beh
| `GET` | `/api/health/deep` | Public | Deep health probe | `pkg/dashboard/server.go:1594` |
| `GET` | `/api/livez` | Public | Kubernetes liveness probe | `pkg/dashboard/server.go:1595` |
| `GET` | `/metrics` | Registered only when `HIVE_METRICS_ENABLED`; requires `Authorization: ****** (403 if the token is unset) | Prometheus metrics | `pkg/dashboard/server.go:1601` |
| `GET` | `/api/status` | Dashboard auth/session | Dashboard aggregate status; `labelPresets[]` (`{key,label,description,labels[],states[]}`) lists the built-in label-filter presets for the Projects cards, led by `needs-human` whose `labels[]` are the effective hard-suppress labels plus `needs-signal`, the hold labels, `stage/071-needs-author` and the reporter-trust awaiting label (#12172); accepts `?fields=a,b` for top-level selection and `?omit=repos` to drop heavy sections, and honors `Accept-Encoding: gzip`; `agents[]` includes busy-pane visibility fields (`kicksUndeliverable`, `busySince`, `lastTranscriptActivity`, `condition`, `conditionMessage`) when a Working pane cannot accept kicks; a working agent also carries `agents[].run` — the live per-run line (#14576): `{since, elapsedS, costUSD, tokens, priced, tests, filesChanged}`, the estimated spend/tokens since the kick was delivered (`lastKickAt`, falling back to `busySince`) (delta of the token collector's per-agent cumulative figure; `priced:false` when the agent has none yet), the last visible test-runner result (`pass`/`fail`/empty) and the last `N files changed` git summary (`-1` when none) parsed from the captured pane; top-level `fleetRun` `{running, hourUSD, hourPriced}` is the governor strip's `running N · $X this hour` figure; `sla` (fleet) and `repos[].sla` (per repo) carry the Delivery SLA block derived from the lifecycle timeline with no extra GitHub calls: `windows.{7d,30d}.{open_to_triage,triage_to_start,start_to_pr,pr_to_merge,open_to_merge}` as `{p50_h,p95_h,n}` (hours, nearest-rank), `history[]` of up to 30 daily buckets `{day,start_to_pr_p50,pr_to_merge_p50,open_to_merge_p50,n_started,n_pr,n_merged}` persisted under `/data/sla-history.json`, `stale_in_progress[]` (`{number,repo,started_at,age_h,agent}` — issues carrying `hive/in-progress` longer than the agent claim TTL with no linked PR), `stale_count`, `merged_7d`, `claim_ttl_s` and, on the fleet block, the effective `targets` (`start_to_pr_h`, `pr_to_merge_h`, `open_to_merge_h`); `staleSince` and `repos[].countsStale` / `countsStaleSince` mark totals served from last-good figures — the previous enumeration (#13382) or the ForgeState cache's last-good counts (#13376) | `pkg/dashboard/server.go:1606` |
| `GET` | `/api/status` | Dashboard auth/session | Dashboard aggregate status; `labelPresets[]` (`{key,label,description,labels[],states[]}`) lists the built-in label-filter presets for the Projects cards, led by `needs-human` whose `labels[]` are the effective hard-suppress labels plus `needs-signal`, the hold labels, `stage/071-needs-author` and the reporter-trust awaiting label (#12172); accepts `?fields=a,b` for top-level selection and `?omit=repos` to drop heavy sections, and honors `Accept-Encoding: gzip`; `agents[]` includes busy-pane visibility fields (`kicksUndeliverable`, `busySince`, `lastTranscriptActivity`, `condition`, `conditionMessage`) when a Working pane cannot accept kicks; a working agent also carries `agents[].run` — the live per-run line (#14576): `{since, elapsedS, costUSD, tokens, priced, tests, filesChanged}`, the estimated spend/tokens since the kick was delivered (`lastKickAt`, falling back to `busySince`) (delta of the token collector's per-agent cumulative figure; `priced:false` when the agent has none yet), the last visible test-runner result (`pass`/`fail`/empty) and the last `N files changed` git summary (`-1` when none) parsed from the captured pane; top-level `fleetRun` `{running, hourUSD, hourPriced}` is the governor strip's `running N · $X this hour` figure; `sla` (fleet) and `repos[].sla` (per repo) carry the Delivery SLA block derived from the lifecycle timeline with no extra GitHub calls: `windows.{7d,30d}.{open_to_triage,triage_to_start,start_to_pr,pr_to_merge,open_to_merge}` as `{p50_h,p95_h,n}` (hours, nearest-rank), `history[]` of up to 30 daily buckets `{day,start_to_pr_p50,pr_to_merge_p50,open_to_merge_p50,n_started,n_pr,n_merged}` persisted under `/data/sla-history.json`, `stale_in_progress[]` (`{number,repo,started_at,age_h,agent}` — issues carrying `hive/in-progress` longer than the agent claim TTL with no linked PR), `stale_count`, `merged_7d`, `claim_ttl_s` and, on the fleet block, the effective `targets` (`start_to_pr_h`, `pr_to_merge_h`, `open_to_merge_h`); `staleSince` and `repos[].countsStale` / `countsStaleSince` mark totals served from last-good figures — the previous enumeration (#13382) or the ForgeState cache's last-good counts (#13376); `ci_failure_signals` `{count, last, kicks, unmatched}` is the webhook CI-failure signal store — distinct red (repo, head, check) signals retained, the most recent one `{repo, number, head_sha, check, conclusion, details_url, agent, at}`, webhook-triggered kicks/nudges since boot, and failure deliveries whose head matched no open PR | `pkg/dashboard/server.go:1606` |
| `GET` | `/api/status/summary` | Dashboard auth/session | Compact agent/governor/budget status summary for pollers; honors `Accept-Encoding: gzip` | `pkg/dashboard/server.go:1607` |
| `GET` | `/api/overview/issues.csv` | Dashboard auth/session | Cached Overview issue-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1608` |
| `GET` | `/api/overview/issues.json` | Dashboard auth/session | Cached Overview issue-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1609` |
Expand Down
Loading
Loading