diff --git a/changelog.d/added-shared-ci-tracking-issues.md b/changelog.d/added-shared-ci-tracking-issues.md new file mode 100644 index 0000000000..04f6735e16 --- /dev/null +++ b/changelog.d/added-shared-ci-tracking-issues.md @@ -0,0 +1 @@ +- When one check fails on several open PRs in a repository (`shared_ci_issues.min_prs`, default 3) or trips the red-main breaker, the hive now files one `[shared-ci] failing across ` tracking issue itself, naming the check, the PRs and a log excerpt, and keeps its own issue up to date; before, only an agent triaging one of the red PRs filed it, so a check red on every contributor and bot PR in a repo could go unreported for a day. An incident someone else filed is left alone; disable with `shared_ci_issues.enabled: false`. diff --git a/src/cmd/hive/shared_ci_issues.go b/src/cmd/hive/shared_ci_issues.go new file mode 100644 index 0000000000..6a92e452ff --- /dev/null +++ b/src/cmd/hive/shared_ci_issues.go @@ -0,0 +1,61 @@ +package main + +import ( + "context" + "fmt" + "log/slog" + "strconv" + "strings" + "time" + + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/dashboard" + "github.com/hivecommons/hive/pkg/github" +) + +// sharedCIIssuesInterval spaces the shared-CI issue sweep. It reads the +// governor's last snapshot, so it costs GitHub calls only in repositories +// where a check is failing across several PRs. +const sharedCIIssuesInterval = 15 * time.Minute + +// runSharedCIIssuesSweep files or updates one `[shared-ci]` tracking issue per +// check failing across a repository (github.SweepSharedCIIssues), from the +// latest actionable snapshot. Gates, dedupe and caps live there; this is the +// config switch and the dashboard audit sink. +func runSharedCIIssuesSweep(ctx context.Context, ghClient *github.Client, cfg *config.Config, actionable *github.ActionableResult, tracker *github.SharedCIIssueTracker, dashSrv *dashboard.Server, logger *slog.Logger) { + if ghClient == nil || actionable == nil { + return + } + if cfg != nil && !cfg.SharedCIIssues.IsEnabled() { + return + } + minPRs := 0 + if cfg != nil { + minPRs = cfg.SharedCIIssues.MinPRsOrDefault() + } + result, err := ghClient.SweepSharedCIIssues(ctx, actionable, github.SharedCIIssueOptions{ + MinPRs: minPRs, + Tracker: tracker, + Audit: func(event github.SharedCIIssueEvent) { + if dashSrv == nil { + return + } + prs := make([]string, len(event.PRs)) + for i, n := range event.PRs { + prs[i] = "#" + strconv.Itoa(n) + } + detail := fmt.Sprintf("repo=%s, check=%s, issue=%d, prs=%s, on_main=%t", + event.Repo, event.Check, event.Issue, strings.Join(prs, ","), event.OnMain) + dashSrv.AuditLogRecord("system", "shared-ci-issue-"+event.Action, detail, "", event.Repo, event.Issue) + }, + }) + if err != nil { + if logger != nil { + logger.Warn("shared-ci issue sweep failed", "error", err) + } + return + } + if len(result.Events) > 0 && logger != nil { + logger.Info("shared-ci issue sweep complete", "clusters", result.Clusters, "written", len(result.Events), "skipped", result.Skipped) + } +} diff --git a/src/cmd/hive/sweep_workers.go b/src/cmd/hive/sweep_workers.go index b23d4c1047..008e774940 100644 --- a/src/cmd/hive/sweep_workers.go +++ b/src/cmd/hive/sweep_workers.go @@ -126,6 +126,7 @@ func (b *boot) sweepJobs() []sweepJob { // Initialize shared hook state before any worker starts. gfi, first, coverage, notify := b.gfiState(), b.firstTimerState(), b.rfcCoverageState(), b.rfcNotifyState() var unparkLast time.Time + sharedCIIssues := github.NewSharedCIIssueTracker() return []sweepJob{ {"automerge", fixed(autoMergeSweepInterval), sweepEssential, func(ctx context.Context, gh *github.Client, cfg *config.Config) { // Workers start before the first eval finishes. A nil snapshot @@ -149,6 +150,11 @@ func (b *boot) sweepJobs() []sweepJob { {"red_main", fixed(redMainSweepInterval), sweepUnstick, func(ctx context.Context, gh *github.Client, cfg *config.Config) { runRedMainSweepIfDue(ctx, gh, nil, b.logger) }}, + // Reads the last snapshot; files a [shared-ci] issue for a check + // failing across a repository when no agent has. + {"shared_ci_issues", fixed(sharedCIIssuesInterval), sweepUnstick, func(ctx context.Context, gh *github.Client, cfg *config.Config) { + runSharedCIIssuesSweep(ctx, gh, cfg, b.lastActionable.Load(), sharedCIIssues, b.dashSrv, b.logger) + }}, {"blocked_issues", fixed(blockedIssuesSweepInterval), sweepUnstick, func(ctx context.Context, gh *github.Client, cfg *config.Config) { runBlockedIssuesSweepIfDue(ctx, gh, cfg, b.dashSrv, nil, b.logger) }}, diff --git a/src/docs/github-write-surface.md b/src/docs/github-write-surface.md index 16c587b07c..f107b2c1da 100644 --- a/src/docs/github-write-surface.md +++ b/src/docs/github-write-surface.md @@ -161,6 +161,7 @@ writes no entry. | Stale-red base refresh (`update-branch` on a red PR behind its base whose failing checks are green on the base tip, once per head and base tip, no comment) | `stale_red_base_refresh.go` | yes (`stale-red-base-refresh`) | yes | | Open-PR dispatch guard (`has-open-pr` label on a refused issue) and duplicate-fix sweep (one `hive:duplicate-fix` comment; `hive/superseded` close of the later hive PR, or `needs-human` + `Parked` fact on escalation) | `open_pr_links.go`, `pr_duplicate_fix_sweep.go` | yes (`duplicate-fix-*`) | yes | | Duplicate-PR sweep suggestions | `duplicate_sweep.go` | yes (`duplicate-sweep-suggested`) | yes (target = surviving PR) | +| Shared-CI tracking issues (one `[shared-ci] failing across ` issue per check failing across a repo; body edits of its own issue only) | `shared_ci_issues.go` | yes (`shared-ci-issue-filed`, `shared-ci-issue-updated`) | yes / issue number | | Hold-label migration (one-time) | `hive_hold_migration.go` | yes (`hold_migration_label_added`, one per item labeled; the migration report file is still written) | yes | | Origin-label migration (one-time; maps `from-review` / `origin/review` / `origin/reviewer` to `agent/reviewer`, deletes retired reviewer-origin label definitions after open items are swapped, removes redundant `origin/` and bare `` labels from agent-filed issues, and backfills `origin/selfreport` on open `hive/selfreport` issues when no provenance label is present) | `origin_labels.go` | yes (`origin_label_migrated`, one per item swapped or backfilled, with `from`, `to`; report file `/data/origin-label-migration-v4.json`) | yes | | `hive/` label retire sweep (every boot until clean, #12539) | `hive_id_label_retire.go` | yes (`hive_id_label_retired`, one per open item stripped, with `label`, `origin_source`; report file `/data/hive-id-label-retire-.json`) | yes | diff --git a/src/docs/operator-reference.md b/src/docs/operator-reference.md index 4b4013d77b..502f4befbe 100644 --- a/src/docs/operator-reference.md +++ b/src/docs/operator-reference.md @@ -43,6 +43,8 @@ The reference is organized into the sections below. Existing links to this page - [Central stage engine rollout](operator-reference/sweeps-and-operations.md#central-stage-engine-rollout) - [Shared-CI rerun sweep (`shared_ci_rerun`)](operator-reference/sweeps-and-operations.md#shared-ci-rerun-sweep-shared_ci_rerun) + +- [Shared-CI tracking issues (`shared_ci_issues`)](operator-reference/sweeps-and-operations.md#shared-ci-tracking-issues-shared_ci_issues) - [Stale-red base refresh (`stale_red_base_refresh`)](operator-reference/sweeps-and-operations.md#stale-red-base-refresh-stale_red_base_refresh) diff --git a/src/docs/operator-reference/sweeps-and-operations.md b/src/docs/operator-reference/sweeps-and-operations.md index 02c8661a61..2336f7ce99 100644 --- a/src/docs/operator-reference/sweeps-and-operations.md +++ b/src/docs/operator-reference/sweeps-and-operations.md @@ -308,6 +308,52 @@ someone re-runs it. The shared-CI rerun sweep |---|---|---| | `shared_ci_rerun.enabled` | `true` (unset) | Set `false` to stop re-running PRs when a `[shared-ci]` incident closes. | +## Shared-CI tracking issues (`shared_ci_issues`) + +The `[shared-ci] failing across ` incident issue is what +the red-main breaker, the PR-request refusal, the kick builders and the rerun +sweep above all key on. Agents file it when they triage a red PR with +`hive-baseline-check.sh`, but when no agent picks one of the red PRs up (they +are all contributor or bot PRs, or forks the hive may only comment on), +nothing is filed and the same red check can sit on a dozen PRs for a day. The +shared-CI issue sweep (`src/pkg/github/shared_ci_issues.go`) files it from the +governor's own snapshot every ~15 minutes: + +- It groups the open PRs (held ones included) by repository and failing + check. A check failing on `min_prs` or more PRs (default 3, the same + threshold the baseline helper uses) is a shared failure. So is a required + check that has tripped the red-main breaker (`red_main_threshold`), even + with no red PR. A fork-publication guard check, red on every fork PR by + design, is not counted. +- A PR cluster must last 30 minutes before an issue is filed, so a burst of + runner errors that clears on the next run files nothing. A red-main check + is filed at once: the breaker already waited. +- When no open `[shared-ci]` issue names the check, it files one under that + exact title, labelled `agent/ci-maintainer` and `kind/bug` with origin + `governor`. The body names the check, the failing PRs, whether the check + is red on the base branch, and up to ten distinct lines from the failing + runs' annotations. A new red-main issue is recorded as the breaker's + tracking issue straight away. +- An open incident it filed (body marker ``) + is edited in place when the PR list or the evidence changes; edits do not + notify, and an unchanged failure costs no write. An incident an agent or a + person filed is left alone: it has already reached the triage queue. +- It never comments on the PRs, never stamps the `` + deferral marker and never closes the issue. Close it once the cause is + repaired; a PR that closes it counts as a fix for a red default branch. +- After an incident for the same check closes it waits 24 hours before + filing again (7 days after a "not planned" close). +- It reads nothing per PR. Only a repository with a shared failure costs + calls: one open-issue listing per pass, one closed-issue listing before a + filing, and the write itself. At most 5 writes per pass. Every write goes + through the write-back gate as `issue_file` and is audited as + `shared-ci-issue-filed` or `shared-ci-issue-updated`. + +| Key | Default | Meaning | +|---|---|---| +| `shared_ci_issues.enabled` | `true` (unset) | Set `false` to stop filing `[shared-ci]` tracking issues. | +| `shared_ci_issues.min_prs` | `3` (unset) | Open PRs that must fail the same check before it counts as shared; values below 2 are raised to 2. | + ## Stale-red base refresh (`stale_red_base_refresh`) A PR that went red on a check its base branch also broke stays red after the @@ -495,6 +541,7 @@ that level, the power the write-back exists to govern? | Welcome / outreach (`pkg/gfi`, `gfi_forge.go`, `pr_clanker_requested.go`) | good-first-issue selection | welcome comment, `good first issue` label | issues hive advertises | `welcome_comment` → L5 with label, L6 without | | Issue filing (`pkg/upstreamwatch/github_filer.go`, `advisory.go`, scanner/deps/regression agents, `dedupe_advisory_close.go`) | agents, advisory digest, upstream watch | **new** issues hive authors, pinned advisory issue + comments | `project.repos` | `issue_file` → L3, no label (this is what listing a repo opts in to; `per_repo.mode: issues-only` keeps only this) | | Code scanning sweep (`pkg/github/code_scanning.go`, `pkg/codescanning`) | hourly sweep (`code_scanning`, opt-in per repo) | **new** grouped issues from code scanning alerts, alert-table body edits, one closing comment + close when every alert is fixed or dismissed | the hive's own `hive/code-scanning` issues in repos with `code_scanning` on | `issue_file` → L3, no label (own items); see [code-scanning.md](../code-scanning.md) | +| Shared-CI tracking issues (`shared_ci_issues.go`) | 15-min sweep (`shared_ci_issues`) | **new** `[shared-ci] failing across ` issue, body edits of its own issue | repos where one check fails on `shared_ci_issues.min_prs`+ open PRs or trips the red-main breaker | `issue_file` → L3, no label (own items) | | Shared-CI rerun (`shared_ci_rerun.go`) | CI failure watcher | `rerun-failed-jobs`, `hive-shared-ci-rerun-` comment | PRs with failed shared workflows | `ci_rerun` → L6 + label | | Transient-CI rerun (`transient_ci_rerun.go`) | eval cycle, before the stuck-PR reaper | `rerun-failed-jobs`, once per head, no comment | hive-authored red PRs whose every failing run carries `hive-ci-transient` | `ci_rerun` (own items: allowed unless the repo is paused, off or issues-only) | | Stale-red base refresh (`cmd/hive/stale_red_base_refresh.go`, `pkg/github/stale_red_base_refresh.go`) | eval cycle, before the stuck-PR reaper, at most every 5 min | **branch push** (`update-branch` merge of the base), once per head and base tip, no comment | red PRs behind their base whose every failing check is green on the base tip; not drafts, held, non-editable forks, dependency-bot branches, 072 or 074 | `base_refresh` → hive-authored PRs, else L6 + label | diff --git a/src/docs/operator-reference/v6-controls.md b/src/docs/operator-reference/v6-controls.md index f35482ad1c..80aab3eae8 100644 --- a/src/docs/operator-reference/v6-controls.md +++ b/src/docs/operator-reference/v6-controls.md @@ -261,6 +261,9 @@ been red for **more than** the threshold: Everything lifts automatically on the first sweep that sees a completed green run of the check. +When no agent has filed the tracking issue by the time the breaker trips, the +[shared-CI issue sweep](sweeps-and-operations.md#shared-ci-tracking-issues-shared_ci_issues) +files it. A repo whose required-check set or check runs cannot be read keeps its last observation (a read error neither trips nor clears the breaker). Trust levels, the trusted-merger definition and the self-merge floor are unchanged. diff --git a/src/pkg/config/config.go b/src/pkg/config/config.go index c979d0f87b..5061943386 100644 --- a/src/pkg/config/config.go +++ b/src/pkg/config/config.go @@ -162,6 +162,10 @@ type Config struct { // `` once incident #n closes // (hivecommons/hive#12253). Default on. SharedCIRerun SharedCIRerunConfig `yaml:"shared_ci_rerun,omitempty" json:"shared_ci_rerun,omitempty"` + // SharedCIIssues files one `[shared-ci]` tracking issue when a check + // fails on several open PRs in a repository (or trips the red-main + // breaker) and no incident is open for it yet. Default on. + SharedCIIssues SharedCIIssuesConfig `yaml:"shared_ci_issues,omitempty" json:"shared_ci_issues,omitzero"` // Search controls the portfolio search indexer (hivecommons/hive#14635). // Default off. Search SearchConfig `yaml:"search,omitempty" json:"search,omitempty"` diff --git a/src/pkg/config/review_config.go b/src/pkg/config/review_config.go index d4ec0e1f73..34ef454103 100644 --- a/src/pkg/config/review_config.go +++ b/src/pkg/config/review_config.go @@ -799,6 +799,44 @@ func (c SharedCIRerunConfig) IsEnabled() bool { return c.Enabled == nil || *c.Enabled } +// DefaultSharedCIIssuesMinPRs is how many open PRs in one repository must +// fail the same check before the shared-CI issue sweep files a tracking +// issue for it. It matches hive-baseline-check.sh's default sibling +// threshold, so the sweep and the agents' triage helper call the same +// failure shared. +const DefaultSharedCIIssuesMinPRs = 3 + +// SharedCIIssuesConfig gates the shared-CI issue sweep +// (github.SweepSharedCIIssues): when one check fails on min_prs or more open +// PRs in a repository, or has tripped the red-main breaker, Hive files (or +// keeps up to date) one `[shared-ci] failing across ` +// tracking issue instead of waiting for an agent to triage a red PR. +type SharedCIIssuesConfig struct { + // Enabled is *bool so unset means on; set `shared_ci_issues.enabled: + // false` to disable. + Enabled *bool `yaml:"enabled,omitempty" json:"enabled,omitempty"` + // MinPRs is the number of open PRs that must fail one check; 0 means + // DefaultSharedCIIssuesMinPRs. Values below 2 are raised to 2: one red + // PR is that PR's own business. + MinPRs int `yaml:"min_prs,omitempty" json:"min_prs,omitempty"` +} + +// IsEnabled reports whether the sweep runs; unset means on. +func (c SharedCIIssuesConfig) IsEnabled() bool { + return c.Enabled == nil || *c.Enabled +} + +// MinPRsOrDefault returns the effective PR threshold. +func (c SharedCIIssuesConfig) MinPRsOrDefault() int { + switch { + case c.MinPRs <= 0: + return DefaultSharedCIIssuesMinPRs + case c.MinPRs < 2: + return 2 + } + return c.MinPRs +} + // AutoMergeRepoConfig carries per-repository auto-merge overrides. type AutoMergeRepoConfig struct { // RequiredChecks overrides auto_merge.required_checks for this repository. diff --git a/src/pkg/dashboard/settings_parity_allowlist.txt b/src/pkg/dashboard/settings_parity_allowlist.txt index e4381b0610..57b0fd952d 100644 --- a/src/pkg/dashboard/settings_parity_allowlist.txt +++ b/src/pkg/dashboard/settings_parity_allowlist.txt @@ -552,6 +552,8 @@ security.sensitive_paths[] # TODO #15640 self_experiments.scheduler.enabled # TODO #11854 self_experiments.scheduler.interval_s # TODO #11854 self_experiments.scheduler.mode # TODO #11854 +shared_ci_issues.enabled # shared-CI tracking issue sweep; yaml-only +shared_ci_issues.min_prs # shared-CI tracking issue sweep; yaml-only shared_ci_rerun.enabled # TODO #11854 swarm.themes..call_to_arms # TODO #11854 swarm.themes..event_name # TODO #11854 diff --git a/src/pkg/github/red_main.go b/src/pkg/github/red_main.go index f5d1184458..67359da749 100644 --- a/src/pkg/github/red_main.go +++ b/src/pkg/github/red_main.go @@ -2,7 +2,6 @@ package github import ( "context" - "fmt" "strings" "time" @@ -34,10 +33,6 @@ import ( // Every lookup failure leaves the repo's previous observation in place: // a read error never trips the breaker and never clears it either. -// redMainMaxIssuePages bounds the open-issue listing used to find the -// `[shared-ci]` tracking issue (newest first, 100 per page). -const redMainMaxIssuePages = 3 - // redMainMaxLiveIssueLookups bounds the issues a refused PR request names // that the refusal path reads live to find a just-filed tracking issue. const redMainMaxLiveIssueLookups = 3 @@ -286,34 +281,13 @@ func (c *Client) redMainAnnotatedPaths(ctx context.Context, owner, repo string, // `[shared-ci] failing across ` issue tracking it in // owner/repo (the lowest number when several are open). func (c *Client) openSharedCITrackingIssues(ctx context.Context, owner, repo string) (map[string]int, error) { - opts := &gh.IssueListByRepoOptions{ - State: "open", - Sort: "created", - Direction: "desc", - ListOptions: gh.ListOptions{PerPage: 100}, - } - out := map[string]int{} - for page := 0; page < redMainMaxIssuePages; page++ { - issues, resp, err := c.client.Issues.ListByRepo(ctx, owner, repo, opts) - if err != nil { - return nil, fmt.Errorf("listing open shared-CI incidents in %s/%s: %w", owner, repo, err) - } - for _, issue := range issues { - if issue == nil || issue.IsPullRequest() { - continue - } - check, ok := SharedCITrackingCheck(issue.GetTitle(), owner+"/"+repo) - if !ok { - continue - } - if n, seen := out[check]; !seen || issue.GetNumber() < n { - out[check] = issue.GetNumber() - } - } - if resp == nil || resp.NextPage == 0 { - break - } - opts.ListOptions.Page = resp.NextPage + issues, err := c.openSharedCIIncidentIssues(ctx, owner, repo) + if err != nil { + return nil, err + } + out := make(map[string]int, len(issues)) + for check, issue := range issues { + out[check] = issue.GetNumber() } return out, nil } diff --git a/src/pkg/github/shared_ci_issues.go b/src/pkg/github/shared_ci_issues.go new file mode 100644 index 0000000000..b6d3b0e465 --- /dev/null +++ b/src/pkg/github/shared_ci_issues.go @@ -0,0 +1,622 @@ +package github + +import ( + "context" + "fmt" + "sort" + "strings" + "sync" + "time" + + gh "github.com/google/go-github/v72/github" + + "github.com/hivecommons/hive/crux/logscrub" + "github.com/hivecommons/hive/pkg/advisory" + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/issueshape" + "github.com/hivecommons/hive/pkg/redmain" + "github.com/hivecommons/hive/pkg/writeback" +) + +// Shared-CI tracking issues: the deterministic half of the shared-CI +// detector. +// +// The `[shared-ci] failing across ` incident issue is +// what the red-main breaker, the PR-request refusal, the kick builders and +// the rerun sweep all key on, but until now only an agent filed it: the +// fix-lane policy tells an agent to run hive-baseline-check.sh on a red PR +// it is repairing and file the incident on a DEFER_TO_INCIDENT verdict. +// When no agent picked one of the red PRs up, nothing was filed. On the +// projectbluefin hive (2026-11-08..10) `zizmor` sat red on ten +// projectbluefin/server PRs and `validate` on nine projectbluefin/finpilot +// PRs for more than a day each: the scanner's kick listed every one of +// them (most as fork PRs it may only comment on) and it moved on, and the +// red-main breaker never saw either check, because zizmor's newest run on +// main predated the cause and finpilot's validate does not run on main. +// +// SweepSharedCIIssues closes that gap from the governor's own snapshot, at +// no extra cost per PR: when one check is failing on MinPRs or more open +// PRs in a repository, or has tripped the red-main breaker, it files one +// tracking issue under the same title the agents use, or keeps the one it +// filed earlier up to date. An incident someone else opened is left alone: +// it has already reached the triage queue. It never comments on the PRs. + +const ( + // SharedCIIssueMarker marks a tracking issue this sweep filed; only + // such an issue's body is ever edited. + SharedCIIssueMarker = "" + + // sharedCIIssueLane is the lane the tracking issue is routed to: CI is + // the ci-maintainer's work. + sharedCIIssueLane = "ci-maintainer" + + // DefaultSharedCIIssueMaxWrites caps the issue creates and body edits + // one pass makes. + DefaultSharedCIIssueMaxWrites = 5 + + // sharedCIIssueSettle is how long a PR cluster must persist before an + // issue is filed for it, so one burst of runner errors that reds a few + // PRs at once and clears on the next run does not file an incident. + // A red-main fact needs no settling: the breaker already waited hours. + sharedCIIssueSettle = 30 * time.Minute + + // After an incident for the same check closes, the sweep waits this + // long before filing a new one: a completed close gives the rerun + // sweep and branch updates time to turn the PRs green; a "not + // planned" close is a maintainer saying no. + sharedCIIssueRefileAfterCompleted = 24 * time.Hour + sharedCIIssueRefileAfterNotPlanned = 7 * 24 * time.Hour + + sharedCIIssueMaxListedPRs = 30 + sharedCIIssueMaxExcerptLines = 10 + // sharedCIIssueMaxIssuePages bounds each issue listing (newest first, + // 100 per page); the red-main sweep's tracking lookup shares it. + sharedCIIssueMaxIssuePages = 3 +) + +// sharedCIIssueLabels are applied to a new tracking issue. +var sharedCIIssueLabels = []string{config.AgentLaneLabel(sharedCIIssueLane), "kind/bug"} + +// SharedCIIssueTitle is the incident title for check in fullRepo, the +// form SharedCITrackingCheck parses. +func SharedCIIssueTitle(check, fullRepo string) string { + return fmt.Sprintf("%s %s failing across %s", sharedCIIncidentTitlePrefix, strings.TrimSpace(check), fullRepo) +} + +// SharedCICluster is one check failing across a repository: on PRs open +// PRs at once, on the default branch past the red-main threshold (Main), +// or both. +type SharedCICluster struct { + // Repo is "owner/repo". + Repo string + // Check is the check-run name as GitHub reports it. + Check string + // PRs are the open PRs failing Check, ascending. + PRs []int + // BaseRed is set when Check is also red on the newest run of one of + // those PRs' base branch tip (PullRequest.BaseRedChecks). + BaseRed bool + // Main is the red-main fact when the breaker tripped on Check. + Main *redmain.Fact + // Excerpt are the distinct annotation lines the failing runs carry for + // Check, without the ": " prefix. + Excerpt []string +} + +func (cl SharedCICluster) key() string { + return strings.ToLower(cl.Repo) + "\x00" + strings.ToLower(cl.Check) +} + +// SharedCIClusters groups red PRs by repository and failing check and keeps +// each group of at least minPRs PRs, plus every tripped red-main fact. +// repoKey normalizes a PR's repo to "owner/repo" (facts are keyed that +// way). A failing check that only refuses fork PRs (ForkGuardChecksFor) is +// not counted: it is red on every fork PR by design. The result is sorted +// by PR count, then repo and check, so a capped pass handles the widest +// failure first. +func SharedCIClusters(prs []PullRequest, facts map[string]*redmain.Fact, repoKey func(string) string, minPRs int) []SharedCICluster { + if minPRs < 2 { + minPRs = 2 + } + if repoKey == nil { + repoKey = func(s string) string { return s } + } + byKey := map[string]*SharedCICluster{} + seenPR := map[string]bool{} + seenLine := map[string]bool{} + ordered := append([]PullRequest(nil), prs...) + sort.SliceStable(ordered, func(i, j int) bool { return ordered[i].Number < ordered[j].Number }) + for _, pr := range ordered { + if pr.Number <= 0 || pr.CIStatus != "failure" { + continue + } + repo := repoKey(pr.Repo) + for _, check := range pr.FailingChecks { + if strings.TrimSpace(check) == "" || ForkGuardChecksFor(pr.FromFork, []string{check}, pr.CIFailureExcerpt) != nil { + continue + } + cl := SharedCICluster{Repo: repo, Check: check} + k := cl.key() + if byKey[k] == nil { + byKey[k] = &cl + } + entry := byKey[k] + prKey := fmt.Sprintf("%s#%d", k, pr.Number) + if seenPR[prKey] { + continue + } + seenPR[prKey] = true + entry.PRs = append(entry.PRs, pr.Number) + for _, name := range pr.BaseRedChecks { + if strings.EqualFold(name, check) { + entry.BaseRed = true + } + } + for _, line := range strings.Split(pr.CIFailureExcerpt, "\n") { + msg, ok := strings.CutPrefix(strings.TrimSpace(line), check+": ") + msg = strings.TrimSpace(msg) + if !ok || msg == "" || seenLine[k+"\x00"+msg] || len(entry.Excerpt) >= sharedCIIssueMaxExcerptLines { + continue + } + seenLine[k+"\x00"+msg] = true + entry.Excerpt = append(entry.Excerpt, msg) + } + } + } + var out []SharedCICluster + for _, cl := range byKey { + if len(cl.PRs) >= minPRs { + out = append(out, *cl) + } + } + for repo, fact := range facts { + if fact == nil || strings.TrimSpace(fact.Check) == "" { + continue + } + cl := SharedCICluster{Repo: repoKey(repo), Check: fact.Check} + merged := false + for i := range out { + if out[i].key() == cl.key() { + out[i].Main = fact + merged = true + break + } + } + if !merged { + if pr := byKey[cl.key()]; pr != nil { + cl = *pr // fewer than minPRs PRs, still worth naming + } + cl.Main = fact + out = append(out, cl) + } + } + sort.Slice(out, func(i, j int) bool { + if len(out[i].PRs) != len(out[j].PRs) { + return len(out[i].PRs) > len(out[j].PRs) + } + if out[i].Repo != out[j].Repo { + return out[i].Repo < out[j].Repo + } + return out[i].Check < out[j].Check + }) + return out +} + +// sharedCIIssueExcerptLine makes one CI annotation line safe for the issue +// body: no code-fence breakout, and no `` span that the issue +// template guard would refuse as an unfilled placeholder. +func sharedCIIssueExcerptLine(s string) string { + s = strings.ReplaceAll(s, "```", "'''") + s = strings.ReplaceAll(s, "<", "‹") + return strings.ReplaceAll(s, ">", "›") +} + +// sharedCIIssuePRCount is "1 open pull request" or "N open pull requests". +func sharedCIIssuePRCount(n int) string { + if n == 1 { + return "1 open pull request" + } + return fmt.Sprintf("%d open pull requests", n) +} + +// RenderSharedCIIssueBody is the tracking issue body for cl. It carries no +// timestamp of its own, so an unchanged failure renders the same body and +// costs no edit. +func RenderSharedCIIssueBody(cl SharedCICluster) string { + var b strings.Builder + b.WriteString(SharedCIIssueMarker + "\n") + branch := "the default branch" + if cl.Main != nil && cl.Main.Branch != "" { + branch = "`" + cl.Main.Branch + "`" + } + switch { + case cl.Main != nil: + fmt.Fprintf(&b, "**Summary:** The required check `%s` has been red on %s of %s since %s, so pull requests "+ + "opened against it are red too", cl.Check, branch, cl.Repo, cl.Main.Since.UTC().Format("2006-01-02 15:04 UTC")) + if n := len(cl.PRs); n > 0 { + fmt.Fprintf(&b, " (%s failing it now)", sharedCIIssuePRCount(n)) + } + b.WriteString(". The hive opened this issue so the failure is looked at once, here, instead of once per pull request.\n\n") + default: + fmt.Fprintf(&b, "**Summary:** The `%s` check is failing on %s in %s at the same time. "+ + "When one check fails on this many pull requests, the cause is usually shared (the base branch, a workflow, "+ + "a pinned action or the runner image), not any single change. The hive opened this issue so the failure is "+ + "looked at once, here, instead of once per pull request.\n\n", cl.Check, sharedCIIssuePRCount(len(cl.PRs)), cl.Repo) + } + b.WriteString("### Evidence\n\n") + fmt.Fprintf(&b, "- Check: `%s`\n", cl.Check) + if len(cl.PRs) > 0 { + refs := make([]string, 0, len(cl.PRs)) + for i, n := range cl.PRs { + if i == sharedCIIssueMaxListedPRs { + refs = append(refs, fmt.Sprintf("and %d more", len(cl.PRs)-i)) + break + } + refs = append(refs, fmt.Sprintf("#%d", n)) + } + fmt.Fprintf(&b, "- Open pull requests failing it (%d): %s\n", len(cl.PRs), strings.Join(refs, ", ")) + } + switch { + case cl.Main != nil: + fmt.Fprintf(&b, "- Default branch: red on %s since %s", branch, cl.Main.Since.UTC().Format("2006-01-02 15:04 UTC")) + if cl.Main.RunURL != "" { + fmt.Fprintf(&b, " (failing run: %s)", cl.Main.RunURL) + } + b.WriteString("\n") + case cl.BaseRed: + b.WriteString("- Default branch: the newest run of this check on the pull requests' base branch is red too.\n") + default: + b.WriteString("- Default branch: the newest run of this check on the base branch is not red. Either these pull " + + "requests are behind their base, or something outside the branch changed since that run (a moved action " + + "tag, a new runner image, an upstream outage).\n") + } + if len(cl.Excerpt) > 0 { + b.WriteString("\nLog excerpt from the failing runs:\n\n```text\n") + for _, line := range cl.Excerpt { + b.WriteString(sharedCIIssueExcerptLine(line) + "\n") + } + b.WriteString("```\n") + } + b.WriteString("\n### Next step\n\n") + b.WriteString("Repair the shared cause once, usually with one pull request against the base branch or the shared " + + "workflow, then close this issue. Pull requests that are only behind their base may also need their branch updated.\n\n") + b.WriteString("_The hive keeps this description in step with the failing pull requests while the failure lasts " + + "(edits do not notify) and does not comment on the pull requests listed here._\n") + return b.String() +} + +// SharedCIIssueTracker remembers when each PR cluster was first seen, so a +// cluster is filed only once it has lasted sharedCIIssueSettle. The caller +// owns one per process; a restart only delays a filing. +type SharedCIIssueTracker struct { + mu sync.Mutex + firstSeen map[string]time.Time +} + +// NewSharedCIIssueTracker returns an empty tracker. +func NewSharedCIIssueTracker() *SharedCIIssueTracker { + return &SharedCIIssueTracker{firstSeen: map[string]time.Time{}} +} + +// observe records this pass's clusters and forgets the ones that are gone. +func (t *SharedCIIssueTracker) observe(clusters []SharedCICluster, now time.Time) { + if t == nil { + return + } + t.mu.Lock() + defer t.mu.Unlock() + present := map[string]bool{} + for _, cl := range clusters { + k := cl.key() + present[k] = true + if _, ok := t.firstSeen[k]; !ok { + t.firstSeen[k] = now + } + } + for k := range t.firstSeen { + if !present[k] { + delete(t.firstSeen, k) + } + } +} + +// settled reports whether cl may be filed at now. +func (t *SharedCIIssueTracker) settled(cl SharedCICluster, now time.Time) bool { + if cl.Main != nil || t == nil { + return true + } + t.mu.Lock() + defer t.mu.Unlock() + first, ok := t.firstSeen[cl.key()] + return ok && now.Sub(first) >= sharedCIIssueSettle +} + +// SharedCIIssueOptions bounds one SweepSharedCIIssues pass. +type SharedCIIssueOptions struct { + // MinPRs is the PR threshold; non-positive means + // config.DefaultSharedCIIssuesMinPRs. + MinPRs int + // MaxWrites caps creates plus edits; non-positive means + // DefaultSharedCIIssueMaxWrites. + MaxWrites int + // Tracker settles new PR clusters; nil files them on first sight. + Tracker *SharedCIIssueTracker + Audit func(SharedCIIssueEvent) + // Now overrides the clock in tests. + Now func() time.Time +} + +// SharedCIIssueEvent is one tracking issue filed or updated. +type SharedCIIssueEvent struct { + Repo string + Check string + Issue int + PRs []int + OnMain bool + // Action is "filed" or "updated". + Action string +} + +// SharedCIIssueResult summarizes one pass. +type SharedCIIssueResult struct { + Clusters int + Events []SharedCIIssueEvent + Skipped int +} + +// SweepSharedCIIssues files or updates one `[shared-ci]` tracking issue per +// check that is failing across a repository (see the file comment). +// actionable is the governor's latest snapshot; the sweep reads PRs and +// red-main facts from it and makes no per-PR call. Only repositories with a +// cluster are read: one open-issue listing each, plus one closed-issue +// listing before a new issue is filed. A failed lookup files nothing. +func (c *Client) SweepSharedCIIssues(ctx context.Context, actionable *ActionableResult, opts SharedCIIssueOptions) (*SharedCIIssueResult, error) { + if c == nil || c.client == nil { + return nil, ErrNoGitHubClient + } + res := &SharedCIIssueResult{} + if actionable == nil { + return res, nil + } + minPRs := opts.MinPRs + if minPRs <= 0 { + minPRs = config.DefaultSharedCIIssuesMinPRs + } + maxWrites := opts.MaxWrites + if maxWrites <= 0 { + maxWrites = DefaultSharedCIIssueMaxWrites + } + now := time.Now() + if opts.Now != nil { + now = opts.Now() + } + prs := make([]PullRequest, 0, len(actionable.PRs.Items)+len(actionable.PRs.Held)) + prs = append(prs, actionable.PRs.Items...) + prs = append(prs, actionable.PRs.Held...) + clusters := SharedCIClusters(prs, actionable.RedMain, c.redMainRepoKey, minPRs) + opts.Tracker.observe(clusters, now) + active := map[string]bool{} + for _, repo := range c.activeRepos() { + active[strings.ToLower(c.redMainRepoKey(repo))] = true + } + ctx = WithRESTCaller(ctx, "hive:shared_ci_issues") + openByRepo := map[string]map[string]*gh.Issue{} + writes := 0 + for _, cl := range clusters { + if ctx.Err() != nil { + return res, ctx.Err() + } + repoKey := strings.ToLower(cl.Repo) + if !active[repoKey] { + continue + } + res.Clusters++ + if writes >= maxWrites { + res.Skipped++ + continue + } + owner, name := c.splitRepo(cl.Repo) + open, known := openByRepo[repoKey] + if !known { + var err error + open, err = c.openSharedCIIncidentIssues(ctx, owner, name) + if err != nil { + c.logger.Debug("shared-ci issues: open incident lookup failed; filing nothing for the repo this pass", "repo", cl.Repo, "error", err) + } + openByRepo[repoKey] = open + } + if open == nil { + res.Skipped++ + continue + } + body := advisory.NeutralizeMentions(RenderSharedCIIssueBody(cl)) + if existing := open[strings.ToLower(cl.Check)]; existing != nil { + ev, wrote, err := c.updateSharedCIIssue(ctx, cl, existing, body) + if wrote { + writes++ + } + if err != nil { + c.logger.Warn("shared-ci issues: tracking issue update failed", "repo", cl.Repo, "check", cl.Check, "issue", existing.GetNumber(), "error", err) + res.Skipped++ + continue + } + if ev != nil { + res.Events = append(res.Events, *ev) + if opts.Audit != nil { + opts.Audit(*ev) + } + } + continue + } + if !opts.Tracker.settled(cl, now) { + continue + } + recent, err := c.sharedCIIssueRecentlyClosed(ctx, owner, name, cl.Check, now) + if err != nil || recent { + if err != nil { + c.logger.Debug("shared-ci issues: closed incident lookup failed; not filing", "repo", cl.Repo, "check", cl.Check, "error", err) + } + res.Skipped++ + continue + } + title := SharedCIIssueTitle(cl.Check, owner+"/"+name) + if d := c.WritebackAllowed(cl.Repo, writeback.Item{HiveAuthored: true}, writeback.KindIssueFile, title); !d.Allowed { + res.Skipped++ + continue + } + writes++ + created, err := c.CreateIssue(ctx, CreateIssueRequest{Repo: owner + "/" + name, Title: title, Body: body, Labels: sharedCIIssueLabels, Origin: config.OriginSourceGovernor}) // writeback:gated — WritebackAllowed consulted above + if err != nil { + c.logger.Warn("shared-ci issues: filing the tracking issue failed", "repo", cl.Repo, "check", cl.Check, "error", err) + res.Skipped++ + continue + } + open[strings.ToLower(cl.Check)] = &gh.Issue{Number: gh.Ptr(created.Number), Title: gh.Ptr(title), Body: gh.Ptr(body)} + if created.AlreadyExisted || created.Consolidated || created.RejectedTwin { + // An open twin already tracks it, or a maintainer rejected the + // same finding: nothing new was filed. + continue + } + if cl.Main != nil { + c.redMainTracker().SetTrackingIssue(c.redMainRepoKey(cl.Repo), cl.Main.Check, created.Number) + } + ev := SharedCIIssueEvent{Repo: owner + "/" + name, Check: cl.Check, Issue: created.Number, PRs: cl.PRs, OnMain: cl.Main != nil, Action: "filed"} + c.logger.Info("shared-ci issues: tracking issue filed", "repo", ev.Repo, "check", ev.Check, "issue", ev.Issue, "prs", len(ev.PRs), "on_main", ev.OnMain) + res.Events = append(res.Events, ev) + if opts.Audit != nil { + opts.Audit(ev) + } + } + return res, nil +} + +// sharedCIIssueOwned reports whether issue is a tracking issue this sweep +// filed: it carries the marker and, when the hive knows its own accounts, +// one of them wrote it. +func (c *Client) sharedCIIssueOwned(issue *gh.Issue) bool { + if issue == nil || !strings.Contains(issue.GetBody(), SharedCIIssueMarker) { + return false + } + return !c.knowsOwnAccounts() || c.isHiveAccount(issue.GetUser()) +} + +// updateSharedCIIssue edits the body of a tracking issue this sweep filed +// when the failure it describes changed. Any other open incident for the +// check is left alone. wrote reports an attempted GitHub write. +func (c *Client) updateSharedCIIssue(ctx context.Context, cl SharedCICluster, existing *gh.Issue, body string) (ev *SharedCIIssueEvent, wrote bool, err error) { + if !c.sharedCIIssueOwned(existing) { + return nil, false, nil + } + owner, name := c.splitRepo(cl.Repo) + if leak, ok := c.scanCanaryText(body, "hive-shared-ci-issue:"+owner+"/"+name); ok && c.canaryFailClosed { + return nil, false, fmt.Errorf("ioscan canary leak detected: agent=%s source=%s", leak.Agent, leak.Source) + } + body = logscrub.ScrubString(body) + if strings.TrimSpace(existing.GetBody()) == strings.TrimSpace(body) { + return nil, false, nil + } + item := writeback.Item{HiveAuthored: true, Number: existing.GetNumber(), Author: safeGetLogin(existing.GetUser()), Labels: issueLabelNames(existing.Labels)} + if d := c.WritebackAllowed(cl.Repo, item, writeback.KindIssueFile, existing.GetTitle()); !d.Allowed { + return nil, false, nil + } + var resp *gh.Response + _, resp, err = c.client.Issues.Edit(ctx, owner, name, existing.GetNumber(), &gh.IssueRequest{Body: gh.Ptr(body)}) // writeback:gated — WritebackAllowed consulted above + c.noteWritebackOutcome(owner+"/"+name, resp, err) + if err != nil { + return nil, true, err + } + existing.Body = gh.Ptr(body) + return &SharedCIIssueEvent{Repo: owner + "/" + name, Check: cl.Check, Issue: existing.GetNumber(), PRs: cl.PRs, OnMain: cl.Main != nil, Action: "updated"}, true, nil +} + +// sharedCIIncidentCheck is SharedCITrackingCheck that also accepts an +// incident an agent titled with its own tag in front +// (`[scanner] [shared-ci] lint failing across o/r`), so the sweep never +// files a second incident beside it. +func sharedCIIncidentCheck(title, fullRepo string) (string, bool) { + if check, ok := SharedCITrackingCheck(title, fullRepo); ok { + return check, true + } + if tag := issueshape.AgentPrefix(title); tag != "" && tag != "shared-ci" { + return SharedCITrackingCheck(issueshape.StripAgentPrefix(title), fullRepo) + } + return "", false +} + +// openSharedCIIncidentIssues maps a lower-cased check name to the open +// `[shared-ci] failing across ` issue in owner/repo +// (the lowest number when several are open). +func (c *Client) openSharedCIIncidentIssues(ctx context.Context, owner, repo string) (map[string]*gh.Issue, error) { + opts := &gh.IssueListByRepoOptions{ + State: "open", + Sort: "created", + Direction: "desc", + ListOptions: gh.ListOptions{PerPage: 100}, + } + out := map[string]*gh.Issue{} + for page := 0; page < sharedCIIssueMaxIssuePages; page++ { + issues, resp, err := c.client.Issues.ListByRepo(ctx, owner, repo, opts) + if err != nil { + return nil, fmt.Errorf("listing open shared-CI incidents in %s/%s: %w", owner, repo, err) + } + for _, issue := range issues { + if issue == nil || issue.IsPullRequest() { + continue + } + check, ok := sharedCIIncidentCheck(issue.GetTitle(), owner+"/"+repo) + if !ok { + continue + } + if prev, seen := out[check]; !seen || issue.GetNumber() < prev.GetNumber() { + out[check] = issue + } + } + if resp == nil || resp.NextPage == 0 { + break + } + opts.ListOptions.Page = resp.NextPage + } + return out, nil +} + +// sharedCIIssueRecentlyClosed reports whether an incident for check in +// owner/repo closed recently enough that a new one must wait: within +// sharedCIIssueRefileAfterCompleted of a completed close, or +// sharedCIIssueRefileAfterNotPlanned of a "not planned" one. +func (c *Client) sharedCIIssueRecentlyClosed(ctx context.Context, owner, repo, check string, now time.Time) (bool, error) { + want := strings.ToLower(strings.TrimSpace(check)) + since := now.Add(-sharedCIIssueRefileAfterNotPlanned) + opts := &gh.IssueListByRepoOptions{ + State: "closed", + Sort: "updated", + Direction: "desc", + Since: cacheableSince(since), + ListOptions: gh.ListOptions{PerPage: 100}, + } + for page := 0; page < sharedCIIssueMaxIssuePages; page++ { + issues, resp, err := c.client.Issues.ListByRepo(ctx, owner, repo, opts) + if err != nil { + return false, fmt.Errorf("listing closed shared-CI incidents in %s/%s: %w", owner, repo, err) + } + for _, issue := range issues { + if issue == nil || issue.IsPullRequest() || issue.ClosedAt == nil { + continue + } + if got, ok := sharedCIIncidentCheck(issue.GetTitle(), owner+"/"+repo); !ok || got != want { + continue + } + wait := sharedCIIssueRefileAfterCompleted + if strings.EqualFold(issue.GetStateReason(), "not_planned") { + wait = sharedCIIssueRefileAfterNotPlanned + } + if now.Sub(issue.GetClosedAt().Time) < wait { + return true, nil + } + } + if resp == nil || resp.NextPage == 0 { + break + } + opts.ListOptions.Page = resp.NextPage + } + return false, nil +} diff --git a/src/pkg/github/shared_ci_issues_test.go b/src/pkg/github/shared_ci_issues_test.go new file mode 100644 index 0000000000..47b67d85d8 --- /dev/null +++ b/src/pkg/github/shared_ci_issues_test.go @@ -0,0 +1,267 @@ +package github + +import ( + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/redmain" +) + +const sharedCIIssueZizmorLine = "zizmor: unit-tests.yml:45: action's hash pin has mismatched or missing version comment: tag points to commit cfdb7ed4ccc7" + +// zizmorPR is a red PR shaped like projectbluefin/server on 2026-11-10: +// zizmor failing on a head whose own diff did not touch the pin. +func zizmorPR(number int, fork bool) PullRequest { + return PullRequest{ + Repo: "widget", Number: number, CIStatus: "failure", FromFork: fork, + FailingChecks: []string{"zizmor"}, + CIFailureExcerpt: sharedCIIssueZizmorLine, + } +} + +// One check red on N+ open PRs is a cluster, whoever authored them; a check +// red on fewer PRs is not; a fork-publication guard (red on every fork PR by +// design) never counts; a tripped red-main fact is a cluster even with no +// red PR. +func TestSharedCIClustersFindsCheckFailingAcrossPRs(t *testing.T) { + both := zizmorPR(649, true) + both.FailingChecks = []string{"zizmor", "unit"} + both.CIFailureExcerpt = sharedCIIssueZizmorLine + "\nunit: TestBoot failed" + prs := []PullRequest{zizmorPR(642, false), zizmorPR(594, true), zizmorPR(613, true), both} + for _, n := range []int{972, 978, 980} { + prs = append(prs, PullRequest{ + Repo: "widget", Number: n, CIStatus: "failure", FromFork: true, + FailingChecks: []string{"Canary"}, + CIFailureExcerpt: "Canary: a fork pull request cannot publish the canary image", + }) + } + prs = append(prs, PullRequest{Repo: "widget", Number: 700, CIStatus: "success"}) + since := time.Date(2026, 11, 9, 6, 16, 0, 0, time.UTC) + facts := map[string]*redmain.Fact{"acme/other": {Repo: "acme/other", Branch: "main", Check: "build", Since: since, RunURL: "https://github.test/acme/other/runs/1"}} + repoKey := func(r string) string { + if strings.Contains(r, "/") { + return r + } + return "acme/" + r + } + + got := SharedCIClusters(prs, facts, repoKey, 3) + if len(got) != 2 { + t.Fatalf("clusters = %+v, want zizmor on acme/widget and build on acme/other", got) + } + z := got[0] + if z.Repo != "acme/widget" || z.Check != "zizmor" || z.Main != nil || z.BaseRed { + t.Fatalf("first cluster = %+v", z) + } + if want := []int{594, 613, 642, 649}; len(z.PRs) != len(want) || z.PRs[0] != 594 || z.PRs[3] != 649 { + t.Fatalf("zizmor PRs = %v, want %v", z.PRs, want) + } + if len(z.Excerpt) != 1 || !strings.HasPrefix(z.Excerpt[0], "unit-tests.yml:45: action's hash pin") { + t.Fatalf("excerpt = %q, want the one distinct zizmor line without its check prefix", z.Excerpt) + } + m := got[1] + if m.Repo != "acme/other" || m.Check != "build" || m.Main == nil || len(m.PRs) != 0 { + t.Fatalf("red-main cluster = %+v", m) + } + + body := RenderSharedCIIssueBody(z) + for _, want := range []string{SharedCIIssueMarker, "`zizmor` check is failing on 4 open pull requests in acme/widget", "#594, #613, #642, #649", "unit-tests.yml:45", "base branch is not red"} { + if !strings.Contains(body, want) { + t.Fatalf("body lacks %q:\n%s", want, body) + } + } + if span, bad := bodyPlaceholder(body); bad { + t.Fatalf("body trips the issue template guard on %q", span) + } + if mb := RenderSharedCIIssueBody(m); !strings.Contains(mb, "red on `main` of acme/other since 2026-11-09 06:16 UTC") || !strings.Contains(mb, "runs/1") { + t.Fatalf("red-main body:\n%s", mb) + } + if line := sharedCIIssueExcerptLine("expected got ```x```"); strings.ContainsAny(line, "<>") || strings.Contains(line, "```") { + t.Fatalf("excerpt line not made safe: %q", line) + } + if title := SharedCIIssueTitle("zizmor", "acme/widget"); title != "[shared-ci] zizmor failing across acme/widget" { + t.Fatalf("title = %q", title) + } else if check, ok := SharedCITrackingCheck(title, "acme/widget"); !ok || check != "zizmor" { + t.Fatalf("the title must parse as the incident the red-main breaker and the rerun sweep read: %q %v", check, ok) + } + if check, ok := sharedCIIncidentCheck("[scanner] [shared-ci] zizmor failing across acme/widget", "acme/widget"); !ok || check != "zizmor" { + t.Fatalf("an agent-tagged incident title must count as the incident: %q %v", check, ok) + } + if _, ok := sharedCIIncidentCheck("[scanner] zizmor is red", "acme/widget"); ok { + t.Fatal("an ordinary agent issue is not an incident") + } + if got := SharedCIClusters(prs, nil, repoKey, 5); len(got) != 0 { + t.Fatalf("min 5 PRs: clusters = %+v", got) + } +} + +// sharedCIIssueServer serves acme/widget's issue list and records creates +// and edits. open is what the open-issue listing returns; closed what the +// closed-issue listing returns. +type sharedCIIssueServer struct { + mu sync.Mutex + open []map[string]any + closed []map[string]any + created []map[string]any + edits []string +} + +func (s *sharedCIIssueServer) handler(t *testing.T) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + s.mu.Lock() + defer s.mu.Unlock() + w.Header().Set("Content-Type", "application/json") + switch { + case r.URL.Path == "/user": + _, _ = io.WriteString(w, `{"login":"hive-bot"}`) + case strings.Contains(r.URL.Path, "/labels"): + _, _ = io.WriteString(w, `{"name":"x"}`) + case r.Method == http.MethodGet && r.URL.Path == "/repos/acme/widget/issues": + list := []map[string]any{} + switch r.URL.Query().Get("state") { + case "open", "": + list = s.open + case "closed": + list = s.closed + } + _ = json.NewEncoder(w).Encode(list) + case r.Method == http.MethodPost && r.URL.Path == "/repos/acme/widget/issues": + var req map[string]any + _ = json.NewDecoder(r.Body).Decode(&req) + s.created = append(s.created, req) + _, _ = io.WriteString(w, `{"id":7701,"number":77,"html_url":"https://github.test/acme/widget/issues/77"}`) + case r.Method == http.MethodPatch && r.URL.Path == "/repos/acme/widget/issues/77": + var req map[string]any + _ = json.NewDecoder(r.Body).Decode(&req) + body, _ := req["body"].(string) + s.edits = append(s.edits, body) + _, _ = io.WriteString(w, `{"number":77}`) + default: + t.Errorf("unexpected %s %s", r.Method, r.URL.String()) + w.WriteHeader(http.StatusNotFound) + } + }) +} + +// The sweep files ONE tracking issue for a settled cluster, under the title +// the agents and the red-main breaker use, then keeps that issue's body in +// step (one edit per change, none when nothing changed) and never edits an +// incident it did not file. +func TestSweepSharedCIIssuesFilesOnceThenUpdatesOwnIssue(t *testing.T) { + s := &sharedCIIssueServer{} + srv := httptest.NewServer(s.handler(t)) + defer srv.Close() + c := newTestClient(t, srv, "acme", []string{"widget"}) + tracker := NewSharedCIIssueTracker() + t0 := time.Date(2026, 11, 10, 2, 0, 0, 0, time.UTC) + snap := &ActionableResult{} + snap.PRs.Items = []PullRequest{zizmorPR(594, true), zizmorPR(637, false), zizmorPR(642, false)} + var events []SharedCIIssueEvent + pass := func(at time.Time) { + t.Helper() + if _, err := c.SweepSharedCIIssues(context.Background(), snap, SharedCIIssueOptions{ + Tracker: tracker, Now: func() time.Time { return at }, + Audit: func(ev SharedCIIssueEvent) { events = append(events, ev) }, + }); err != nil { + t.Fatal(err) + } + } + + pass(t0) + if len(s.created) != 0 { + t.Fatalf("filed on first sight, before the cluster settled: %+v", s.created) + } + pass(t0.Add(sharedCIIssueSettle + time.Minute)) + if len(s.created) != 1 { + t.Fatalf("creates = %d, want 1", len(s.created)) + } + if title := s.created[0]["title"]; title != "[shared-ci] zizmor failing across acme/widget" { + t.Fatalf("title = %v", title) + } + body, _ := s.created[0]["body"].(string) + if !strings.Contains(body, SharedCIIssueMarker) || !strings.Contains(body, "#594, #637, #642") { + t.Fatalf("body:\n%s", body) + } + labels, _ := json.Marshal(s.created[0]["labels"]) + if !strings.Contains(string(labels), "agent/ci-maintainer") { + t.Fatalf("labels = %s, want the ci-maintainer lane", labels) + } + if len(events) != 1 || events[0].Action != "filed" || events[0].Issue != 77 { + t.Fatalf("events = %+v", events) + } + + // GitHub now lists the issue. The same failure costs no edit. + s.mu.Lock() + s.open = []map[string]any{{"number": 77, "title": "[shared-ci] zizmor failing across acme/widget", "body": body, "user": map[string]any{"login": "hive-bot"}}} + s.mu.Unlock() + pass(t0.Add(time.Hour)) + if len(s.created) != 1 || len(s.edits) != 0 { + t.Fatalf("unchanged failure: creates=%d edits=%d", len(s.created), len(s.edits)) + } + + // A fourth PR goes red: one edit, naming it. + snap.PRs.Items = append(snap.PRs.Items, zizmorPR(645, true)) + pass(t0.Add(75 * time.Minute)) + if len(s.created) != 1 || len(s.edits) != 1 || !strings.Contains(s.edits[0], "#645") { + t.Fatalf("grown failure: creates=%d edits=%v", len(s.created), s.edits) + } + if last := events[len(events)-1]; last.Action != "updated" || len(last.PRs) != 4 { + t.Fatalf("last event = %+v", last) + } + + // An incident an agent or a person opened is theirs: never edited, + // never duplicated. + s.mu.Lock() + s.open = []map[string]any{{"number": 77, "title": "[shared-ci] zizmor failing across acme/widget", "body": "Filed by hand.", "user": map[string]any{"login": "kylerankin"}}} + s.mu.Unlock() + snap.PRs.Items = append(snap.PRs.Items, zizmorPR(647, true)) + pass(t0.Add(90 * time.Minute)) + if len(s.created) != 1 || len(s.edits) != 1 { + t.Fatalf("foreign incident: creates=%d edits=%d", len(s.created), len(s.edits)) + } +} + +// A recently closed incident for the same check holds a new filing back; a +// red-main fact files at once (the breaker already waited) and tells the +// breaker its tracking issue. +func TestSweepSharedCIIssuesRespectsRecentCloseAndFilesRedMain(t *testing.T) { + s := &sharedCIIssueServer{} + srv := httptest.NewServer(s.handler(t)) + defer srv.Close() + c := newTestClient(t, srv, "acme", []string{"widget"}) + now := time.Date(2026, 11, 10, 12, 0, 0, 0, time.UTC) + + since := now.Add(-30 * time.Hour) + c.ObserveRedMain("widget", "main", map[string]redmain.RedRun{"build": {URL: "https://github.test/acme/widget/runs/9", CompletedAt: since}}, since) + fact := c.RedMainFact("widget", now) + if fact == nil { + t.Fatal("red-main fact not tripped") + } + snap := &ActionableResult{RedMain: map[string]*redmain.Fact{"acme/widget": fact}} + snap.PRs.Items = []PullRequest{zizmorPR(594, true), zizmorPR(637, false), zizmorPR(642, false)} + + s.closed = []map[string]any{{ + "number": 70, "title": "[shared-ci] zizmor failing across acme/widget", + "state": "closed", "state_reason": "completed", "closed_at": now.Add(-2 * time.Hour).Format(time.RFC3339), + }} + res, err := c.SweepSharedCIIssues(context.Background(), snap, SharedCIIssueOptions{Now: func() time.Time { return now }}) + if err != nil { + t.Fatal(err) + } + if len(s.created) != 1 || s.created[0]["title"] != "[shared-ci] build failing across acme/widget" { + t.Fatalf("creates = %+v, want only the red-main build incident (zizmor closed 2h ago)", s.created) + } + if res.Clusters != 2 || len(res.Events) != 1 || !res.Events[0].OnMain { + t.Fatalf("result = %+v", res) + } + if f := c.RedMainFact("widget", now); f == nil || f.TrackingIssue != 77 { + t.Fatalf("red-main fact after filing = %+v, want tracking issue 77", f) + } +}