Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/added-shared-ci-tracking-issues.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- When one check fails on several open PRs in a repository (`shared_ci_issues.min_prs`, default 3) or trips the red-main breaker, the hive now files one `[shared-ci] <check> failing across <owner/repo>` tracking issue itself, naming the check, the PRs and a log excerpt, and keeps its own issue up to date; before, only an agent triaging one of the red PRs filed it, so a check red on every contributor and bot PR in a repo could go unreported for a day. An incident someone else filed is left alone; disable with `shared_ci_issues.enabled: false`.
61 changes: 61 additions & 0 deletions src/cmd/hive/shared_ci_issues.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
package main

import (
"context"
"fmt"
"log/slog"
"strconv"
"strings"
"time"

"github.com/hivecommons/hive/pkg/config"
"github.com/hivecommons/hive/pkg/dashboard"
"github.com/hivecommons/hive/pkg/github"
)

// sharedCIIssuesInterval spaces the shared-CI issue sweep. It reads the
// governor's last snapshot, so it costs GitHub calls only in repositories
// where a check is failing across several PRs.
const sharedCIIssuesInterval = 15 * time.Minute

// runSharedCIIssuesSweep files or updates one `[shared-ci]` tracking issue per
// check failing across a repository (github.SweepSharedCIIssues), from the
// latest actionable snapshot. Gates, dedupe and caps live there; this is the
// config switch and the dashboard audit sink.
func runSharedCIIssuesSweep(ctx context.Context, ghClient *github.Client, cfg *config.Config, actionable *github.ActionableResult, tracker *github.SharedCIIssueTracker, dashSrv *dashboard.Server, logger *slog.Logger) {
if ghClient == nil || actionable == nil {
return
}
if cfg != nil && !cfg.SharedCIIssues.IsEnabled() {
return
}
minPRs := 0
if cfg != nil {
minPRs = cfg.SharedCIIssues.MinPRsOrDefault()
}
result, err := ghClient.SweepSharedCIIssues(ctx, actionable, github.SharedCIIssueOptions{
MinPRs: minPRs,
Tracker: tracker,
Audit: func(event github.SharedCIIssueEvent) {
if dashSrv == nil {
return
}
prs := make([]string, len(event.PRs))
for i, n := range event.PRs {
prs[i] = "#" + strconv.Itoa(n)
}
detail := fmt.Sprintf("repo=%s, check=%s, issue=%d, prs=%s, on_main=%t",
event.Repo, event.Check, event.Issue, strings.Join(prs, ","), event.OnMain)
dashSrv.AuditLogRecord("system", "shared-ci-issue-"+event.Action, detail, "", event.Repo, event.Issue)
},
})
if err != nil {
if logger != nil {
logger.Warn("shared-ci issue sweep failed", "error", err)
}
return
}
if len(result.Events) > 0 && logger != nil {
logger.Info("shared-ci issue sweep complete", "clusters", result.Clusters, "written", len(result.Events), "skipped", result.Skipped)
}
}
6 changes: 6 additions & 0 deletions src/cmd/hive/sweep_workers.go
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,7 @@ func (b *boot) sweepJobs() []sweepJob {
// Initialize shared hook state before any worker starts.
gfi, first, coverage, notify := b.gfiState(), b.firstTimerState(), b.rfcCoverageState(), b.rfcNotifyState()
var unparkLast time.Time
sharedCIIssues := github.NewSharedCIIssueTracker()
return []sweepJob{
{"automerge", fixed(autoMergeSweepInterval), sweepEssential, func(ctx context.Context, gh *github.Client, cfg *config.Config) {
// Workers start before the first eval finishes. A nil snapshot
Expand All @@ -149,6 +150,11 @@ func (b *boot) sweepJobs() []sweepJob {
{"red_main", fixed(redMainSweepInterval), sweepUnstick, func(ctx context.Context, gh *github.Client, cfg *config.Config) {
runRedMainSweepIfDue(ctx, gh, nil, b.logger)
}},
// Reads the last snapshot; files a [shared-ci] issue for a check
// failing across a repository when no agent has.
{"shared_ci_issues", fixed(sharedCIIssuesInterval), sweepUnstick, func(ctx context.Context, gh *github.Client, cfg *config.Config) {
runSharedCIIssuesSweep(ctx, gh, cfg, b.lastActionable.Load(), sharedCIIssues, b.dashSrv, b.logger)
}},
{"blocked_issues", fixed(blockedIssuesSweepInterval), sweepUnstick, func(ctx context.Context, gh *github.Client, cfg *config.Config) {
runBlockedIssuesSweepIfDue(ctx, gh, cfg, b.dashSrv, nil, b.logger)
}},
Expand Down
1 change: 1 addition & 0 deletions src/docs/github-write-surface.md
Original file line number Diff line number Diff line change
Expand Up @@ -161,6 +161,7 @@ writes no entry.
| Stale-red base refresh (`update-branch` on a red PR behind its base whose failing checks are green on the base tip, once per head and base tip, no comment) | `stale_red_base_refresh.go` | yes (`stale-red-base-refresh`) | yes |
| Open-PR dispatch guard (`has-open-pr` label on a refused issue) and duplicate-fix sweep (one `hive:duplicate-fix` comment; `hive/superseded` close of the later hive PR, or `needs-human` + `Parked` fact on escalation) | `open_pr_links.go`, `pr_duplicate_fix_sweep.go` | yes (`duplicate-fix-*`) | yes |
| Duplicate-PR sweep suggestions | `duplicate_sweep.go` | yes (`duplicate-sweep-suggested`) | yes (target = surviving PR) |
| Shared-CI tracking issues (one `[shared-ci] <check> failing across <owner/repo>` issue per check failing across a repo; body edits of its own issue only) | `shared_ci_issues.go` | yes (`shared-ci-issue-filed`, `shared-ci-issue-updated`) | yes / issue number |
| Hold-label migration (one-time) | `hive_hold_migration.go` | yes (`hold_migration_label_added`, one per item labeled; the migration report file is still written) | yes |
| Origin-label migration (one-time; maps `from-review` / `origin/review` / `origin/reviewer` to `agent/reviewer`, deletes retired reviewer-origin label definitions after open items are swapped, removes redundant `origin/<lane>` and bare `<lane>` labels from agent-filed issues, and backfills `origin/selfreport` on open `hive/selfreport` issues when no provenance label is present) | `origin_labels.go` | yes (`origin_label_migrated`, one per item swapped or backfilled, with `from`, `to`; report file `/data/origin-label-migration-v4.json`) | yes |
| `hive/<hive-id>` label retire sweep (every boot until clean, #12539) | `hive_id_label_retire.go` | yes (`hive_id_label_retired`, one per open item stripped, with `label`, `origin_source`; report file `/data/hive-id-label-retire-<hive-id>.json`) | yes |
Expand Down
2 changes: 2 additions & 0 deletions src/docs/operator-reference.md
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,8 @@ The reference is organized into the sections below. Existing links to this page
- [Central stage engine rollout](operator-reference/sweeps-and-operations.md#central-stage-engine-rollout)
<a id="shared-ci-rerun-sweep-shared_ci_rerun"></a>
- [Shared-CI rerun sweep (`shared_ci_rerun`)](operator-reference/sweeps-and-operations.md#shared-ci-rerun-sweep-shared_ci_rerun)
<a id="shared-ci-tracking-issues-shared_ci_issues"></a>
- [Shared-CI tracking issues (`shared_ci_issues`)](operator-reference/sweeps-and-operations.md#shared-ci-tracking-issues-shared_ci_issues)
<a id="stale-red-base-refresh-stale_red_base_refresh"></a>
- [Stale-red base refresh (`stale_red_base_refresh`)](operator-reference/sweeps-and-operations.md#stale-red-base-refresh-stale_red_base_refresh)
<a id="open-pr-dispatch-guard-and-duplicate-fix-sweep-scanner"></a>
Expand Down
47 changes: 47 additions & 0 deletions src/docs/operator-reference/sweeps-and-operations.md
Original file line number Diff line number Diff line change
Expand Up @@ -308,6 +308,52 @@ someone re-runs it. The shared-CI rerun sweep
|---|---|---|
| `shared_ci_rerun.enabled` | `true` (unset) | Set `false` to stop re-running PRs when a `[shared-ci]` incident closes. |

## Shared-CI tracking issues (`shared_ci_issues`)

The `[shared-ci] <check> failing across <owner/repo>` incident issue is what
the red-main breaker, the PR-request refusal, the kick builders and the rerun
sweep above all key on. Agents file it when they triage a red PR with
`hive-baseline-check.sh`, but when no agent picks one of the red PRs up (they
are all contributor or bot PRs, or forks the hive may only comment on),
nothing is filed and the same red check can sit on a dozen PRs for a day. The
shared-CI issue sweep (`src/pkg/github/shared_ci_issues.go`) files it from the
governor's own snapshot every ~15 minutes:

- It groups the open PRs (held ones included) by repository and failing
check. A check failing on `min_prs` or more PRs (default 3, the same
threshold the baseline helper uses) is a shared failure. So is a required
check that has tripped the red-main breaker (`red_main_threshold`), even
with no red PR. A fork-publication guard check, red on every fork PR by
design, is not counted.
- A PR cluster must last 30 minutes before an issue is filed, so a burst of
runner errors that clears on the next run files nothing. A red-main check
is filed at once: the breaker already waited.
- When no open `[shared-ci]` issue names the check, it files one under that
exact title, labelled `agent/ci-maintainer` and `kind/bug` with origin
`governor`. The body names the check, the failing PRs, whether the check
is red on the base branch, and up to ten distinct lines from the failing
runs' annotations. A new red-main issue is recorded as the breaker's
tracking issue straight away.
- An open incident it filed (body marker `<!-- hive-shared-ci-tracker -->`)
is edited in place when the PR list or the evidence changes; edits do not
notify, and an unchanged failure costs no write. An incident an agent or a
person filed is left alone: it has already reached the triage queue.
- It never comments on the PRs, never stamps the `<!-- hive-shared-ci-<n> -->`
deferral marker and never closes the issue. Close it once the cause is
repaired; a PR that closes it counts as a fix for a red default branch.
- After an incident for the same check closes it waits 24 hours before
filing again (7 days after a "not planned" close).
- It reads nothing per PR. Only a repository with a shared failure costs
calls: one open-issue listing per pass, one closed-issue listing before a
filing, and the write itself. At most 5 writes per pass. Every write goes
through the write-back gate as `issue_file` and is audited as
`shared-ci-issue-filed` or `shared-ci-issue-updated`.

| Key | Default | Meaning |
|---|---|---|
| `shared_ci_issues.enabled` | `true` (unset) | Set `false` to stop filing `[shared-ci]` tracking issues. |
| `shared_ci_issues.min_prs` | `3` (unset) | Open PRs that must fail the same check before it counts as shared; values below 2 are raised to 2. |

## Stale-red base refresh (`stale_red_base_refresh`)

A PR that went red on a check its base branch also broke stays red after the
Expand Down Expand Up @@ -495,6 +541,7 @@ that level, the power the write-back exists to govern?
| Welcome / outreach (`pkg/gfi`, `gfi_forge.go`, `pr_clanker_requested.go`) | good-first-issue selection | welcome comment, `good first issue` label | issues hive advertises | `welcome_comment` → L5 with label, L6 without |
| Issue filing (`pkg/upstreamwatch/github_filer.go`, `advisory.go`, scanner/deps/regression agents, `dedupe_advisory_close.go`) | agents, advisory digest, upstream watch | **new** issues hive authors, pinned advisory issue + comments | `project.repos` | `issue_file` → L3, no label (this is what listing a repo opts in to; `per_repo.mode: issues-only` keeps only this) |
| Code scanning sweep (`pkg/github/code_scanning.go`, `pkg/codescanning`) | hourly sweep (`code_scanning`, opt-in per repo) | **new** grouped issues from code scanning alerts, alert-table body edits, one closing comment + close when every alert is fixed or dismissed | the hive's own `hive/code-scanning` issues in repos with `code_scanning` on | `issue_file` → L3, no label (own items); see [code-scanning.md](../code-scanning.md) |
| Shared-CI tracking issues (`shared_ci_issues.go`) | 15-min sweep (`shared_ci_issues`) | **new** `[shared-ci] <check> failing across <owner/repo>` issue, body edits of its own issue | repos where one check fails on `shared_ci_issues.min_prs`+ open PRs or trips the red-main breaker | `issue_file` → L3, no label (own items) |
| Shared-CI rerun (`shared_ci_rerun.go`) | CI failure watcher | `rerun-failed-jobs`, `hive-shared-ci-rerun-<n>` comment | PRs with failed shared workflows | `ci_rerun` → L6 + label |
| Transient-CI rerun (`transient_ci_rerun.go`) | eval cycle, before the stuck-PR reaper | `rerun-failed-jobs`, once per head, no comment | hive-authored red PRs whose every failing run carries `hive-ci-transient` | `ci_rerun` (own items: allowed unless the repo is paused, off or issues-only) |
| Stale-red base refresh (`cmd/hive/stale_red_base_refresh.go`, `pkg/github/stale_red_base_refresh.go`) | eval cycle, before the stuck-PR reaper, at most every 5 min | **branch push** (`update-branch` merge of the base), once per head and base tip, no comment | red PRs behind their base whose every failing check is green on the base tip; not drafts, held, non-editable forks, dependency-bot branches, 072 or 074 | `base_refresh` → hive-authored PRs, else L6 + label |
Expand Down
3 changes: 3 additions & 0 deletions src/docs/operator-reference/v6-controls.md
Original file line number Diff line number Diff line change
Expand Up @@ -261,6 +261,9 @@ been red for **more than** the threshold:

Everything lifts automatically on the first sweep that sees a completed green
run of the check.
When no agent has filed the tracking issue by the time the breaker trips, the
[shared-CI issue sweep](sweeps-and-operations.md#shared-ci-tracking-issues-shared_ci_issues)
files it.
A repo whose required-check set or check runs cannot be read keeps its last
observation (a read error neither trips nor clears the breaker). Trust levels,
the trusted-merger definition and the self-merge floor are unchanged.
Expand Down
4 changes: 4 additions & 0 deletions src/pkg/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -162,6 +162,10 @@ type Config struct {
// `<!-- hive-shared-ci-<n> -->` once incident #n closes
// (hivecommons/hive#12253). Default on.
SharedCIRerun SharedCIRerunConfig `yaml:"shared_ci_rerun,omitempty" json:"shared_ci_rerun,omitempty"`
// SharedCIIssues files one `[shared-ci]` tracking issue when a check
// fails on several open PRs in a repository (or trips the red-main
// breaker) and no incident is open for it yet. Default on.
SharedCIIssues SharedCIIssuesConfig `yaml:"shared_ci_issues,omitempty" json:"shared_ci_issues,omitzero"`
// Search controls the portfolio search indexer (hivecommons/hive#14635).
// Default off.
Search SearchConfig `yaml:"search,omitempty" json:"search,omitempty"`
Expand Down
38 changes: 38 additions & 0 deletions src/pkg/config/review_config.go
Original file line number Diff line number Diff line change
Expand Up @@ -799,6 +799,44 @@ func (c SharedCIRerunConfig) IsEnabled() bool {
return c.Enabled == nil || *c.Enabled
}

// DefaultSharedCIIssuesMinPRs is how many open PRs in one repository must
// fail the same check before the shared-CI issue sweep files a tracking
// issue for it. It matches hive-baseline-check.sh's default sibling
// threshold, so the sweep and the agents' triage helper call the same
// failure shared.
const DefaultSharedCIIssuesMinPRs = 3

// SharedCIIssuesConfig gates the shared-CI issue sweep
// (github.SweepSharedCIIssues): when one check fails on min_prs or more open
// PRs in a repository, or has tripped the red-main breaker, Hive files (or
// keeps up to date) one `[shared-ci] <check> failing across <owner/repo>`
// tracking issue instead of waiting for an agent to triage a red PR.
type SharedCIIssuesConfig struct {
// Enabled is *bool so unset means on; set `shared_ci_issues.enabled:
// false` to disable.
Enabled *bool `yaml:"enabled,omitempty" json:"enabled,omitempty"`
// MinPRs is the number of open PRs that must fail one check; 0 means
// DefaultSharedCIIssuesMinPRs. Values below 2 are raised to 2: one red
// PR is that PR's own business.
MinPRs int `yaml:"min_prs,omitempty" json:"min_prs,omitempty"`
}

// IsEnabled reports whether the sweep runs; unset means on.
func (c SharedCIIssuesConfig) IsEnabled() bool {
return c.Enabled == nil || *c.Enabled
}

// MinPRsOrDefault returns the effective PR threshold.
func (c SharedCIIssuesConfig) MinPRsOrDefault() int {
switch {
case c.MinPRs <= 0:
return DefaultSharedCIIssuesMinPRs
case c.MinPRs < 2:
return 2
}
return c.MinPRs
}

// AutoMergeRepoConfig carries per-repository auto-merge overrides.
type AutoMergeRepoConfig struct {
// RequiredChecks overrides auto_merge.required_checks for this repository.
Expand Down
2 changes: 2 additions & 0 deletions src/pkg/dashboard/settings_parity_allowlist.txt
Original file line number Diff line number Diff line change
Expand Up @@ -552,6 +552,8 @@ security.sensitive_paths[] # TODO #15640
self_experiments.scheduler.enabled # TODO #11854
self_experiments.scheduler.interval_s # TODO #11854
self_experiments.scheduler.mode # TODO #11854
shared_ci_issues.enabled # shared-CI tracking issue sweep; yaml-only
shared_ci_issues.min_prs # shared-CI tracking issue sweep; yaml-only
shared_ci_rerun.enabled # TODO #11854
swarm.themes.<key>.call_to_arms # TODO #11854
swarm.themes.<key>.event_name # TODO #11854
Expand Down
40 changes: 7 additions & 33 deletions src/pkg/github/red_main.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@ package github

import (
"context"
"fmt"
"strings"
"time"

Expand Down Expand Up @@ -34,10 +33,6 @@ import (
// Every lookup failure leaves the repo's previous observation in place:
// a read error never trips the breaker and never clears it either.

// redMainMaxIssuePages bounds the open-issue listing used to find the
// `[shared-ci]` tracking issue (newest first, 100 per page).
const redMainMaxIssuePages = 3

// redMainMaxLiveIssueLookups bounds the issues a refused PR request names
// that the refusal path reads live to find a just-filed tracking issue.
const redMainMaxLiveIssueLookups = 3
Expand Down Expand Up @@ -286,34 +281,13 @@ func (c *Client) redMainAnnotatedPaths(ctx context.Context, owner, repo string,
// `[shared-ci] <check> failing across <owner/repo>` issue tracking it in
// owner/repo (the lowest number when several are open).
func (c *Client) openSharedCITrackingIssues(ctx context.Context, owner, repo string) (map[string]int, error) {
opts := &gh.IssueListByRepoOptions{
State: "open",
Sort: "created",
Direction: "desc",
ListOptions: gh.ListOptions{PerPage: 100},
}
out := map[string]int{}
for page := 0; page < redMainMaxIssuePages; page++ {
issues, resp, err := c.client.Issues.ListByRepo(ctx, owner, repo, opts)
if err != nil {
return nil, fmt.Errorf("listing open shared-CI incidents in %s/%s: %w", owner, repo, err)
}
for _, issue := range issues {
if issue == nil || issue.IsPullRequest() {
continue
}
check, ok := SharedCITrackingCheck(issue.GetTitle(), owner+"/"+repo)
if !ok {
continue
}
if n, seen := out[check]; !seen || issue.GetNumber() < n {
out[check] = issue.GetNumber()
}
}
if resp == nil || resp.NextPage == 0 {
break
}
opts.ListOptions.Page = resp.NextPage
issues, err := c.openSharedCIIncidentIssues(ctx, owner, repo)
if err != nil {
return nil, err
}
out := make(map[string]int, len(issues))
for check, issue := range issues {
out[check] = issue.GetNumber()
}
return out, nil
}
Expand Down
Loading
Loading