Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/fixed-16438-gate-regression-guard.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- A spoke that boots from the ConfigMap seed (runtime overlay moved aside or missing) no longer drops `review.require_labels`, `project.paused_repos`, `writeback.per_repo` or `project.issue_filter.require_labels` in silence: each gate a previous overlay on the PVC carried that the effective config lacks or loosens is logged as `WARN safety gate dropped` with its previous value, listed in `/api/status` as `config.gate_regressions`, and shown as a Settings banner; `config.refuse_gate_regression: true` refuses to start instead.
6 changes: 6 additions & 0 deletions src/cmd/hive/boot_config_deps.go
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,9 @@ type bootConfigDeps struct {
runHub func(logger *slog.Logger, configPath string)
// loadConfig is config.LoadWithDashboardOverlay.
loadConfig func(path string) (*config.Config, error)
// gateRegressions compares the loaded config against the previous
// runtime overlays on the PVC (#16438); nil-returning in tests.
gateRegressions func(cfg *config.Config) []config.GateRegression
// fileLogger builds the rolling-file logger from the loaded config.
fileLogger func(cfg *config.Config) *slog.Logger
// hiveID loads or generates the persistent hive identity.
Expand Down Expand Up @@ -98,6 +101,9 @@ func defaultBootConfigDeps() bootConfigDeps {
clearUpgradeMarker: func() error { return os.Remove(upgradeMarkerPath) },
runHub: runHub,
loadConfig: config.LoadWithDashboardOverlay,
gateRegressions: func(cfg *config.Config) []config.GateRegression {
return config.DetectPriorOverlayGateRegressions(cfg, config.PriorOverlayPaths())
},
fileLogger: func(cfg *config.Config) *slog.Logger {
l := cfg.Governor.Logging
return setupLogger(l.Dir, l.MaxSizeMB, l.MaxAgeDays, l.MaxBackups, l.Compress, l.Level)
Expand Down
61 changes: 61 additions & 0 deletions src/cmd/hive/boot_config_deps_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,7 @@ func newBootConfigFake(t *testing.T) *bootConfigFake {
cfg.Debug.PprofAddr = config.PprofAddrOff
return cfg, nil
},
gateRegressions: func(*config.Config) []config.GateRegression { return nil },
// Every logger the phase installs is captured, including the stdout
// bootstrap logger's replacement, so assertions see the whole boot.
fileLogger: func(*config.Config) *slog.Logger {
Expand Down Expand Up @@ -514,3 +515,63 @@ func TestBootConfigWith_HubModeNotGatedByProxyInjectGuard(t *testing.T) {
t.Fatalf("returned=%v ok=%v code=%d hubRun=%v, want the hub started", returned, ok, code, f.hubRun)
}
}

// #16438: a safety gate a previous runtime overlay carried that the effective
// config lacks is logged once per gate at WARN and kept on the config for
// /api/status; with config.refuse_gate_regression it is fatal, with its own
// exit code and the gates named in the log.
func TestBootConfigWith_GateRegressionWarnsOrRefuses(t *testing.T) {
regs := []config.GateRegression{
{Gate: config.GateReviewRequireLabels, Previous: "[hive-review]", Effective: "none", Source: "/data/hive.yaml.runtime.bak"},
{Gate: config.GatePausedRepos + "[acme/widgets]", Previous: "paused", Effective: "not paused", Source: "/data/hive.yaml.dashboard"},
}
t.Run("warns and boots by default", func(t *testing.T) {
f := newBootConfigFake(t)
f.deps.gateRegressions = func(*config.Config) []config.GateRegression { return regs }
b := &boot{}
if returned, ok, code := runBootConfig(t, b, f.deps); !returned || !ok {
t.Fatalf("returned=%v ok=%v code=%d, want a normal boot", returned, ok, code)
}
log := f.log.String()
if n := strings.Count(log, "safety gate dropped"); n != len(regs) {
t.Fatalf("want %d 'safety gate dropped' lines, got %d:\n%s", len(regs), n, log)
}
for _, want := range []string{"review.require_labels", "[hive-review]", "acme/widgets", "level=WARN"} {
if !strings.Contains(log, want) {
t.Fatalf("gate warning lacks %q:\n%s", want, log)
}
}
if len(b.cfg.GateRegressions) != len(regs) {
t.Fatalf("cfg.GateRegressions = %v, want the detector's list", b.cfg.GateRegressions)
}
})
t.Run("refuses when config.refuse_gate_regression is on", func(t *testing.T) {
f := newBootConfigFake(t)
load := f.deps.loadConfig
f.deps.loadConfig = func(p string) (*config.Config, error) {
cfg, err := load(p)
cfg.ConfigGuard.RefuseGateRegression = true
return cfg, err
}
f.deps.gateRegressions = func(*config.Config) []config.GateRegression { return regs }
returned, _, code := runBootConfig(t, &boot{}, f.deps)
if returned || code != gateRegressionExitCode {
t.Fatalf("returned=%v code=%d, want exit(%d)", returned, code, gateRegressionExitCode)
}
log := f.log.String()
for _, want := range []string{"refusing to start", "refuse_gate_regression", "review.require_labels", "acme/widgets"} {
if !strings.Contains(log, want) {
t.Fatalf("refusal log lacks %q:\n%s", want, log)
}
}
})
t.Run("quiet when nothing regressed", func(t *testing.T) {
f := newBootConfigFake(t)
if returned, ok, code := runBootConfig(t, &boot{}, f.deps); !returned || !ok {
t.Fatalf("returned=%v ok=%v code=%d, want a normal boot", returned, ok, code)
}
if strings.Contains(f.log.String(), "safety gate dropped") {
t.Fatal("gate warning logged with no regressions")
}
})
}
23 changes: 23 additions & 0 deletions src/cmd/hive/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -444,6 +444,11 @@ const (
// HIVE_PROXY_ADVISORY_OK=true). Its own code so a refusing pod's
// termination state says "fix the env", not "bug".
proxyInjectConfigExitCode = 19
// gateRegressionExitCode marks an exit caused by config.refuse_gate_regression
// (hivecommons/hive#16438): a previous runtime overlay carried a safety
// gate the effective config lacks. Its own code so the termination state
// says "restore the gate or the overlay", not "bug".
gateRegressionExitCode = 21
)

// singletonLockPath resolves where the process singleton lock lives. Every
Expand Down Expand Up @@ -654,6 +659,24 @@ func (b *boot) bootConfigWith(deps bootConfigDeps) bool {
"agents", b.cfg.RemovedAgents,
)

// Safety-gate regression check (#16438). A boot from the ConfigMap seed
// (runtime overlay moved aside, truncated, or never written) silently
// dropped review.require_labels on a live hive and the reviewer posted on
// unlabelled PRs for hours. Compare the effective config against whatever
// previous overlay is still on the PVC and say so, once per gate; refuse
// to start when the operator asked for that.
b.cfg.GateRegressions = deps.gateRegressions(b.cfg)
for _, g := range b.cfg.GateRegressions {
b.logger.Warn("safety gate dropped: "+g.String(),
"gate", g.Gate, "previous", g.Previous, "effective", g.Effective, "source", g.Source)
}
if len(b.cfg.GateRegressions) > 0 && b.cfg.ConfigGuard.RefuseGateRegression {
err := &config.GateRegressionError{Regressions: b.cfg.GateRegressions}
b.logger.Error("refusing to start: safety gate regression (config.refuse_gate_regression is on)", "error", err.Error())
deps.exit(gateRegressionExitCode)
return false
}

// Surface config provenance: when the persisted runtime config exists, init
// containers restore it over the ConfigMap seed on restart, so edits made
// only to the seed (or only to the live file) silently lose to it.
Expand Down
8 changes: 8 additions & 0 deletions src/deploy/entrypoint.sh
Original file line number Diff line number Diff line change
Expand Up @@ -1143,6 +1143,14 @@ PYEOF
fi
# Write the (merged) config as the disaster-recovery snapshot. Always
# under the new name; the legacy file is left untouched on the PVC.
#
# Reaching this branch on a hive that USED to have a runtime config means
# the safety gates it carried (review.require_labels, paused_repos, …)
# are now whatever the seed says β€” usually nothing (#16438). The Go
# binary compares the loaded config against hive.yaml.runtime.bak and
# hive.yaml.dashboard at boot and WARNs per dropped gate (fatal with
# config.refuse_gate_regression); an operator moving the runtime config
# aside should name the copy hive.yaml.runtime.bak so it is found.
cp "$HIVE_CONFIG_PATH" "$HIVE_CONFIG_RUNTIME" 2>/dev/null || true
hive_harden_runtime_config "$HIVE_CONFIG_RUNTIME"
echo "[entrypoint] K8s mode β€” ConfigMap is the seed, runtime config written to $HIVE_CONFIG_RUNTIME"
Expand Down
2 changes: 1 addition & 1 deletion src/docs/api-reference/health-status-events.md
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f
| `GET` | `/api/health/deep` | Public | Deep health probe | `pkg/dashboard/server.go:1644` |
| `GET` | `/api/livez` | Public | Kubernetes liveness probe | `pkg/dashboard/server.go:1645` |
| `GET` | `/metrics` | Registered only when `HIVE_METRICS_ENABLED`; requires `Authorization: ****** (403 if the token is unset) | Prometheus metrics | `pkg/dashboard/server.go:1651` |
| `GET` | `/api/status` | Dashboard auth/session | Dashboard aggregate status; `labelPresets[]` (`{key,label,description,labels[],states[]}`) lists the built-in label-filter presets for the Projects cards, led by `needs-human` whose `labels[]` are the effective hard-suppress labels plus `needs-signal`, the hold labels, `stage/071-needs-author` and the reporter-trust awaiting label (#12172); accepts `?fields=a,b` for top-level selection and `?omit=repos` to drop heavy sections, and honors `Accept-Encoding: gzip`; `agents[]` includes busy-pane visibility fields (`kicksUndeliverable`, `busySince`, `lastTranscriptActivity`, `condition`, `conditionMessage`) when a Working pane cannot accept kicks; a working agent also carries `agents[].run` β€” the live per-run line (#14576): `{since, elapsedS, costUSD, tokens, priced, tests, filesChanged}`, the estimated spend/tokens since the kick was delivered (`lastKickAt`, falling back to `busySince`) (delta of the token collector's per-agent cumulative figure; `priced:false` when the agent has none yet), the last visible test-runner result (`pass`/`fail`/empty) and the last `N files changed` git summary (`-1` when none) parsed from the captured pane; top-level `fleetRun` `{running, hourUSD, hourPriced, dayUSD, weekUSD, monthUSD, windowsPriced}` is the governor strip's rotating `running N Β· $X this hour / today / this week / this month` figure (#15614) β€” the hour is the rise since the top of the clock hour, the wider windows are the daily cost ledger summed from 00:00 UTC today, Monday 00:00 UTC (ISO week) and the 1st 00:00 UTC plus the rise since the ledger's last snapshot, `windowsPriced:false` when there is no ledger yet; `sla` (fleet) and `repos[].sla` (per repo) carry the Delivery SLA block derived from the lifecycle timeline with no extra GitHub calls: `windows.{7d,30d}.{open_to_triage,triage_to_start,start_to_pr,pr_to_merge,open_to_merge}` as `{p50_h,p95_h,n}` (hours, nearest-rank), `history[]` of up to 30 daily buckets `{day,start_to_pr_p50,pr_to_merge_p50,open_to_merge_p50,n_started,n_pr,n_merged}` persisted under `/data/sla-history.json`, `stale_in_progress[]` (`{number,repo,started_at,age_h,agent}` β€” issues carrying `hive/in-progress` longer than the agent claim TTL with no linked PR), `stale_count`, `merged_7d`, `claim_ttl_s` and, on the fleet block, the effective `targets` (`start_to_pr_h`, `pr_to_merge_h`, `open_to_merge_h`); `staleSince` and `repos[].countsStale` / `countsStaleSince` mark totals served from last-good figures β€” the previous enumeration (#13382) or the ForgeState cache's last-good counts (#13376); `ci_failure_signals` `{count, last, kicks, unmatched}` is the webhook CI-failure signal store β€” distinct red (repo, head, check) signals retained, the most recent one `{repo, number, head_sha, check, conclusion, details_url, agent, at}`, webhook-triggered kicks/nudges since boot, and failure deliveries whose head matched no open PR; `hotspots` `{count, actionable, filed}` is the merge-hotspot advisor's one-line summary (files at/above the thresholds, those with a split recommendation, those with a modularize issue filed), absent when the advisor is not wired β€” details on `GET /api/hotspots`; `writeback.dedupe` `{enabled, dedupe_same_item, dedupe_burst, rate_capped, hourly_cap, burst_items, burst_window_s, same_item_ttl_s}` is the write-back gate's content layer β€” the effective `writeback.dedupe.*` settings and the last-hour counts of writes held as `duplicate`, `burst` and `rate-capped` (each also listed under `GET /api/writeback/suppressed`) | `pkg/dashboard/server.go:1656` |
| `GET` | `/api/status` | Dashboard auth/session | Dashboard aggregate status; `labelPresets[]` (`{key,label,description,labels[],states[]}`) lists the built-in label-filter presets for the Projects cards, led by `needs-human` whose `labels[]` are the effective hard-suppress labels plus `needs-signal`, the hold labels, `stage/071-needs-author` and the reporter-trust awaiting label (#12172); accepts `?fields=a,b` for top-level selection and `?omit=repos` to drop heavy sections, and honors `Accept-Encoding: gzip`; `agents[]` includes busy-pane visibility fields (`kicksUndeliverable`, `busySince`, `lastTranscriptActivity`, `condition`, `conditionMessage`) when a Working pane cannot accept kicks; a working agent also carries `agents[].run` β€” the live per-run line (#14576): `{since, elapsedS, costUSD, tokens, priced, tests, filesChanged}`, the estimated spend/tokens since the kick was delivered (`lastKickAt`, falling back to `busySince`) (delta of the token collector's per-agent cumulative figure; `priced:false` when the agent has none yet), the last visible test-runner result (`pass`/`fail`/empty) and the last `N files changed` git summary (`-1` when none) parsed from the captured pane; top-level `fleetRun` `{running, hourUSD, hourPriced, dayUSD, weekUSD, monthUSD, windowsPriced}` is the governor strip's rotating `running N Β· $X this hour / today / this week / this month` figure (#15614) β€” the hour is the rise since the top of the clock hour, the wider windows are the daily cost ledger summed from 00:00 UTC today, Monday 00:00 UTC (ISO week) and the 1st 00:00 UTC plus the rise since the ledger's last snapshot, `windowsPriced:false` when there is no ledger yet; `sla` (fleet) and `repos[].sla` (per repo) carry the Delivery SLA block derived from the lifecycle timeline with no extra GitHub calls: `windows.{7d,30d}.{open_to_triage,triage_to_start,start_to_pr,pr_to_merge,open_to_merge}` as `{p50_h,p95_h,n}` (hours, nearest-rank), `history[]` of up to 30 daily buckets `{day,start_to_pr_p50,pr_to_merge_p50,open_to_merge_p50,n_started,n_pr,n_merged}` persisted under `/data/sla-history.json`, `stale_in_progress[]` (`{number,repo,started_at,age_h,agent}` β€” issues carrying `hive/in-progress` longer than the agent claim TTL with no linked PR), `stale_count`, `merged_7d`, `claim_ttl_s` and, on the fleet block, the effective `targets` (`start_to_pr_h`, `pr_to_merge_h`, `open_to_merge_h`); `staleSince` and `repos[].countsStale` / `countsStaleSince` mark totals served from last-good figures β€” the previous enumeration (#13382) or the ForgeState cache's last-good counts (#13376); `ci_failure_signals` `{count, last, kicks, unmatched}` is the webhook CI-failure signal store β€” distinct red (repo, head, check) signals retained, the most recent one `{repo, number, head_sha, check, conclusion, details_url, agent, at}`, webhook-triggered kicks/nudges since boot, and failure deliveries whose head matched no open PR; `hotspots` `{count, actionable, filed}` is the merge-hotspot advisor's one-line summary (files at/above the thresholds, those with a split recommendation, those with a modularize issue filed), absent when the advisor is not wired β€” details on `GET /api/hotspots`; `writeback.dedupe` `{enabled, dedupe_same_item, dedupe_burst, rate_capped, hourly_cap, burst_items, burst_window_s, same_item_ttl_s}` is the write-back gate's content layer β€” the effective `writeback.dedupe.*` settings and the last-hour counts of writes held as `duplicate`, `burst` and `rate-capped` (each also listed under `GET /api/writeback/suppressed`); `config.gate_regressions[]` `{gate, previous, effective, source}` is the boot-time safety-gate regression list (#16438) β€” gates a previous runtime overlay on the PVC carried that the running config lacks or loosens (`review.require_labels`, `project.issue_filter.require_labels`, `project.paused_repos[<repo>]`, `writeback.per_repo[<repo>].mode`), empty on a healthy boot | `pkg/dashboard/server.go:1656` |
| `GET` | `/api/status/summary` | Dashboard auth/session | Compact agent/governor/budget status summary for pollers; honors `Accept-Encoding: gzip` | `pkg/dashboard/server.go:1657` |
| `GET` | `/api/overview/issues.csv` | Dashboard auth/session | Cached Overview issue-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1658` |
| `GET` | `/api/overview/issues.json` | Dashboard auth/session | Cached Overview issue-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1659` |
Expand Down
17 changes: 17 additions & 0 deletions src/docs/config-layering.md
Original file line number Diff line number Diff line change
Expand Up @@ -189,6 +189,23 @@ Two consequences worth stating outright for a non-Kubernetes hive:
the hive to whatever the seed was provisioned with, silently downgrading
every hold-gated agent. Back up the volume, not just the seed file.

### Safety gates do not survive a seed boot β€” but the boot says so

A boot from the ConfigMap seed (no runtime config on the PVC, or one moved
aside during a recovery) runs with whatever gates the *seed* has, which on a
long-lived hive is usually none: `review.require_labels`,
`project.paused_repos`, `writeback.per_repo` and
`project.issue_filter.require_labels` all live in the overlay. On the a1 spoke
that silently re-enabled reviews on every unlabelled PR for four hours
([#16438](https://github.com/hivecommons/hive/issues/16438)). The Go binary now
compares the loaded config against any previous overlay still on the PVC
(`hive.yaml.runtime`, `hive.yaml.runtime.bak`, `hive.yaml.dashboard`) and logs
`WARN safety gate dropped: …` once per gate with the previous value; the same
list is served as `config.gate_regressions` on `/api/status` and shown as a
banner at the top of Settings. `config.refuse_gate_regression: true` turns the
warning into a refusal to start. Nothing about how the gates are *enforced*
changes; the entrypoint's choice of boot source is unchanged too.

### Migration

The rename is **copy-forward, never destructive**. Writers emit
Expand Down
Loading
Loading