diff --git a/scripts/check-label-taxonomy.py b/scripts/check-label-taxonomy.py new file mode 100644 index 00000000..53ffa434 --- /dev/null +++ b/scripts/check-label-taxonomy.py @@ -0,0 +1,238 @@ +#!/usr/bin/env python3 +"""Fail when automation references a label the catalogue cannot honour (#3216). + +The 2026-09-17 label/census audit found the catalogue had grown to 30 labels +with nothing checking that the ones automation touches actually work, and the +residue it could not settle unilaterally was exactly that gap: Dependabot's +config named three labels -- `github-actions`, `go`, `containers` -- that did +not exist, so every Dependabot PR silently dropped them. Nothing failed. The +taxonomy drifted for ten days between the audit naming it and an operator +creating them. + +GitHub applies a label that exists and drops one that does not, so a +nonexistent label is invisible: the PR opens, the grouping works, the triage +filter never matches. That is why this is a check and not a review step. + +Two kinds of automation touch labels, and only one of them needs the label to +pre-exist: + +1. **applies** -- `.github/dependabot.yml` lists labels for Dependabot to put + on the PRs it opens. The label must exist before the PR does, or it is + dropped without a word. This is where the 2026-09-17 finding lived. +2. **creates** -- the alarm watchers (`scripts/*-watch.py`, `main-health-watch`) + each own one label and run `gh label create` for it on demand, so absence + self-heals on the next sweep. The audit reached the same conclusion for + these: zero open usage does not make a watcher-created label obsolete. + +The ledger below is the decision the audit left open, now recorded next to +the thing that depends on it -- every automation-referenced label, and which +of the two kinds it is. Offline it cross-checks that ledger against the +sources; with `--live` it also asks GitHub whether each label is really there +and really has a description. + +The 24 human-applied topic labels (`enhancement`, `bug`, `ops`, ...) are +deliberately out of scope: nothing in the repo references them by name, so +there is no drift for this guard to catch, and claiming otherwise would make +it a style opinion. + +Usage: + python scripts/check-label-taxonomy.py # offline, CI-safe + python scripts/check-label-taxonomy.py --live # also query GitHub +""" +from __future__ import annotations + +import argparse +import json +import re +import shutil +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent + +# Declared taxonomy. kind="applies" means the label must pre-exist for +# Dependabot to attach it; kind="creates" means a watcher recreates it on +# demand. `since` records when the audit's open question was settled. +LEDGER: dict[str, dict[str, str]] = { + # --- applies: Dependabot puts these on the PRs it opens ----------------- + "dependencies": {"kind": "applies", "why": "every dependabot.yml entry carries it"}, + "python": {"kind": "applies", "why": "pip entries"}, + "go": {"kind": "applies", "why": "gomod entry; created 2026-09-27, closing #3216 decision 1"}, + "github-actions": {"kind": "applies", "why": "github-actions entry; created 2026-09-27, closing #3216 decision 1"}, + "containers": {"kind": "applies", "why": "both docker entries; created 2026-09-27, closing #3216 decision 1"}, + "frontend": {"kind": "applies", "why": "npm entry; distinct from the dashboard product-area topic (#3216 decision 2)"}, + # --- creates: a watcher owns the label and recreates it on demand --------- + "ci-queue-stall": {"kind": "creates", "why": "scripts/ci-queue-watch.py"}, + "disk-usage-alarm": {"kind": "creates", "why": "scripts/disk-usage-watch.py"}, + "backup-staleness-alarm": {"kind": "creates", "why": "scripts/backup-staleness-watch.py"}, + "compose-drift-alarm": {"kind": "creates", "why": "scripts/compose-drift-watch.py"}, + "main-red-alarm": {"kind": "creates", "why": "scripts/main-health-watch.py"}, +} + +DEPENDABOT = Path(".github") / "dependabot.yml" +WATCHER_GLOB = "*-watch.py" +WATCHER_LABELS = re.compile(r'^LABEL\s*=\s*"([^"]+)"', re.MULTILINE) +DEPENDABOT_LABELS = re.compile(r"labels:\s*\[([^\]]*)\]") +DEPENDABOT_LABELS_BLOCK = re.compile(r"^\s*labels:\s*$") +DEPENDABOT_BLOCK_ITEM = re.compile(r"^\s*-\s*(\S+)\s*$") + +# A page that comes back full is a truncated answer, not a complete one. +LIVE_PAGE = 1000 + + +def dependabot_applies(root: Path) -> dict[str, set[str]]: + """Map each label named in dependabot.yml to the entries naming it. + + Per-ecosystem rather than a flat set, so a failure names which entry + reached for a label that is not there. + + Both YAML spellings are read: the inline flow form the file uses today + (`labels: [dependencies, go]`) and the block form (`labels:` then `- go`). + Reading only the first would be a false pass, not a false alarm: someone + reformatting to block style -- the way a YAML formatter or a human tidying + a long label list would -- would have their labels stop being checked + entirely, with no failure anywhere to notice. + """ + path = root / DEPENDABOT + if not path.is_file(): + return {} + used: dict[str, set[str]] = {} + current = "" + lines = path.read_text(errors="replace").splitlines() + for index, line in enumerate(lines): + eco = re.match(r"\s*-\s*package-ecosystem:\s*(\S+)", line) + if eco: + current = eco.group(1) + inline = DEPENDABOT_LABELS.search(line) + if inline: + for raw in inline.group(1).split(","): + name = raw.strip().strip("\"'") + if name: + used.setdefault(name, set()).add(current) + continue + if DEPENDABOT_LABELS_BLOCK.match(line): + for follow in lines[index + 1 :]: + item = DEPENDABOT_BLOCK_ITEM.match(follow) + if item is None: + break + used.setdefault(item.group(1).strip().strip("\"'"), set()).add(current) + return used + + +def watcher_creates(root: Path) -> dict[str, set[str]]: + """Map each watcher-owned LABEL constant to the scripts declaring it.""" + used: dict[str, set[str]] = {} + for script in sorted((root / "scripts").glob(WATCHER_GLOB)): + for name in WATCHER_LABELS.findall(script.read_text(errors="replace")): + used.setdefault(name, set()).add(script.name) + return used + + +def live_catalogue(repo: str) -> dict[str, str]: + """Ask GitHub for the live catalogue as {name: description}. + + Empty dict when `gh` is unavailable or unauthenticated, so --live degrades + to a clear refusal rather than a false pass. A response that fills the + page is also treated as no answer: a truncated catalogue would report + every label past the cut as deleted, which is a scarier and equally wrong + reading of the same tree. + """ + if shutil.which("gh") is None: + return {} + proc = subprocess.run( + ["gh", "label", "list", "-R", repo, "--limit", str(LIVE_PAGE), "--json", "name,description"], + capture_output=True, + text=True, + ) + if proc.returncode != 0: + print(f"warning: gh label list failed, skipping live verification: {proc.stderr.strip()}", file=sys.stderr) + return {} + labels = json.loads(proc.stdout) + if len(labels) >= LIVE_PAGE: + print( + f"warning: gh returned {len(labels)} labels, the page limit; treating as unreadable " + f"rather than reporting the unlisted remainder as deleted", + file=sys.stderr, + ) + return {} + return {l["name"]: (l.get("description") or "") for l in labels} + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--repo-root", type=Path, default=ROOT, help="tree to check (default: this repo)") + parser.add_argument("--live", action="store_true", help="also verify labels exist on GitHub with a description") + parser.add_argument("--repo", default="", help="OWNER/REPO for --live (default: the origin remote)") + args = parser.parse_args() + root: Path = args.repo_root + + applies = dependabot_applies(root) + creates = watcher_creates(root) + referenced = set(applies) | set(creates) + failures: list[str] = [] + + # 1. A label automation applies but the ledger never adjudicated. This is + # the 2026-09-17 defect verbatim: dependabot.yml grew an ecosystem + # entry, someone named a label in it, and nothing forced the question + # "does this label exist, and who creates it?" to be answered. + for name, ecosystems in sorted(applies.items()): + if name not in LEDGER: + where = ", ".join(sorted(ecosystems)) + failures.append( + f"{DEPENDABOT}: labels {name!r} (ecosystem: {where}) but it is not in the " + f"check-label-taxonomy.py ledger; declare it kind=applies (must pre-exist) " + f"or kind=creates (a watcher recreates it)" + ) + + # 2. A ledger entry nothing references. Either the label stopped being + # automated -- in which case its `why` is now a lie and the entry goes + # stale -- or the regex missed a reference, which is worse to find here. + for name, entry in sorted(LEDGER.items()): + if name not in referenced: + failures.append( + f"ledger: {name!r} is declared but nothing in dependabot.yml or a watcher references it; " + f"drop the entry or fix the reference it was meant to cover" + ) + + # 3. A watcher declares a label the ledger does not know. Same class as 1, + # on the other side of the ledger. + for name, scripts in sorted(creates.items()): + if name not in LEDGER: + where = ", ".join(sorted(scripts)) + failures.append( + f"{where}: creates label {name!r} which is not in the ledger; declare it kind=creates" + ) + + # 4. Live: the catalogue must actually honour the ledger. A label can be + # correctly declared here and still be missing on GitHub, which is + # exactly how the 2026-09-17 finding survived an audit that never + # re-read the catalogue afterwards. + if args.live: + repo = args.repo + if not repo: + remote = subprocess.run(["git", "remote", "get-url", "origin"], cwd=root, capture_output=True, text=True) + repo = (remote.stdout.strip().removesuffix(".git").split("github.com/")[-1]) + catalogue = live_catalogue(repo) + if not catalogue: + print("error: --live needs a readable gh label list; cannot verify", file=sys.stderr) + return 2 + for name, entry in sorted(LEDGER.items()): + if name not in catalogue: + severity = "dropped on every PR" if entry["kind"] == "applies" else "recreated on next sweep" + failures.append(f"{repo}: {name!r} is not in the live catalogue ({severity})") + elif not catalogue[name].strip(): + failures.append(f"{repo}: {name!r} has an empty description; every other label documents itself") + + if failures: + print(f"{len(failures)} label taxonomy failure(s):") + for failure in failures: + print(f" - {failure}") + return 1 + scope = "live" if args.live else "offline" + print(f"label taxonomy holds ({scope}): {len(LEDGER)} automation labels, {len(referenced)} referenced, all declared") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/docs/test_3216_label_taxonomy.py b/tests/docs/test_3216_label_taxonomy.py new file mode 100644 index 00000000..bc11def7 --- /dev/null +++ b/tests/docs/test_3216_label_taxonomy.py @@ -0,0 +1,303 @@ +#!/usr/bin/env python3 +"""Tests for scripts/check-label-taxonomy.py (#3216). + +The guard exists because the 2026-09-17 label/census audit found Dependabot +naming three labels that did not exist (`github-actions`, `go`, `containers`) +and nothing in CI noticed: GitHub drops a nonexistent label silently, so every +Dependabot PR opened without them and no filter ever matched. A check that +cannot fail on that exact tree is worse than no check, because it is read as +coverage, so each rule is pinned here against a synthetic repo root -- the +script under test is the real one, pointed at a throwaway tree via +--repo-root, so no repository state is mutated and no copy of the logic is +under test. + +Pinned here: + +1. the real tree passes offline (a guard that fails on everything guards + nothing); +2. dependabot.yml naming a label the ledger never adjudicated fails, naming + the label and its ecosystem -- the 2026-09-17 defect verbatim; +3. a watcher declaring an undeclared label fails; +4. a ledger entry nothing references fails, so a label that stopped being + automated cannot sit in the taxonomy as a stale claim; +5. --live reports a label that is absent from the real catalogue, and one + whose description is empty (the `frontend` defect fixed in this change); +6. --live without a readable `gh` refuses with exit 2 rather than passing + silently. +""" +from __future__ import annotations + +import json +import os +import pathlib +import re +import shutil +import subprocess +import sys + +import pytest + +REPO_ROOT = pathlib.Path(__file__).resolve().parents[2] +SCRIPT = REPO_ROOT / "scripts" / "check-label-taxonomy.py" + + +def origin_slug() -> str: + """OWNER/REPO from the origin remote. + + Derived rather than hardcoded: a stale slug makes `gh label list` fail + against a repository that does not exist, and the live test then skips + while appearing to have verified the real catalogue. + """ + remote = subprocess.run( + ["git", "remote", "get-url", "origin"], cwd=REPO_ROOT, capture_output=True, text=True + ) + slug = remote.stdout.strip() + slug = slug.removesuffix(".git").split("github.com/")[-1].strip("/") + if "/" not in slug: + pytest.skip(f"cannot derive OWNER/REPO from origin remote: {remote.stdout!r}") + return slug + + +REPO = origin_slug() + +# Read the script's own page limit rather than mirroring it, so raising the +# limit in the guard cannot quietly turn this truncation test into a 1000-row +# page the stub never produces. +LIVE_PAGE = int( + re.search(r"^LIVE_PAGE\s*=\s*(\d+)", SCRIPT.read_text(encoding="utf-8"), re.MULTILINE).group(1) +) + +# The labels the real dependabot.yml applies, and the labels the real watchers +# own. A synthetic tree carrying exactly these is the "in sync" baseline every +# negative test perturbs -- and it must carry BOTH halves, or the five +# watcher-owned ledger entries read as stale and their failure lines mask the +# rule the test is actually about. +APPLIES = ["dependencies", "github-actions", "go", "containers", "python", "frontend"] +WATCHES = { + "ci-queue-watch.py": "ci-queue-stall", + "disk-usage-watch.py": "disk-usage-alarm", + "backup-staleness-watch.py": "backup-staleness-alarm", + "compose-drift-watch.py": "compose-drift-alarm", + "main-health-watch.py": "main-red-alarm", +} + + +def dependabot(labels: list[str]) -> str: + body = "version: 2\nupdates:\n" + for name in labels: + body += ( + " - package-ecosystem: npm\n" + " directory: /arcane/home/honeypot-dashboard/frontend-next\n" + f" labels: [dependencies, {name}]\n" + ) + return body + + +def build_tree(root: pathlib.Path, *, applies: list[str], watches: dict[str, str]) -> pathlib.Path: + """Write a minimal repo root the guard can be pointed at.""" + (root / ".github").mkdir(parents=True, exist_ok=True) + (root / "scripts").mkdir(parents=True, exist_ok=True) + (root / ".github" / "dependabot.yml").write_text(dependabot(applies)) + for script, label in watches.items(): + (root / "scripts" / script).write_text( + f'#!/usr/bin/env python3\n"""watcher"""\n\nLABEL = "{label}"\n' + ) + return root + + +def run(root: pathlib.Path, *extra: str) -> subprocess.CompletedProcess: + return subprocess.run( + [sys.executable, str(SCRIPT), "--repo-root", str(root), *extra], + capture_output=True, + text=True, + ) + + +def fake_gh(tmp_path: pathlib.Path, catalogue: dict[str, str] | None) -> pathlib.Path: + """Put a stub `gh` first on PATH that reports `catalogue` as the labels.""" + bindir = tmp_path / "bin" + bindir.mkdir(parents=True, exist_ok=True) + if catalogue is None: + (bindir / "gh").write_text("#!/bin/sh\nexit 1\n") + else: + payload = tmp_path / "catalogue.json" + payload.write_text(json.dumps([{"name": n, "description": d} for n, d in catalogue.items()])) + (bindir / "gh").write_text(f'#!/bin/sh\ncat "{payload}"\n') + (bindir / "gh").chmod(0o755) + return bindir + + +def full_catalogue(overrides: dict[str, str | None] | None = None) -> dict[str, str]: + """A catalogue where every ledger label exists and documents itself.""" + names = [ + "dependencies", "python", "go", "github-actions", "containers", "frontend", + "ci-queue-stall", "disk-usage-alarm", "backup-staleness-alarm", + "compose-drift-alarm", "main-red-alarm", + ] + catalogue = {n: f"{n} description" for n in names} + for name, description in (overrides or {}).items(): + if description is None: + catalogue.pop(name, None) + else: + catalogue[name] = description + return catalogue + + +# --- 1. the real tree passes ------------------------------------------------ + +def test_real_tree_passes_offline(): + proc = subprocess.run([sys.executable, str(SCRIPT)], capture_output=True, text=True) + assert proc.returncode == 0, proc.stdout + proc.stderr + assert "holds (offline)" in proc.stdout + + +def test_real_tree_passes_live(): + """The real catalogue must satisfy --live, if gh can be reached here.""" + if shutil.which("gh") is None: + pytest.skip("gh not on PATH") + probe = subprocess.run( + [sys.executable, str(SCRIPT), "--live", "--repo", REPO], capture_output=True, text=True + ) + if probe.returncode == 2: + pytest.skip(f"gh label list unavailable here: {probe.stderr.strip()}") + assert probe.returncode == 0, probe.stdout + probe.stderr + + +# --- 2. dependabot names a label nobody adjudicated ------------------------- + +def test_undeclared_dependabot_label_fails(tmp_path): + """The 2026-09-17 defect: dependabot.yml names a label with no ledger row.""" + root = build_tree(tmp_path / "tree", applies=APPLIES + ["ghost-ecosystem-label"], watches=WATCHES) + proc = run(root) + assert proc.returncode == 1, proc.stdout + assert "ghost-ecosystem-label" in proc.stdout + assert "not in the" in proc.stdout + assert "kind=applies" in proc.stdout, "failure should say how to resolve it" + + +def test_undeclared_label_names_its_ecosystem(tmp_path): + root = build_tree(tmp_path / "tree", applies=APPLIES + ["unlisted"], watches=WATCHES) + proc = run(root) + assert proc.returncode == 1 + assert "unlisted" in proc.stdout + assert "npm" in proc.stdout, "the failure should name the ecosystem that reached for it" + + +def test_undeclared_watcher_label_fails(tmp_path): + root = build_tree( + tmp_path / "tree", applies=APPLIES, watches={**WATCHES, "new-watch.py": "unlisted-alarm"} + ) + proc = run(root) + assert proc.returncode == 1, proc.stdout + assert "unlisted-alarm" in proc.stdout + assert "kind=creates" in proc.stdout + + +# --- 3. a ledger entry nothing references ----------------------------------- + +def test_stale_ledger_entry_fails(tmp_path): + """A tree that stops automating a label must not leave it in the taxonomy.""" + root = build_tree(tmp_path / "tree", applies=["dependencies"], watches={}) + proc = run(root) + assert proc.returncode == 1, proc.stdout + assert "is declared but nothing in dependabot.yml" in proc.stdout + assert "containers" in proc.stdout, "the unreferenced ledger entry should be named" + + +# --- 4. --live checks the catalogue, not just the ledger -------------------- + +def test_live_reports_label_absent_from_catalogue(tmp_path, monkeypatch): + root = build_tree(tmp_path / "tree", applies=APPLIES, watches=WATCHES) + bindir = fake_gh(tmp_path, full_catalogue({"containers": None})) + monkeypatch.setenv("PATH", f"{bindir}{os.pathsep}{os.environ['PATH']}") + proc = run(root, "--live", "--repo", REPO) + assert proc.returncode == 1, proc.stdout + assert "not in the live catalogue" in proc.stdout + assert "containers" in proc.stdout + assert "dropped on every PR" in proc.stdout, "an applies-label absence is the silent one" + + +def test_live_reports_empty_description(tmp_path, monkeypatch): + """The `frontend` defect: present, but documenting nothing.""" + root = build_tree(tmp_path / "tree", applies=APPLIES, watches=WATCHES) + bindir = fake_gh(tmp_path, full_catalogue({"frontend": ""})) + monkeypatch.setenv("PATH", f"{bindir}{os.pathsep}{os.environ['PATH']}") + proc = run(root, "--live", "--repo", REPO) + assert proc.returncode == 1, proc.stdout + assert "frontend" in proc.stdout + assert "empty description" in proc.stdout + + +def test_live_separator_only_counts_creates_as_recoverable(tmp_path, monkeypatch): + root = build_tree(tmp_path / "tree", applies=APPLIES, watches=WATCHES) + bindir = fake_gh(tmp_path, full_catalogue({"main-red-alarm": None})) + monkeypatch.setenv("PATH", f"{bindir}{os.pathsep}{os.environ['PATH']}") + proc = run(root, "--live", "--repo", REPO) + assert proc.returncode == 1, proc.stdout + assert "recreated on next sweep" in proc.stdout + + +def test_live_passes_when_catalogue_is_complete(tmp_path, monkeypatch): + root = build_tree(tmp_path / "tree", applies=APPLIES, watches=WATCHES) + bindir = fake_gh(tmp_path, full_catalogue()) + monkeypatch.setenv("PATH", f"{bindir}{os.pathsep}{os.environ['PATH']}") + proc = run(root, "--live", "--repo", REPO) + assert proc.returncode == 0, proc.stdout + proc.stderr + assert "holds (live)" in proc.stdout + + +# --- 5. --live must refuse, not pass, when it cannot see the catalogue ----- + +def test_live_without_gh_refuses(tmp_path, monkeypatch): + root = build_tree(tmp_path / "tree", applies=APPLIES, watches=WATCHES) + monkeypatch.setenv("PATH", str(tmp_path / "empty-bin")) + (tmp_path / "empty-bin").mkdir(exist_ok=True) + proc = run(root, "--live", "--repo", REPO) + assert proc.returncode == 2, proc.stdout + assert "cannot verify" in proc.stderr + + +def test_live_refuses_a_full_page_rather_than_calling_the_rest_deleted(tmp_path, monkeypatch): + """A response that fills the page is truncated, not a catalogue without + the other labels -- reporting those as deleted would be its own false + alarm, and a scarier one.""" + root = build_tree(tmp_path / "tree", applies=APPLIES, watches=WATCHES) + page = [{"name": f"label-{i}", "description": "d"} for i in range(LIVE_PAGE)] + bindir = tmp_path / "bin" + bindir.mkdir() + payload = tmp_path / "page.json" + payload.write_text(json.dumps(page)) + (bindir / "gh").write_text(f'#!/bin/sh\ncat "{payload}"\n') + (bindir / "gh").chmod(0o755) + monkeypatch.setenv("PATH", f"{bindir}{os.pathsep}{os.environ['PATH']}") + proc = run(root, "--live", "--repo", REPO) + assert proc.returncode == 2, proc.stdout + assert "page limit" in proc.stderr + + +# --- 6. both YAML spellings of a label list are read ----------------------- + +def test_block_style_label_list_is_not_a_false_pass(tmp_path): + """Reformatting `labels:` to a YAML block must not silently stop the + labels being checked -- that is a false pass, the failure mode this whole + guard exists to avoid.""" + root = build_tree(tmp_path / "tree", applies=APPLIES, watches=WATCHES) + # Rewrite one ecosystem's list in block style, adding an undeclared label. + path = root / ".github" / "dependabot.yml" + text = path.read_text() + text = text.replace( + " labels: [dependencies, frontend]\n", + " labels:\n - dependencies\n - frontend\n - block-style-ghost\n", + ) + path.write_text(text) + proc = run(root) + assert proc.returncode == 1, proc.stdout + assert "block-style-ghost" in proc.stdout, "block-style label was not read at all" + + +def test_inline_labels_still_parsed(tmp_path): + """The form the real file uses must keep working after block support.""" + root = build_tree(tmp_path / "tree", applies=APPLIES, watches=WATCHES) + proc = run(root) + assert proc.returncode == 0, proc.stdout + assert "holds (offline)" in proc.stdout