From ba97e8969b72afe09c4141cbe2cfab4f3887c34e Mon Sep 17 00:00:00 2001 From: Antawari Date: Wed, 29 Jul 2026 13:33:34 -0600 Subject: [PATCH 1/3] Make every cf-gate say what it examined, not just that it passed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The board printed eleven bare labels and one measurement. complexipy was the only honest line because it alone runs in-process and builds its GateVerdict directly; every other gate crossed a subprocess boundary and the measurement did not survive the crossing. Three breaks, not one: 1. print_verdict built GateVerdict(gate, violations, error) and never passed notices or evidence. `notices` was a print-only parameter, so the wire shipped "notices": [], "evidence": {} for every gate — empty at the source. 2. _structured_verdict never read either field back off the parsed JSON. 3. No cf-gate computed a denominator at all. None counted what it examined, so closing (1) and (2) alone would still have printed a bare label. print_verdict now takes `measured` (one line, rides GateVerdict.notices to the wire) and `evidence` (the same measurement in machine form). The pre-existing `notices` parameter stays the gate's multi-line human prose and deliberately does NOT reach the wire, so a board line stays one line. _structured_verdict reads both back. Each of the seven cf-gates now threads a count out of the walk it actually performed — never a second walk, never a number the gate did not use. cf-file-budget additionally stops hand-rolling its own JSON and routes through the shared print_verdict, so it finally honours CF_QUALITY_JSON. A PASS with no denominator is indistinguishable from a PASS that examined nothing, so zero is the case that had to work: every gate renders a visible 0 on an empty tree, proven by control-rod tests rather than asserted on hand-built verdicts. gate_runner.py stays at exactly 500 lines, the file ceiling. Co-Authored-By: Claude Opus 5 (1M context) --- src/cf_quality/exemptions.py | 20 +++-- src/cf_quality/file_budget.py | 27 +++--- src/cf_quality/gate_runner.py | 12 +-- src/cf_quality/import_contract.py | 27 ++++-- src/cf_quality/mirror_check.py | 10 ++- src/cf_quality/no_bon_ref.py | 22 +++-- src/cf_quality/recursion_check.py | 14 ++- src/cf_quality/reporting.py | 22 ++++- src/cf_quality/sticky_check.py | 18 +++- tests/test_file_budget.py | 43 ++++++--- tests/test_gate_denominator.py | 144 ++++++++++++++++++++++++++++++ tests/test_gate_runner.py | 2 +- tests/test_import_contract.py | 10 ++- tests/test_mirror_check.py | 30 +++---- tests/test_no_bon_ref.py | 26 +++--- tests/test_recursion_check.py | 2 +- tests/test_reporting.py | 24 +++++ tests/test_sticky_check.py | 48 +++++----- 18 files changed, 379 insertions(+), 122 deletions(-) create mode 100644 tests/test_gate_denominator.py diff --git a/src/cf_quality/exemptions.py b/src/cf_quality/exemptions.py index 950e9bb..44870b4 100644 --- a/src/cf_quality/exemptions.py +++ b/src/cf_quality/exemptions.py @@ -285,9 +285,11 @@ def _ratchet_report(entry_count: int, frozen_count: int) -> tuple[list[GateViola return violations, lines -def check(root: Path) -> tuple[list[GateViolation], list[str]]: - """Run checks (a)-(e) against a repo root; returns (violations, report lines).""" +def check(root: Path) -> tuple[list[GateViolation], list[str], dict[str, int]]: + """Run checks (a)-(e) — (violations, report lines, the counts measured).""" suppressions, violations = _scan_src(root) + surface = tuple(discover_scan_paths(root)) + counts = {"suppressions": len(suppressions), "scan_paths": len(surface), "entries": 0} config = _load_config(root) if config is None: if suppressions: @@ -296,16 +298,17 @@ def check(root: Path) -> tuple[list[GateViolation], list[str]]: message="gated suppressions found in src/ but exemptions.json is missing", context={"suppressions": len(suppressions)}, ) - return violations, ["no exemptions.json and no gated suppressions — nothing to register"] + quiet = "no exemptions.json and no gated suppressions — nothing to register" + return violations, [quiet], counts entries, frozen_count = config - surface = tuple(discover_scan_paths(root)) + counts["entries"] = len(entries) anchors = exemption_anchors.audit(root, entries, suppressions, surface) match_violations, registered_lines = _match_suppressions(suppressions, entries, anchors.rotted) violations.extend(anchors.violations) violations.extend(match_violations) ratchet_violations, lines = _ratchet_report(len(entries), frozen_count) violations.extend(ratchet_violations) - return violations, [*lines, *anchors.notices, *registered_lines] + return violations, [*lines, *anchors.notices, *registered_lines], counts def _unregistered_violation( @@ -446,7 +449,7 @@ def main(argv: list[str] | None = None) -> int: args = parser.parse_args(argv) root = Path(args.root).resolve() try: - violations, lines = check(root) + violations, lines, counts = check(root) fold_in_lines, fold_in_exit = _run_fold_ins(root) except GateError as error: return print_verdict("cf-exemptions", [], error) @@ -456,6 +459,11 @@ def main(argv: list[str] | None = None) -> int: "cf-exemptions", violations, notices=notices, + measured=( + f"— measured {counts['suppressions']} suppression(s) over " + f"{counts['scan_paths']} scan path(s) against {counts['entries']} entry(ies)" + ), + evidence=counts, clean_summary="cf-exemptions: OK", fail_summary=f"cf-exemptions: FAIL ({len(violations)} violation(s))", ) diff --git a/src/cf_quality/file_budget.py b/src/cf_quality/file_budget.py index 6878f6b..f0e926c 100644 --- a/src/cf_quality/file_budget.py +++ b/src/cf_quality/file_budget.py @@ -52,6 +52,7 @@ from typing import Any from cf_quality.errors import GateError, GateViolation +from cf_quality.reporting import print_verdict NEW_FILE_BUDGET = 500 SKIP_DIR_NAMES = {"__pycache__", "node_modules", "build", "dist", "venv"} @@ -229,8 +230,8 @@ def _check_packages(budget: Budget, measured: dict[str, int]) -> list[GateViolat return violations -def check_tree(root: Path, budget: Budget) -> tuple[list[GateViolation], list[str]]: - """Measure the tree against the baseline; return (violations, notices).""" +def check_tree(root: Path, budget: Budget) -> tuple[list[GateViolation], list[str], int]: + """Measure the tree; return (violations, notices, the count of files measured).""" measured = { path.relative_to(root).as_posix(): measure_file(path) for path in iter_python_files(root) } @@ -246,7 +247,7 @@ def check_tree(root: Path, budget: Budget) -> tuple[list[GateViolation], list[st if entry.frozen_lines is not None and rel not in measured: notices.append(f"shrink: {rel} {entry.frozen_lines} -> deleted (drop from baseline)") violations.extend(_check_packages(budget, measured)) - return violations, notices + return violations, notices, len(measured) def init_tree(root: Path) -> dict[str, Any]: @@ -264,15 +265,17 @@ def init_tree(root: Path) -> dict[str, Any]: def _run_check(root: Path, config: Path) -> int: - violations, notices = check_tree(root, load_budget(config)) - for notice in notices: - print(notice) - if violations: - report = {"gate": "cf-file-budget", "violations": [v.to_dict() for v in violations]} - print(json.dumps(report, indent=2)) - return 1 - print("cf-file-budget: clean") - return 0 + budget = load_budget(config) + violations, notices, files = check_tree(root, budget) + return print_verdict( + "cf-file-budget", + violations, + notices=notices, + measured=f"— measured {files} file(s) against {len(budget.files)} frozen entry(ies)", + evidence={"files_measured": files, "frozen_files": len(budget.files)}, + clean_summary="cf-file-budget: clean", + fail_summary=f"cf-file-budget: FAIL ({len(violations)} violation(s))", + ) def _run_init(root: Path, config: Path) -> int: diff --git a/src/cf_quality/gate_runner.py b/src/cf_quality/gate_runner.py index 30530f0..17ebaf2 100644 --- a/src/cf_quality/gate_runner.py +++ b/src/cf_quality/gate_runner.py @@ -189,13 +189,14 @@ def _structured_verdict(gate: str, data: Mapping[str, Any]) -> GateVerdict: """Build a verdict from a cf-* gate's JSON — full shape or the subset. Honours an ``error`` field so a gate that exits 2 under its own GateError - contract resolves to a GateError verdict; the subset (file_budget) has none. + contract resolves to a GateError verdict; a partial emitter's has none. """ - violations = [_violation_from_dict(item) for item in data.get("violations", [])] return GateVerdict( gate=data.get("gate", gate), - violations=violations, + violations=[_violation_from_dict(item) for item in data.get("violations", [])], error=_error_from_dict(data.get("error")), + notices=list(data.get("notices", [])), + evidence=dict(data.get("evidence", {})), ) @@ -203,7 +204,7 @@ def _verdict_from_proc(gate: str, proc: subprocess.CompletedProcess[str]) -> Gat """Parse a cf-* gate's output into a verdict, robust to its three wire shapes. 1. a full GateVerdict JSON (or any object carrying ``error``) → use it; - 2. the ``{gate, violations}`` subset (file_budget) → verdict, error=None; + 2. the ``{gate, violations}`` subset (a partial emitter) → verdict, error=None; 3. non-JSON human text (e.g. sticky-check) → exit-code semantics: rc 0 is clean, any non-zero is one violation carrying the output (NOT a GateError — unparseable output is not a gate-could-not-run condition). @@ -225,8 +226,7 @@ def _run_cf_gate( ) -> GateVerdict: """Run a cf-* console gate in JSON mode and parse its verdict.""" sub_env = {**env, "CF_QUALITY_JSON": "1"} - proc = _exec([str(_tool(gate)), *args], cwd, sub_env) - return _verdict_from_proc(gate, proc) + return _verdict_from_proc(gate, _exec([str(_tool(gate)), *args], cwd, sub_env)) # --- the stages (declared once, mirroring quality-gate.yml) ----------------- diff --git a/src/cf_quality/import_contract.py b/src/cf_quality/import_contract.py index 0991506..dbaeca1 100644 --- a/src/cf_quality/import_contract.py +++ b/src/cf_quality/import_contract.py @@ -259,17 +259,19 @@ def _is_dynamic_import(call: ast.Call) -> bool: return isinstance(func, ast.Attribute) and func.attr == "import_module" -def scan_dynamic_imports(root: Path) -> list[GateViolation]: - """PASS 3 — module-level dynamic imports inside contract-protected layers.""" +def scan_dynamic_imports(root: Path) -> tuple[list[GateViolation], int]: + """PASS 3 — module-level dynamic imports; findings and the modules scanned.""" root = root.resolve() table = _load_contract_table(root) if table is None: - return [] + return [], 0 source_root = resolve_source_root(root) protected = _named_top_level(_contracts(table)) & set(_top_level_packages(source_root)) violations: list[GateViolation] = [] + scanned = 0 for package in sorted(protected): for path in sorted((source_root / package).rglob("*.py")): + scanned += 1 rel = path.relative_to(root).as_posix() violations.extend( GateViolation( @@ -284,7 +286,7 @@ def scan_dynamic_imports(root: Path) -> list[GateViolation]: for call in _module_level_calls(_read_tree(path)) if _is_dynamic_import(call) ) - return violations + return violations, scanned def _run_lint_imports(root: Path) -> tuple[int, str]: @@ -355,10 +357,14 @@ def _edge_violations(code: str, summary: str, output: str) -> list[GateViolation def _missing_contract_verdict(root: Path) -> int: packages = _top_level_packages(resolve_source_root(root)) + measured = f"— linted 0 contract clause(s) over {len(packages)} top-level package(s)" + evidence = {"contract_clauses": 0, "top_level_packages": len(packages)} if not packages: return print_verdict( "cf-import-contract", [], + measured=measured, + evidence=evidence, clean_summary="cf-import-contract: OK (no top-level packages, no contract required)", ) violation = _contract_violation( @@ -367,7 +373,7 @@ def _missing_contract_verdict(root: Path) -> int: "contract ([tool.importlinter] in pyproject.toml)", {"packages": packages}, ) - return print_verdict("cf-import-contract", [violation]) + return print_verdict("cf-import-contract", [violation], measured=measured, evidence=evidence) _CLEAN_SUMMARY = ( @@ -411,10 +417,17 @@ def _run_gate(root: Path) -> int: return _missing_contract_verdict(root) violations = lint_contract(root) violations += _pass_two(root, tolerate_config_error=bool(violations)) - violations += scan_dynamic_imports(root) + dynamic, modules = scan_dynamic_imports(root) + violations += dynamic + clauses = len(_contracts(table)) notices: list[str] = [] if violations else _honored_notices(table) return print_verdict( - "cf-import-contract", violations, notices=notices, clean_summary=_CLEAN_SUMMARY + "cf-import-contract", + violations, + notices=notices, + measured=f"— linted {clauses} contract clause(s) over {modules} module(s) scanned", + evidence={"contract_clauses": clauses, "modules_scanned": modules}, + clean_summary=_CLEAN_SUMMARY, ) diff --git a/src/cf_quality/mirror_check.py b/src/cf_quality/mirror_check.py index 4c4ccd6..b77156f 100644 --- a/src/cf_quality/mirror_check.py +++ b/src/cf_quality/mirror_check.py @@ -279,8 +279,8 @@ def check_mirrors( *, max_pin_age_days: int = DEFAULT_MAX_PIN_AGE_DAYS, today: dt.date | None = None, -) -> list[GateViolation]: - """Check every declared mirror in ``repo/MIRRORS.md``; return all findings.""" +) -> tuple[list[GateViolation], int]: + """Check every declared mirror in ``repo/MIRRORS.md`` — findings and row count.""" mirrors_path = repo / "MIRRORS.md" if not mirrors_path.is_file(): raise GateError( @@ -293,7 +293,7 @@ def check_mirrors( violations: list[GateViolation] = [] for row in rows: violations.extend(_check_row(repo, row, max_pin_age_days, effective_today)) - return violations + return violations, len(rows) def render_template() -> str: @@ -345,12 +345,14 @@ def main(argv: list[str] | None = None) -> int: path = init_mirrors(repo) print(f"wrote {path}") return 0 - violations = check_mirrors(repo, max_pin_age_days=args.max_pin_age_days) + violations, rows = check_mirrors(repo, max_pin_age_days=args.max_pin_age_days) except GateError as error: return print_verdict("cf-mirror-check", [], error) return print_verdict( "cf-mirror-check", violations, + measured=f"— checked {rows} declared mirror row(s) in MIRRORS.md", + evidence={"mirror_rows": rows}, clean_summary="cf-mirror-check: OK", fail_summary=f"cf-mirror-check: FAIL ({len(violations)} violation(s))", ) diff --git a/src/cf_quality/no_bon_ref.py b/src/cf_quality/no_bon_ref.py index 6750427..4b51eb6 100644 --- a/src/cf_quality/no_bon_ref.py +++ b/src/cf_quality/no_bon_ref.py @@ -115,8 +115,8 @@ def scan_file(path: Path, root: Path) -> list[GateViolation]: ] -def scan_tree(root: Path) -> list[GateViolation]: - """Scan the whole code/config tree; raise GateError when the root is absent.""" +def scan_tree(root: Path) -> tuple[list[GateViolation], int]: + """Sweep the code/config tree — findings and files swept; GateError when absent.""" if not root.exists(): raise GateError( code="GATE_PATH_MISSING", @@ -124,9 +124,11 @@ def scan_tree(root: Path) -> list[GateViolation]: context={"path": str(root)}, ) violations: list[GateViolation] = [] + swept = 0 for path in iter_source_files(root): + swept += 1 violations.extend(scan_file(path, root)) - return sorted(violations, key=lambda v: (v.path, v.line or 0)) + return sorted(violations, key=lambda v: (v.path, v.line or 0)), swept # --- the reasoned, ratcheted exemption registry ----------------------------- @@ -212,12 +214,12 @@ def _partition( return failing, blessed -def check(root: Path) -> tuple[list[GateViolation], list[str]]: - """Sweep the tree, apply the reasoned registry, ratchet it — (violations, notices).""" - found = scan_tree(root) +def check(root: Path) -> tuple[list[GateViolation], list[str], int]: + """Sweep, apply the reasoned registry, ratchet — (violations, notices, files swept).""" + found, swept = scan_tree(root) config = load_exemptions(root) if config is None: - return found, [] + return found, [], swept entries, frozen = config failing, blessed = _partition(found, entries) failing.extend(_ratchet_violation(len(entries), frozen)) @@ -225,7 +227,7 @@ def check(root: Path) -> tuple[list[GateViolation], list[str]]: f"=== TICKET-REF EXEMPTIONS: {len(entries)} entries / frozen_count {frozen} ===", *blessed, ] - return failing, notices + return failing, notices, swept def main(argv: list[str] | None = None) -> int: @@ -237,13 +239,15 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("--root", default=".", help="repo root to sweep (default: cwd)") args = parser.parse_args(argv) try: - violations, notices = check(Path(args.root).resolve()) + violations, notices, swept = check(Path(args.root).resolve()) except GateError as exc: return print_verdict("cf-no-bon-ref", [], exc) return print_verdict( "cf-no-bon-ref", violations, notices=notices, + measured=f"— swept {swept} file(s) of the code/config tree for ticket refs", + evidence={"files_swept": swept}, clean_summary="cf-no-bon-ref: OK (no ticket references in the code/config tree)", fail_summary=f"cf-no-bon-ref: FAIL ({len(violations)} ticket reference(s))", ) diff --git a/src/cf_quality/recursion_check.py b/src/cf_quality/recursion_check.py index f73cde7..72b9292 100644 --- a/src/cf_quality/recursion_check.py +++ b/src/cf_quality/recursion_check.py @@ -209,8 +209,8 @@ def scan_file(path: Path, root: Path) -> list[GateViolation]: return violations -def scan_tree(root: Path) -> list[GateViolation]: - """Scan every ``*.py`` under ``root``; raise GateError if the root is absent.""" +def scan_tree(root: Path) -> tuple[list[GateViolation], int]: + """Scan every ``*.py`` under ``root`` — findings and files walked; GateError if absent.""" if not root.exists(): raise GateError( code="GATE_PATH_MISSING", @@ -218,9 +218,11 @@ def scan_tree(root: Path) -> list[GateViolation]: context={"path": str(root)}, ) violations: list[GateViolation] = [] + walked = 0 for path in sorted(root.rglob("*.py")): + walked += 1 violations.extend(scan_file(path, root=root)) - return violations + return violations, walked def main(argv: list[str] | None = None) -> int: @@ -241,12 +243,16 @@ def main(argv: list[str] | None = None) -> int: args = parser.parse_args(argv) try: paths = args.paths or [str(resolve_source_root(Path()))] - violations = [v for path in paths for v in scan_tree(Path(path))] + scans = [scan_tree(Path(path)) for path in paths] except GateError as exc: return print_verdict("cf-recursion-check", [], exc) + violations = [v for found, _ in scans for v in found] + walked = sum(count for _, count in scans) return print_verdict( "cf-recursion-check", violations, + measured=f"— walked {walked} Python file(s) for undeclared self-recursion", + evidence={"files_walked": walked, "trees": len(scans)}, clean_summary="cf-recursion-check: OK (all self-recursion carries a declared bound)", fail_summary=f"cf-recursion-check: FAIL ({len(violations)} undeclared recursion(s))", ) diff --git a/src/cf_quality/reporting.py b/src/cf_quality/reporting.py index 014c7b1..9e0f3ae 100644 --- a/src/cf_quality/reporting.py +++ b/src/cf_quality/reporting.py @@ -23,6 +23,7 @@ import os import sys from collections.abc import Mapping, Sequence +from typing import Any from cf_quality.errors import GateError, GateVerdict, GateViolation @@ -51,6 +52,8 @@ def print_verdict( *, json_output: bool | None = None, notices: Sequence[str] = (), + measured: str | None = None, + evidence: Mapping[str, Any] | None = None, clean_summary: str | None = None, fail_summary: str | None = None, ) -> int: @@ -61,8 +64,21 @@ def print_verdict( (clean) when supplied. Opt-in (``json_output`` true, or the ``CF_QUALITY_JSON`` env var): print only the :class:`GateVerdict` JSON wire form. Either way the return value is the derived exit code. + + ``measured`` is THE denominator: the ONE line saying what the gate + examined. It rides the verdict — to the wire, and to the aggregated board + line — and prints first in human mode; ``evidence`` is that same + measurement in machine form. ``notices`` stays the gate's multi-line HUMAN + report prose (registered exemptions, shrink reports) and never reaches the + wire, where it would bury the denominator in a blob. """ - verdict = GateVerdict(gate=gate, violations=list(violations), error=error) + verdict = GateVerdict( + gate=gate, + violations=list(violations), + error=error, + notices=[measured] if measured is not None else [], + evidence=dict(evidence or {}), + ) if json_output is None: json_output = json_output_enabled() if json_output: @@ -83,11 +99,11 @@ def _emit_human( clean_summary: str | None, fail_summary: str | None, ) -> int: - """Notices, then findings, then a summary — the local-reader default.""" + """The denominator, then notices, findings, and a summary — the default.""" if verdict.error is not None: print(json.dumps(verdict.error.to_dict(), ensure_ascii=False), file=sys.stderr) return verdict.exit_code - for notice in notices: + for notice in (*verdict.notices, *notices): print(notice) for violation in verdict.violations: print(f"{violation_location(violation)}: {violation.code}: {violation.message}") diff --git a/src/cf_quality/sticky_check.py b/src/cf_quality/sticky_check.py index 6b46972..bf15842 100644 --- a/src/cf_quality/sticky_check.py +++ b/src/cf_quality/sticky_check.py @@ -260,10 +260,20 @@ def _membrane_violations(claude_md: Path, canonical_lines: list[str]) -> list[Ga ] -def check(claude_md: Path) -> list[GateViolation]: +def check(claude_md: Path) -> tuple[list[GateViolation], int]: + """Gauge one CLAUDE.md — its findings, and the file count examined (1 or 0). + + The presence test is taken ONCE and threaded, so the reported denominator + is the very fact the gauge decided on, never a second look at the disk. + """ + present = claude_md.is_file() + return _check_present(claude_md, present), int(present) + + +def _check_present(claude_md: Path, present: bool) -> list[GateViolation]: """Gauge one CLAUDE.md against the kit's canonical sticky block.""" client = repo_config.load(claude_md.parent).client_repo - if not claude_md.is_file(): + if not present: if client: return [] # the membrane waives the mount; the CLI prints it loud return [ @@ -356,7 +366,7 @@ def _resolve_target(raw: str) -> Path: def _run_check(target: Path) -> int: - violations = check(target) + violations, examined = check(target) notices: list[str] = [] if not violations and repo_config.load(target.parent).client_repo: notices.append(CLIENT_MEMBRANE_NOTICE) # the waiver is loud, never silent @@ -364,6 +374,8 @@ def _run_check(target: Path) -> int: "cf-sticky-check", violations, notices=notices, + measured=f"— examined {examined} CLAUDE.md against the canonical sticky block", + evidence={"claude_md_examined": examined}, clean_summary="cf-sticky-check: OK", fail_summary=f"cf-sticky-check: FAIL ({len(violations)} violation(s))", ) diff --git a/tests/test_file_budget.py b/tests/test_file_budget.py index 4280450..43045a3 100644 --- a/tests/test_file_budget.py +++ b/tests/test_file_budget.py @@ -115,7 +115,10 @@ def test_bad_file_entry_type_raises_typed_gate_error(self, tmp_path: Path) -> No class TestCheckNewFiles: - def test_new_file_over_500_lines_fails(self, tmp_path: Path, capsys: Any) -> None: + def test_new_file_over_500_lines_fails( + self, tmp_path: Path, capsys: Any, monkeypatch: Any + ) -> None: + monkeypatch.setenv("CF_QUALITY_JSON", "1") write_py(tmp_path / "src" / "big.py", NEW_FILE_BUDGET + 1) assert main(["check", "--root", str(tmp_path)]) == 1 report = report_from(capsys.readouterr().out) @@ -129,7 +132,10 @@ def test_new_file_at_exactly_500_lines_passes(self, tmp_path: Path) -> None: write_py(tmp_path / "src" / "fits.py", NEW_FILE_BUDGET) assert main(["check", "--root", str(tmp_path)]) == 0 - def test_report_lists_every_offender(self, tmp_path: Path, capsys: Any) -> None: + def test_report_lists_every_offender( + self, tmp_path: Path, capsys: Any, monkeypatch: Any + ) -> None: + monkeypatch.setenv("CF_QUALITY_JSON", "1") write_py(tmp_path / "a.py", 600) write_py(tmp_path / "b.py", 700) assert main(["check", "--root", str(tmp_path)]) == 1 @@ -139,7 +145,10 @@ def test_report_lists_every_offender(self, tmp_path: Path, capsys: Any) -> None: class TestFrozenFiles: - def test_frozen_file_that_grew_fails(self, tmp_path: Path, capsys: Any) -> None: + def test_frozen_file_that_grew_fails( + self, tmp_path: Path, capsys: Any, monkeypatch: Any + ) -> None: + monkeypatch.setenv("CF_QUALITY_JSON", "1") write_py(tmp_path / "src" / "handle.py", 700) write_budget(tmp_path, {"files": {"src/handle.py": 694}, "packages": {"src": 694}}) assert main(["check", "--root", str(tmp_path)]) == 1 @@ -178,8 +187,9 @@ class TestPackageBudget: """The sibling-file-accretion answer (refuter gaming vector #1).""" def test_handle_extra_at_499_next_to_frozen_handle_fails( - self, tmp_path: Path, capsys: Any + self, tmp_path: Path, capsys: Any, monkeypatch: Any ) -> None: + monkeypatch.setenv("CF_QUALITY_JSON", "1") # Fixture package: the big-legacy-module anchor case. handle.py frozen at 700, # package frozen at 700 total. The burn agent dodges the per-file # gate by creating handle_extra.py at 499 lines — the package @@ -209,7 +219,10 @@ def test_new_file_fits_inside_room_freed_by_shrink(self, tmp_path: Path) -> None ) assert main(["check", "--root", str(tmp_path)]) == 0 - def test_package_budget_counts_subdirectories(self, tmp_path: Path, capsys: Any) -> None: + def test_package_budget_counts_subdirectories( + self, tmp_path: Path, capsys: Any, monkeypatch: Any + ) -> None: + monkeypatch.setenv("CF_QUALITY_JSON", "1") # Hiding the sibling one directory deeper does not dodge the draw. write_py(tmp_path / "src" / "engine" / "handle.py", 700) write_py(tmp_path / "src" / "engine" / "extra" / "handle_extra.py", 499) @@ -238,7 +251,10 @@ def test_declared_file_does_not_draw_against_package_budget(self, tmp_path: Path ) assert main(["check", "--root", str(tmp_path)]) == 0 - def test_declared_file_still_obeys_new_file_cap(self, tmp_path: Path, capsys: Any) -> None: + def test_declared_file_still_obeys_new_file_cap( + self, tmp_path: Path, capsys: Any, monkeypatch: Any + ) -> None: + monkeypatch.setenv("CF_QUALITY_JSON", "1") write_py(tmp_path / "src" / "engine" / "webhook.py", 501) write_budget( tmp_path, @@ -316,9 +332,12 @@ def test_statement_joining_cannot_fake_a_shrink(self, tmp_path: Path, capsys: An assert main(["check", "--root", str(tmp_path)]) == 1 out = capsys.readouterr().out assert "ratchet the baseline down" not in out # no shrink notice was printed - assert "FILE_BUDGET_GREW" in violation_codes(report_from(out)) + assert "FILE_BUDGET_GREW" in out - def test_joined_shrink_cannot_free_package_headroom(self, tmp_path: Path, capsys: Any) -> None: + def test_joined_shrink_cannot_free_package_headroom( + self, tmp_path: Path, capsys: Any, monkeypatch: Any + ) -> None: + monkeypatch.setenv("CF_QUALITY_JSON", "1") # The full refuter repro: joined handle.py + a brand-new 380-line # sibling = 1280 real statements in a package frozen at 700. write_joined_py(tmp_path / "src" / "engine" / "handle.py", 900, 3) @@ -332,8 +351,9 @@ def test_joined_shrink_cannot_free_package_headroom(self, tmp_path: Path, capsys assert "PACKAGE_BUDGET_EXCEEDED" in codes def test_new_file_cap_counts_statements_not_just_lines( - self, tmp_path: Path, capsys: Any + self, tmp_path: Path, capsys: Any, monkeypatch: Any ) -> None: + monkeypatch.setenv("CF_QUALITY_JSON", "1") # 600 statements squeezed onto 200 physical lines is a >500 file in truth. write_joined_py(tmp_path / "src" / "mod.py", 600, 3) assert main(["check", "--root", str(tmp_path)]) == 1 @@ -377,7 +397,7 @@ def test_greenfield_sub_500_siblings_pass_and_the_boundary_is_disclosed( write_py(tmp_path / "src" / "engine" / f"engine_{name}.py", 499) data = init_tree(tmp_path) assert data == {"files": {}, "packages": {}} # nothing seeded the draw - violations, _ = check_tree(tmp_path, Budget(files={}, packages={})) + violations, _, _ = check_tree(tmp_path, Budget(files={}, packages={})) assert violations == [] # 2495 lines in one package, green — the boundary import cf_quality.file_budget as fb @@ -388,10 +408,11 @@ def test_greenfield_sub_500_siblings_pass_and_the_boundary_is_disclosed( class TestCheckTreeApi: def test_check_tree_returns_typed_violations(self, tmp_path: Path) -> None: write_py(tmp_path / "big.py", 600) - violations, notices = check_tree(tmp_path, Budget(files={}, packages={})) + violations, notices, files = check_tree(tmp_path, Budget(files={}, packages={})) assert len(violations) == 1 assert violations[0].to_dict()["code"] == "FILE_BUDGET_EXCEEDED" assert notices == [] + assert files == 1 # the denominator is the tree it actually measured class TestMainErrors: diff --git a/tests/test_gate_denominator.py b/tests/test_gate_denominator.py new file mode 100644 index 0000000..e01bd62 --- /dev/null +++ b/tests/test_gate_denominator.py @@ -0,0 +1,144 @@ +"""Every cf-* gate states WHAT IT EXAMINED — the denominator contract. + +A board line reading ``PASS cf-no-bon-ref`` is a name, not an event: it reads +the same whether the gate swept four hundred files or none at all. So every +gate carries ONE denominator line on its verdict (``notices``) and the same +measurement in machine form (``evidence``), and both ride the wire the runner +parses back — which is what makes the aggregated board line say what was +measured instead of merely that something passed. + +The control rod is the ZERO case: a gate driven to examine nothing must render +a visible ``0``, never an omitted line. A suppressed zero is exactly the vacuous +PASS this contract exists to expose, so it is asserted per gate, in the rendered +output, on a tree built to be empty. +""" + +from __future__ import annotations + +import json +import os +from pathlib import Path +from typing import Any + +from cf_quality import gate_runner +from cf_quality.errors import GateVerdict, GateViolation +from cf_quality.exemptions import main as exemptions_main +from cf_quality.file_budget import main as file_budget_main +from cf_quality.import_contract import main as import_contract_main +from cf_quality.mirror_check import main as mirror_main +from cf_quality.mirror_check import render_template +from cf_quality.no_bon_ref import main as no_bon_ref_main +from cf_quality.recursion_check import main as recursion_main +from cf_quality.sticky_check import main as sticky_main + +# --- the wire carries the measurement (real JSON, no mock) ------------------- + + +def test_wire_carries_populated_notices_and_evidence( + tmp_path: Path, capsys: Any, monkeypatch: Any +) -> None: + (tmp_path / "a.py").write_text("x = 1\n", encoding="utf-8") + (tmp_path / "b.css").write_text("body { color: red; }\n", encoding="utf-8") + monkeypatch.setenv("CF_QUALITY_JSON", "1") + + assert no_bon_ref_main(["--root", str(tmp_path)]) == 0 + verdict = json.loads(capsys.readouterr().out) + + assert verdict["passed"] is True + assert verdict["notices"] == ["— swept 2 file(s) of the code/config tree for ticket refs"] + assert verdict["evidence"] == {"files_swept": 2} + + +def test_file_budget_speaks_the_wire_form_like_every_other_gate( + tmp_path: Path, capsys: Any, monkeypatch: Any +) -> None: + # It used to hand-roll its own JSON and print 'cf-file-budget: clean' even + # under CF_QUALITY_JSON=1 — the wire form is now the shared one. + (tmp_path / "a.py").write_text("x = 1\n", encoding="utf-8") + monkeypatch.setenv("CF_QUALITY_JSON", "1") + + assert file_budget_main(["check", "--root", str(tmp_path)]) == 0 + verdict = json.loads(capsys.readouterr().out) + + assert verdict["gate"] == "cf-file-budget" + assert verdict["notices"] == ["— measured 1 file(s) against 0 frozen entry(ies)"] + assert verdict["evidence"] == {"files_measured": 1, "frozen_files": 0} + + +def test_structured_verdict_round_trips_notices_and_evidence() -> None: + source = GateVerdict( + gate="cf-x", + violations=[GateViolation(code="X_BROKE", message="m", path="p", line=3)], + notices=["— measured 7 thing(s) against a 2-thing floor"], + evidence={"things": 7, "floor": 2}, + ) + + parsed = gate_runner._structured_verdict("cf-x", source.to_dict()) + + assert parsed.notices == source.notices + assert parsed.evidence == source.evidence + assert [v.code for v in parsed.violations] == ["X_BROKE"] + + +def test_aggregated_board_line_carries_the_gate_denominator(tmp_path: Path, capsys: Any) -> None: + # End to end: the real console script runs, its JSON is parsed back, and the + # runner's board line shows the measurement instead of a bare PASS. + (tmp_path / "a.py").write_text("x = 1\n", encoding="utf-8") + verdict = gate_runner._run_cf_gate( + "cf-no-bon-ref", ["--root", str(tmp_path)], cwd=tmp_path, env=os.environ + ) + assert verdict.passed + + gate_runner._emit_human(gate_runner._Aggregate(verdicts=[verdict], exit_code=0)) + board = capsys.readouterr().out.splitlines()[0] + + assert board == ( + "PASS cf-no-bon-ref — swept 1 file(s) of the code/config tree for ticket refs" + ) + + +# --- the control rod: a gate that examined nothing must SHOW the zero -------- + + +def test_zero_denominator_is_visible_for_no_bon_ref(tmp_path: Path, capsys: Any) -> None: + assert no_bon_ref_main(["--root", str(tmp_path)]) == 0 + assert "— swept 0 file(s)" in capsys.readouterr().out + + +def test_zero_denominator_is_visible_for_recursion_check(tmp_path: Path, capsys: Any) -> None: + assert recursion_main([str(tmp_path)]) == 0 + assert "— walked 0 Python file(s)" in capsys.readouterr().out + + +def test_zero_denominator_is_visible_for_file_budget(tmp_path: Path, capsys: Any) -> None: + assert file_budget_main(["check", "--root", str(tmp_path)]) == 0 + assert "— measured 0 file(s)" in capsys.readouterr().out + + +def test_zero_denominator_is_visible_for_sticky_check(tmp_path: Path, capsys: Any) -> None: + # The client-repo waiver: no CLAUDE.md to examine, and the gate is green — + # the one shape where a PASS legitimately measured nothing, and says so. + (tmp_path / ".cf-quality.toml").write_text( + "[tool.cf-quality]\nclient_repo = true\n", encoding="utf-8" + ) + + assert sticky_main(["check", str(tmp_path)]) == 0 + assert "— examined 0 CLAUDE.md" in capsys.readouterr().out + + +def test_zero_denominator_is_visible_for_exemptions(tmp_path: Path, capsys: Any) -> None: + assert exemptions_main(["--root", str(tmp_path)]) == 0 + assert "— measured 0 suppression(s) over 0 scan path(s)" in capsys.readouterr().out + + +def test_zero_denominator_is_visible_for_import_contract(tmp_path: Path, capsys: Any) -> None: + assert import_contract_main(["--root", str(tmp_path)]) == 0 + assert "— linted 0 contract clause(s)" in capsys.readouterr().out + + +def test_zero_denominator_is_visible_for_mirror_check(tmp_path: Path, capsys: Any) -> None: + # A MIRRORS.md carrying the header and no rows declares no cross-repo copies. + (tmp_path / "MIRRORS.md").write_text(render_template(), encoding="utf-8") + + assert mirror_main(["check", "--repo", str(tmp_path)]) == 0 + assert "— checked 0 declared mirror row(s)" in capsys.readouterr().out diff --git a/tests/test_gate_runner.py b/tests/test_gate_runner.py index 506dfc6..8d49b1f 100644 --- a/tests/test_gate_runner.py +++ b/tests/test_gate_runner.py @@ -168,7 +168,7 @@ def test_parser_full_gateverdict_json_is_used_verbatim() -> None: def test_parser_violations_subset_json_builds_verdict() -> None: - # file_budget emits {gate, violations} — no passed/exit_code/error fields. + # A partial emitter sends {gate, violations} — no passed/exit_code/error. raw = json.dumps( { "gate": "cf-file-budget", diff --git a/tests/test_import_contract.py b/tests/test_import_contract.py index 7b6fe5e..a119dcf 100644 --- a/tests/test_import_contract.py +++ b/tests/test_import_contract.py @@ -230,7 +230,7 @@ def test_module_level_import_module_in_protected_layer_fails(tmp_path: Path) -> tmp_path, core='import importlib\n_impl = importlib.import_module("tenants.acme")\n', ) - violations = scan_dynamic_imports(tmp_path) + violations, _ = scan_dynamic_imports(tmp_path) assert [v.code for v in violations] == ["CONTRACT_DYNAMIC_IMPORT"] assert violations[0].path == "core/__init__.py" assert violations[0].line == 2 @@ -239,7 +239,7 @@ def test_module_level_import_module_in_protected_layer_fails(tmp_path: Path) -> def test_module_level_dunder_import_in_protected_layer_fails(tmp_path: Path) -> None: _mount(tmp_path, core='_impl = __import__("tenants.acme")\n') - violations = scan_dynamic_imports(tmp_path) + violations, _ = scan_dynamic_imports(tmp_path) assert [v.code for v in violations] == ["CONTRACT_DYNAMIC_IMPORT"] assert violations[0].line == 1 @@ -255,7 +255,11 @@ def test_function_body_dynamic_import_is_not_module_level(tmp_path: Path) -> Non ' return importlib.import_module("tenants.acme")\n' ), ) - assert scan_dynamic_imports(tmp_path) == [] + findings, scanned = scan_dynamic_imports(tmp_path) + assert findings == [] + # clean because the modules WERE opened, not because the walk found nothing: + # core/__init__.py + tenants/__init__.py + tenants/acme.py. + assert scanned == 3 # --- control rods through the CLI (the parsed-output surface) ----------------- diff --git a/tests/test_mirror_check.py b/tests/test_mirror_check.py index 9afa4d9..bb0cc6b 100644 --- a/tests/test_mirror_check.py +++ b/tests/test_mirror_check.py @@ -86,13 +86,13 @@ class TestCheck: def test_clean_mirror_reports_no_violations(self, tmp_path: Path) -> None: digest = make_mirror(tmp_path, "data/intro.md", b"the law travels sticky\n") write_mirrors(tmp_path, row("data/intro.md", digest)) - assert check_mirrors(tmp_path, today=TODAY) == [] + assert check_mirrors(tmp_path, today=TODAY)[0] == [] def test_diverged_mirror_fails_with_both_hashes(self, tmp_path: Path) -> None: make_mirror(tmp_path, "data/intro.md", b"drifted content\n") declared = sha256_of(b"the original content\n") write_mirrors(tmp_path, row("data/intro.md", declared)) - violations = check_mirrors(tmp_path, today=TODAY) + violations, _ = check_mirrors(tmp_path, today=TODAY) assert [v.code for v in violations] == ["MIRROR_DIVERGED"] v = violations[0] assert v.path == "data/intro.md" @@ -101,7 +101,7 @@ def test_diverged_mirror_fails_with_both_hashes(self, tmp_path: Path) -> None: def test_missing_local_file_fails(self, tmp_path: Path) -> None: write_mirrors(tmp_path, row("data/ghost.md", sha256_of(b"x"))) - violations = check_mirrors(tmp_path, today=TODAY) + violations, _ = check_mirrors(tmp_path, today=TODAY) assert [v.code for v in violations] == ["MIRROR_FILE_MISSING"] assert violations[0].path == "data/ghost.md" @@ -111,7 +111,7 @@ def test_row_missing_any_field_fails(self, tmp_path: Path) -> None: f"| sticky-intro | data/intro.md | | d.md | {'a' * 40} | {digest} | 2026-06-01 |\n" ) write_mirrors(tmp_path, incomplete) - violations = check_mirrors(tmp_path, today=TODAY) + violations, _ = check_mirrors(tmp_path, today=TODAY) assert [v.code for v in violations] == ["MIRROR_ROW_INCOMPLETE"] assert violations[0].context["missing_fields"] == ["parent repo"] @@ -119,7 +119,7 @@ def test_stale_pin_fails_at_default_90_days(self, tmp_path: Path) -> None: digest = make_mirror(tmp_path, "data/intro.md", b"content\n") old = (TODAY - dt.timedelta(days=120)).isoformat() write_mirrors(tmp_path, row("data/intro.md", digest, pinned=old)) - violations = check_mirrors(tmp_path, today=TODAY) + violations, _ = check_mirrors(tmp_path, today=TODAY) assert [v.code for v in violations] == ["MIRROR_PIN_STALE"] assert violations[0].context["age_days"] == 120 assert violations[0].context["max_pin_age_days"] == 90 @@ -128,20 +128,20 @@ def test_pin_exactly_at_max_age_is_not_stale(self, tmp_path: Path) -> None: digest = make_mirror(tmp_path, "data/intro.md", b"content\n") edge = (TODAY - dt.timedelta(days=90)).isoformat() write_mirrors(tmp_path, row("data/intro.md", digest, pinned=edge)) - assert check_mirrors(tmp_path, today=TODAY) == [] + assert check_mirrors(tmp_path, today=TODAY)[0] == [] def test_custom_max_pin_age_days_is_honored(self, tmp_path: Path) -> None: digest = make_mirror(tmp_path, "data/intro.md", b"content\n") old = (TODAY - dt.timedelta(days=10)).isoformat() write_mirrors(tmp_path, row("data/intro.md", digest, pinned=old)) - violations = check_mirrors(tmp_path, max_pin_age_days=7, today=TODAY) + violations, _ = check_mirrors(tmp_path, max_pin_age_days=7, today=TODAY) assert [v.code for v in violations] == ["MIRROR_PIN_STALE"] def test_row_with_empty_pinned_date_is_incomplete(self, tmp_path: Path) -> None: # Refuter: a blank pin made a declaration immortal (expiry opt-out). digest = make_mirror(tmp_path, "data/intro.md", b"content\n") write_mirrors(tmp_path, row("data/intro.md", digest, pinned="")) - violations = check_mirrors(tmp_path, today=TODAY) + violations, _ = check_mirrors(tmp_path, today=TODAY) assert [v.code for v in violations] == ["MIRROR_ROW_INCOMPLETE"] assert "pinned date" in violations[0].context["missing_fields"] @@ -149,7 +149,7 @@ def test_future_pinned_date_fails_as_future(self, tmp_path: Path) -> None: # Refuter: a 2099 pin gave negative age, so staleness could never fire. digest = make_mirror(tmp_path, "data/intro.md", b"content\n") write_mirrors(tmp_path, row("data/intro.md", digest, pinned="2099-01-01")) - violations = check_mirrors(tmp_path, today=TODAY) + violations, _ = check_mirrors(tmp_path, today=TODAY) assert [v.code for v in violations] == ["MIRROR_PIN_FUTURE"] assert violations[0].context["pinned_date"] == "2099-01-01" @@ -157,22 +157,22 @@ def test_max_pin_age_zero_expires_every_dated_pin_but_today(self, tmp_path: Path digest = make_mirror(tmp_path, "data/intro.md", b"content\n") yesterday = (TODAY - dt.timedelta(days=1)).isoformat() write_mirrors(tmp_path, row("data/intro.md", digest, pinned=yesterday)) - violations = check_mirrors(tmp_path, max_pin_age_days=0, today=TODAY) + violations, _ = check_mirrors(tmp_path, max_pin_age_days=0, today=TODAY) assert [v.code for v in violations] == ["MIRROR_PIN_STALE"] write_mirrors(tmp_path, row("data/intro.md", digest, pinned=TODAY.isoformat())) - assert check_mirrors(tmp_path, max_pin_age_days=0, today=TODAY) == [] + assert check_mirrors(tmp_path, max_pin_age_days=0, today=TODAY)[0] == [] def test_unparseable_pinned_date_fails_as_invalid(self, tmp_path: Path) -> None: digest = make_mirror(tmp_path, "data/intro.md", b"content\n") write_mirrors(tmp_path, row("data/intro.md", digest, pinned="last tuesday")) - violations = check_mirrors(tmp_path, today=TODAY) + violations, _ = check_mirrors(tmp_path, today=TODAY) assert [v.code for v in violations] == ["MIRROR_PIN_INVALID"] assert violations[0].context["pinned_date"] == "last tuesday" def test_hash_comparison_is_case_insensitive(self, tmp_path: Path) -> None: digest = make_mirror(tmp_path, "data/intro.md", b"content\n") write_mirrors(tmp_path, row("data/intro.md", digest.upper())) - assert check_mirrors(tmp_path, today=TODAY) == [] + assert check_mirrors(tmp_path, today=TODAY)[0] == [] def test_missing_mirrors_md_raises_typed_gate_error(self, tmp_path: Path) -> None: with pytest.raises(GateError) as exc: @@ -188,7 +188,7 @@ def test_multiple_rows_collect_all_violations(self, tmp_path: Path) -> None: + row("data/good.md", sha256_of(b"other"), artifact="diverged") ) write_mirrors(tmp_path, rows) - codes = sorted(v.code for v in check_mirrors(tmp_path, today=TODAY)) + codes = sorted(v.code for v in check_mirrors(tmp_path, today=TODAY)[0]) assert codes == ["MIRROR_DIVERGED", "MIRROR_FILE_MISSING"] @@ -199,7 +199,7 @@ def test_init_writes_template_that_parses_clean(self, tmp_path: Path) -> None: text = path.read_text(encoding="utf-8") assert text == render_template() assert parse_mirrors(text) == [] # template carries no live rows - assert check_mirrors(tmp_path, today=TODAY) == [] # green by construction + assert check_mirrors(tmp_path, today=TODAY)[0] == [] # green by construction def test_init_refuses_to_overwrite_existing(self, tmp_path: Path) -> None: (tmp_path / "MIRRORS.md").write_text("precious\n", encoding="utf-8") diff --git a/tests/test_no_bon_ref.py b/tests/test_no_bon_ref.py index 88c5839..4965770 100644 --- a/tests/test_no_bon_ref.py +++ b/tests/test_no_bon_ref.py @@ -38,7 +38,7 @@ def _write(root: Path, rel: str, body: str) -> Path: def test_ref_in_python_comment_is_a_violation(tmp_path: Path) -> None: _write(tmp_path, "src/widget.py", f"# touch hardening ({_REF})\nx = 1\n") - violations = scan_tree(tmp_path) + violations, _ = scan_tree(tmp_path) assert len(violations) == 1 v = violations[0] assert v.code == "TICKET_REF_IN_SOURCE" @@ -51,27 +51,27 @@ def test_ref_in_css_and_gitignore_and_config_all_caught(tmp_path: Path) -> None: _write(tmp_path, "src/styles/builder.css", f"/* TOUCH HARDENING ({_REF}) */\n") _write(tmp_path, ".gitignore", f"# Playwright ({_REF} e2e)\ntest-results\n") _write(tmp_path, "playwright.config.js", f"// device matrix ({_REF}).\n") - paths = {v.path for v in scan_tree(tmp_path)} + paths = {v.path for v in scan_tree(tmp_path)[0]} assert paths == {"src/styles/builder.css", ".gitignore", "playwright.config.js"} def test_ref_in_test_name_is_caught(tmp_path: Path) -> None: # the law explicitly governs TEST NAMES, so the tests/ tree is swept. _write(tmp_path, "tests/test_more.py", f'"""covers {_REF}."""\nx = 1\n') - paths = {v.path for v in scan_tree(tmp_path)} + paths = {v.path for v in scan_tree(tmp_path)[0]} assert "tests/test_more.py" in paths def test_multiple_refs_one_violation_per_line(tmp_path: Path) -> None: _write(tmp_path, "src/a.py", f"# {_REF}\n# {_REF2}\nok = 1\n") - violations = scan_tree(tmp_path) + violations, _ = scan_tree(tmp_path) assert len(violations) == 2 assert {v.line for v in violations} == {1, 2} def test_clean_tree_has_no_violations(tmp_path: Path) -> None: _write(tmp_path, "src/widget.py", "# touch hardening (phone-first)\nx = 1\n") - assert scan_tree(tmp_path) == [] + assert scan_tree(tmp_path)[0] == [] # --- jurisdiction: docs/ carry provenance, not banned ----------------------- @@ -82,26 +82,26 @@ def test_docs_dir_is_out_of_jurisdiction(tmp_path: Path) -> None: # the law governs CODE, not provenance prose. _write(tmp_path, "docs/design/plan.md", f"Epic {_REF} ships the builder.\n") _write(tmp_path, "docs/law-debt.md", f"- {_REF2} backend typed-error debt\n") - assert scan_tree(tmp_path) == [] + assert scan_tree(tmp_path)[0] == [] def test_markdown_anywhere_is_prose_not_code(tmp_path: Path) -> None: # a README or markdown note is documentation provenance, even outside docs/. _write(tmp_path, "ts/README.md", f"Built under {_REF}.\n") _write(tmp_path, "src/NOTES.md", f"see {_REF2}\n") - assert scan_tree(tmp_path) == [] + assert scan_tree(tmp_path)[0] == [] def test_vendored_and_vcs_and_caches_are_skipped(tmp_path: Path) -> None: _write(tmp_path, "node_modules/dep/index.js", f"// {_REF}\n") _write(tmp_path, "__pycache__/x.txt", f"{_REF}\n") _write(tmp_path, ".git/COMMIT_EDITMSG", f"{_REF}\n") - assert scan_tree(tmp_path) == [] + assert scan_tree(tmp_path)[0] == [] def test_binary_files_are_skipped(tmp_path: Path) -> None: (tmp_path / "asset.png").write_bytes(b"\x89PNG\x00\x00" + _REF.encode() + b"\x00") - assert scan_tree(tmp_path) == [] + assert scan_tree(tmp_path)[0] == [] # --- the reasoned, ratcheted exemption registry ----------------------------- @@ -115,7 +115,7 @@ def test_registered_exemption_blesses_a_code_path_ref(tmp_path: Path) -> None: '{"frozen_count": 1, "entries": [' '{"path": "src/generated/*", "reason": "vendored upstream codegen carries its tag"}]}', ) - violations, notices = check(tmp_path) + violations, notices, _ = check(tmp_path) assert violations == [] assert any("src/generated/schema.py" in line for line in notices) @@ -127,7 +127,7 @@ def test_exemption_not_matching_still_fails(tmp_path: Path) -> None: "no-bon-ref-exemptions.json", '{"frozen_count": 1, "entries": [{"path": "src/other/*", "reason": "elsewhere"}]}', ) - violations, _ = check(tmp_path) + violations, _, _ = check(tmp_path) assert [v.path for v in violations] == ["src/hand.py"] @@ -137,7 +137,7 @@ def test_entries_over_frozen_count_fails_ratchet(tmp_path: Path) -> None: "no-bon-ref-exemptions.json", '{"frozen_count": 0, "entries": [{"path": "src/x/*", "reason": "r"}]}', ) - violations, _ = check(tmp_path) + violations, _, _ = check(tmp_path) assert any(v.code == "EXEMPTION_COUNT_EXCEEDED" for v in violations) @@ -180,5 +180,5 @@ def test_main_config_error_returns_two(tmp_path: Path) -> None: def test_isinstance_findings_are_gate_violations(tmp_path: Path) -> None: _write(tmp_path, "src/bad.py", f"# {_REF}\n") - violations = scan_tree(tmp_path) + violations, _ = scan_tree(tmp_path) assert all(isinstance(v, GateViolation) for v in violations) diff --git a/tests/test_recursion_check.py b/tests/test_recursion_check.py index c4561c5..16db627 100644 --- a/tests/test_recursion_check.py +++ b/tests/test_recursion_check.py @@ -393,7 +393,7 @@ def test_scan_tree_walks_nested_packages(tmp_path: Path) -> None: pkg.mkdir(parents=True) (pkg / "ok.py").write_text("def flat():\n return 1\n", encoding="utf-8") (pkg / "bad.py").write_text("def loop():\n loop()\n", encoding="utf-8") - violations = scan_tree(tmp_path / "src") + violations, _ = scan_tree(tmp_path / "src") assert [v.path for v in violations] == ["pkg/bad.py"] diff --git a/tests/test_reporting.py b/tests/test_reporting.py index 7d019a4..37dc6be 100644 --- a/tests/test_reporting.py +++ b/tests/test_reporting.py @@ -72,6 +72,13 @@ def test_notices_precede_findings(self, capsys: pytest.CaptureFixture[str]) -> N out = capsys.readouterr().out assert out.index("a notice") < out.index("CODE_X") + def test_denominator_precedes_the_report_prose( + self, capsys: pytest.CaptureFixture[str] + ) -> None: + print_verdict("cf-x", [], notices=["a notice"], measured="— measured 0 file(s)") + out = capsys.readouterr().out + assert out.index("— measured 0 file(s)") < out.index("a notice") + def test_default_human_output_is_not_json(self, capsys: pytest.CaptureFixture[str]) -> None: print_verdict("cf-x", [_violation(line=7)], fail_summary="cf-x: FAIL (1)") out = capsys.readouterr().out @@ -116,6 +123,23 @@ def test_json_summary_and_notices_are_suppressed( assert "cf-x: OK" not in out assert json.loads(out)["passed"] is True + def test_wire_carries_the_denominator_and_not_the_report_prose( + self, capsys: pytest.CaptureFixture[str] + ) -> None: + # The crux of the split: the ONE measurement line rides the wire, the + # gate's multi-line human report does not (it would bury the board). + print_verdict( + "cf-x", + [], + json_output=True, + notices=["registered: src/a.py:3 — covered by entry 0", "second prose line"], + measured="— measured 4 file(s) against a 2-file floor", + evidence={"files": 4, "floor": 2}, + ) + report: dict[str, Any] = json.loads(capsys.readouterr().out) + assert report["notices"] == ["— measured 4 file(s) against a 2-file floor"] + assert report["evidence"] == {"files": 4, "floor": 2} + def test_json_gate_error_goes_to_stderr(self, capsys: pytest.CaptureFixture[str]) -> None: err = GateError(code="GATE_BOOM", message="cannot run") code = print_verdict("cf-x", [], err, json_output=True) diff --git a/tests/test_sticky_check.py b/tests/test_sticky_check.py index 873f7bd..cd7736a 100644 --- a/tests/test_sticky_check.py +++ b/tests/test_sticky_check.py @@ -77,7 +77,7 @@ def test_canonical_text_ignores_consumer_copies(tmp_path: Path) -> None: data = repo / "data" data.mkdir() (data / "sticky-intro.md").write_text("## A forged law\n", encoding="utf-8") - violations = check(repo / "CLAUDE.md") + violations, _ = check(repo / "CLAUDE.md") assert [v.code for v in violations] == ["STICKY_INTRO_ABSENT"] @@ -86,18 +86,18 @@ def test_canonical_text_ignores_consumer_copies(tmp_path: Path) -> None: def test_check_passes_when_block_present(tmp_path: Path) -> None: repo = _repo_with(tmp_path, "# Consumer repo\n\n" + canonical_text()) - assert check(repo / "CLAUDE.md") == [] + assert check(repo / "CLAUDE.md")[0] == [] def test_check_normalizes_crlf_line_endings_only(tmp_path: Path) -> None: crlf = ("# Consumer repo\n\n" + canonical_text()).replace("\n", "\r\n") repo = _repo_with(tmp_path, crlf) - assert check(repo / "CLAUDE.md") == [] + assert check(repo / "CLAUDE.md")[0] == [] def test_check_fails_when_block_absent(tmp_path: Path) -> None: repo = _repo_with(tmp_path, "# Consumer repo with no law\n") - violations = check(repo / "CLAUDE.md") + violations, _ = check(repo / "CLAUDE.md") assert len(violations) == 1 assert violations[0].code == "STICKY_INTRO_ABSENT" assert violations[0].path.endswith("CLAUDE.md") @@ -105,13 +105,13 @@ def test_check_fails_when_block_absent(tmp_path: Path) -> None: def test_check_fails_when_claude_md_missing(tmp_path: Path) -> None: repo = _repo_with(tmp_path, None) - violations = check(repo / "CLAUDE.md") + violations, _ = check(repo / "CLAUDE.md") assert [v.code for v in violations] == ["STICKY_CLAUDE_MD_MISSING"] def test_check_fails_on_tampered_block_with_diff(tmp_path: Path) -> None: repo = _repo_with(tmp_path, "# Consumer repo\n\n" + _tampered_block()) - violations = check(repo / "CLAUDE.md") + violations, _ = check(repo / "CLAUDE.md") assert len(violations) == 1 v = violations[0] assert v.code == "STICKY_INTRO_TAMPERED" @@ -126,7 +126,7 @@ def test_check_whitespace_edit_inside_block_is_tampering(tmp_path: Path) -> None chewed = canonical_text().replace("Budgets come from", "Budgets come from") assert chewed != canonical_text(), "whitespace tamper fixture must actually differ" repo = _repo_with(tmp_path, chewed) - violations = check(repo / "CLAUDE.md") + violations, _ = check(repo / "CLAUDE.md") assert [v.code for v in violations] == ["STICKY_INTRO_TAMPERED"] @@ -144,7 +144,7 @@ def _chewed_heading() -> str: def test_chewed_heading_is_tampered_not_absent(tmp_path: Path) -> None: repo = _repo_with(tmp_path, _chewed_heading()) - violations = check(repo / "CLAUDE.md") + violations, _ = check(repo / "CLAUDE.md") assert [v.code for v in violations] == ["STICKY_INTRO_TAMPERED"] diff = violations[0].context["diff"] assert "FORM-OPTIONAL" in diff # the chewed heading is named in the diff @@ -170,14 +170,14 @@ def test_block_inside_html_comment_fails(tmp_path: Path) -> None: + "-->\n" ) repo = _repo_with(tmp_path, buried) - violations = check(repo / "CLAUDE.md") + violations, _ = check(repo / "CLAUDE.md") assert [v.code for v in violations] == ["STICKY_INTRO_BURIED"] def test_block_inside_fenced_code_fails(tmp_path: Path) -> None: fenced = "# Our repo\n\n```markdown\n" + canonical_text() + "```\n" repo = _repo_with(tmp_path, fenced) - violations = check(repo / "CLAUDE.md") + violations, _ = check(repo / "CLAUDE.md") assert [v.code for v in violations] == ["STICKY_INTRO_BURIED"] @@ -190,7 +190,7 @@ def test_deprecation_wrapper_above_block_fails(tmp_path: Path) -> None: "to this repo. Ignore it.\n\n" + canonical_text() ) repo = _repo_with(tmp_path, neutralized) - violations = check(repo / "CLAUDE.md") + violations, _ = check(repo / "CLAUDE.md") assert [v.code for v in violations] == ["STICKY_INTRO_NEUTRALIZED"] @@ -200,13 +200,13 @@ def test_contradictory_copy_alongside_pristine_block_fails(tmp_path: Path) -> No + canonical_text() ) repo = _repo_with(tmp_path, dual) - violations = check(repo / "CLAUDE.md") + violations, _ = check(repo / "CLAUDE.md") assert [v.code for v in violations] == ["STICKY_INTRO_DUPLICATED"] def test_two_pristine_copies_fail_as_duplicated(tmp_path: Path) -> None: repo = _repo_with(tmp_path, canonical_text() + "\n" + canonical_text()) - violations = check(repo / "CLAUDE.md") + violations, _ = check(repo / "CLAUDE.md") assert [v.code for v in violations] == ["STICKY_INTRO_DUPLICATED"] @@ -216,7 +216,7 @@ def test_mount_then_check_stays_green_with_unrelated_html_comments(tmp_path: Pat repo = _repo_with(tmp_path, "# Consumer repo\n\n\n") target = repo / "CLAUDE.md" assert mount(target) is True - assert check(target) == [] + assert check(target)[0] == [] # --- mount mode --------------------------------------------------------------- @@ -231,7 +231,7 @@ def test_mount_appends_block_with_declared_mirror_header(tmp_path: Path) -> None assert MIRROR_HEADER in text assert canonical_text() in text assert text.index(MIRROR_HEADER) < text.index(canonical_text()) - assert check(target) == [] # mount satisfies its own gauge + assert check(target)[0] == [] # mount satisfies its own gauge def test_mount_header_is_the_ratified_one_liner() -> None: @@ -255,7 +255,7 @@ def test_mount_creates_claude_md_when_missing(tmp_path: Path) -> None: repo = _repo_with(tmp_path, None) target = repo / "CLAUDE.md" assert mount(target) is True - assert check(target) == [] + assert check(target)[0] == [] def test_mount_refuses_to_mount_over_chewed_gum(tmp_path: Path) -> None: @@ -325,7 +325,7 @@ def _declare_client(repo: Path, value: str = "true") -> None: def test_client_repo_without_claude_md_passes(tmp_path: Path) -> None: repo = _repo_with(tmp_path, None) _declare_client(repo) - assert check(repo / "CLAUDE.md") == [] + assert check(repo / "CLAUDE.md")[0] == [] def test_main_client_waiver_is_loud_never_silent( @@ -344,14 +344,14 @@ def test_main_client_waiver_is_loud_never_silent( def test_undeclared_repo_without_claude_md_still_fails(tmp_path: Path) -> None: repo = _repo_with(tmp_path, None) - violations = check(repo / "CLAUDE.md") + violations, _ = check(repo / "CLAUDE.md") assert [v.code for v in violations] == ["STICKY_CLAUDE_MD_MISSING"] def test_client_repo_false_behaves_as_undeclared(tmp_path: Path) -> None: repo = _repo_with(tmp_path, None) _declare_client(repo, value="false") - violations = check(repo / "CLAUDE.md") + violations, _ = check(repo / "CLAUDE.md") assert [v.code for v in violations] == ["STICKY_CLAUDE_MD_MISSING"] @@ -361,14 +361,14 @@ def test_client_repo_false_behaves_as_undeclared(tmp_path: Path) -> None: def test_client_repo_carrying_pristine_block_fails(tmp_path: Path) -> None: repo = _repo_with(tmp_path, "# Client repo\n\n" + canonical_text()) _declare_client(repo) - violations = check(repo / "CLAUDE.md") + violations, _ = check(repo / "CLAUDE.md") assert [v.code for v in violations] == ["STICKY_CLIENT_MEMBRANE_BREACHED"] def test_client_repo_carrying_chewed_block_fails(tmp_path: Path) -> None: repo = _repo_with(tmp_path, _tampered_block()) _declare_client(repo) - violations = check(repo / "CLAUDE.md") + violations, _ = check(repo / "CLAUDE.md") assert [v.code for v in violations] == ["STICKY_CLIENT_MEMBRANE_BREACHED"] @@ -382,7 +382,7 @@ def test_client_repo_carrying_v1_era_block_fails(tmp_path: Path) -> None: ) repo = _repo_with(tmp_path, v1_style) _declare_client(repo) - violations = check(repo / "CLAUDE.md") + violations, _ = check(repo / "CLAUDE.md") assert [v.code for v in violations] == ["STICKY_CLIENT_MEMBRANE_BREACHED"] @@ -391,14 +391,14 @@ def test_client_repo_with_benign_claude_md_passes(tmp_path: Path) -> None: # carry its own instructions. repo = _repo_with(tmp_path, "# Client repo\n\nClient-lane instructions only.\n") _declare_client(repo) - assert check(repo / "CLAUDE.md") == [] + assert check(repo / "CLAUDE.md")[0] == [] def test_client_repo_prose_mention_is_not_a_breach(tmp_path: Path) -> None: # Naming the law in prose is not carrying its chrome. repo = _repo_with(tmp_path, "# Client repo\n\nWe follow the BubbleGum Law upstream.\n") _declare_client(repo) - assert check(repo / "CLAUDE.md") == [] + assert check(repo / "CLAUDE.md")[0] == [] # Tampered/incomplete declaration fails loud and typed. From 494b2f1418947d1e3e05746d7b0bccfaf47b6bef Mon Sep 17 00:00:00 2001 From: Antawari Date: Wed, 29 Jul 2026 18:37:42 -0600 Subject: [PATCH 2/3] Count what the gate actually opened, not what the walk offered it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The first pass gave every cf-gate a denominator. Three of them counted the wrong thing, so the line was confident and false — worse than a bare label, because an operator can act on it. cf-no-bon-ref counted files the walk offered, including the binaries it skips unread. A tree of one PNG reported "swept 1 file(s)" having opened nothing. scan_file now returns whether it READ the file, and the sweep counts only reads. cf-exemptions counted surface BASES. A declared source_root, or a present src/, is always exactly one base — so three Python files and zero Python files produced byte-identical lines. That is a gate selecting by the value it guards, reproduced inside the rung meant to cure it. The surface is now resolved once into a file list, threaded into the scanner, and the count is that very walk. cf-import-contract claimed one surface covered another ("linted N clause(s) over M module(s)"), fusing PASS 3's AST walk into a sentence about PASS 1, and shipped two different evidence schemas from its two branches. Both branches now speak one schema and the line names each surface separately. cf-file-budget reported only its frozen FILE entries and stayed silent on the package ceilings it also enforces; both baselines now ride the evidence. Tests assert each zero against the SHAPE that hides the defect — one unreadable binary, a src/ holding no Python — because an empty directory drives every counting bug to zero and proves nothing. --- src/cf_quality/exemptions.py | 47 +++++++----- src/cf_quality/file_budget.py | 11 ++- src/cf_quality/gate_runner.py | 2 +- src/cf_quality/import_contract.py | 19 ++++- src/cf_quality/no_bon_ref.py | 27 ++++--- tests/test_file_budget.py | 2 +- tests/test_gate_denominator.py | 115 +++++++++++++++++++++++++----- tests/test_no_bon_ref.py | 6 +- 8 files changed, 179 insertions(+), 50 deletions(-) diff --git a/src/cf_quality/exemptions.py b/src/cf_quality/exemptions.py index 44870b4..14cc0a9 100644 --- a/src/cf_quality/exemptions.py +++ b/src/cf_quality/exemptions.py @@ -61,6 +61,7 @@ import subprocess # fold-in scripts run via sys.executable, fixed argv, no shell (S603 gated below) import sys import tokenize +from collections.abc import Sequence from pathlib import Path from typing import Any @@ -167,7 +168,17 @@ def _type_ignore_suppressions( ] -def _scan_src(root: Path) -> tuple[list[Suppression], list[GateViolation]]: +def _surface_files(surface: Sequence[Path]) -> list[Path]: + """Every ``*.py`` the scanner opens, from an ALREADY-resolved surface.""" + files: list[Path] = [] + for base in surface: + files.extend([base] if base.is_file() else sorted(base.rglob("*.py"))) + return files + + +def _scan_src( + root: Path, files: Sequence[Path] | None = None +) -> tuple[list[Suppression], list[GateViolation]]: """Scan the discovered Python surface for suppression comments. The ``(suppressions, violations)`` ARITY is a published contract, not an @@ -175,20 +186,22 @@ def _scan_src(root: Path) -> tuple[list[Suppression], list[GateViolation]]: values from this to cross-check its mirrored resolver (see :func:`_matches`). The anchor audit therefore takes its surface from :mod:`cf_quality.exemption_surface` rather than riding a third element. + + ``files`` threads in the file list a caller already resolved, so the surface + is discovered ONCE and the count a caller reports is this very walk. Omitted + — the one-argument shape the consumer pin calls — it resolves its own. """ + if files is None: + files = _surface_files(discover_scan_paths(root)) suppressions: list[Suppression] = [] violations: list[GateViolation] = [] - for base in discover_scan_paths(root): - files = [base] if base.is_file() else sorted(base.rglob("*.py")) - for file_path in files: - rel_path = file_path.relative_to(root).as_posix() - spans = symbol_spans(file_path) - for line, text in _comment_tokens(file_path): - found, broken = _classify_comment( - rel_path, line, text, enclosing_symbol(spans, line) - ) - suppressions.extend(found) - violations.extend(broken) + for file_path in files: + rel_path = file_path.relative_to(root).as_posix() + spans = symbol_spans(file_path) + for line, text in _comment_tokens(file_path): + found, broken = _classify_comment(rel_path, line, text, enclosing_symbol(spans, line)) + suppressions.extend(found) + violations.extend(broken) return suppressions, violations @@ -287,9 +300,10 @@ def _ratchet_report(entry_count: int, frozen_count: int) -> tuple[list[GateViola def check(root: Path) -> tuple[list[GateViolation], list[str], dict[str, int]]: """Run checks (a)-(e) — (violations, report lines, the counts measured).""" - suppressions, violations = _scan_src(root) surface = tuple(discover_scan_paths(root)) - counts = {"suppressions": len(suppressions), "scan_paths": len(surface), "entries": 0} + files = _surface_files(surface) + suppressions, violations = _scan_src(root, files) + counts = {"suppressions": len(suppressions), "files_scanned": len(files), "entries": 0} config = _load_config(root) if config is None: if suppressions: @@ -460,8 +474,9 @@ def main(argv: list[str] | None = None) -> int: violations, notices=notices, measured=( - f"— measured {counts['suppressions']} suppression(s) over " - f"{counts['scan_paths']} scan path(s) against {counts['entries']} entry(ies)" + f"— read {counts['files_scanned']} Python file(s), found " + f"{counts['suppressions']} suppression(s) against " + f"{counts['entries']} registered entry(ies)" ), evidence=counts, clean_summary="cf-exemptions: OK", diff --git a/src/cf_quality/file_budget.py b/src/cf_quality/file_budget.py index f0e926c..7848959 100644 --- a/src/cf_quality/file_budget.py +++ b/src/cf_quality/file_budget.py @@ -271,8 +271,15 @@ def _run_check(root: Path, config: Path) -> int: "cf-file-budget", violations, notices=notices, - measured=f"— measured {files} file(s) against {len(budget.files)} frozen entry(ies)", - evidence={"files_measured": files, "frozen_files": len(budget.files)}, + measured=( + f"— measured {files} file(s) against {len(budget.files)} frozen file entry(ies) " + f"and {len(budget.packages)} frozen package budget(s)" + ), + evidence={ + "files_measured": files, + "frozen_files": len(budget.files), + "frozen_packages": len(budget.packages), + }, clean_summary="cf-file-budget: clean", fail_summary=f"cf-file-budget: FAIL ({len(violations)} violation(s))", ) diff --git a/src/cf_quality/gate_runner.py b/src/cf_quality/gate_runner.py index 17ebaf2..4d06689 100644 --- a/src/cf_quality/gate_runner.py +++ b/src/cf_quality/gate_runner.py @@ -205,7 +205,7 @@ def _verdict_from_proc(gate: str, proc: subprocess.CompletedProcess[str]) -> Gat 1. a full GateVerdict JSON (or any object carrying ``error``) → use it; 2. the ``{gate, violations}`` subset (a partial emitter) → verdict, error=None; - 3. non-JSON human text (e.g. sticky-check) → exit-code semantics: rc 0 is + 3. non-JSON human text (a foreign emitter) → exit-code semantics: rc 0 is clean, any non-zero is one violation carrying the output (NOT a GateError — unparseable output is not a gate-could-not-run condition). """ diff --git a/src/cf_quality/import_contract.py b/src/cf_quality/import_contract.py index dbaeca1..e77ab83 100644 --- a/src/cf_quality/import_contract.py +++ b/src/cf_quality/import_contract.py @@ -355,10 +355,23 @@ def _edge_violations(code: str, summary: str, output: str) -> list[GateViolation return [_contract_violation(code, f"{summary}: {edge}", {"edge": edge}) for edge in edges] +def _measured(clauses: int, modules: int) -> str: + """The denominator — two DIFFERENT surfaces, each named for what it is. + + ``clauses`` is what PASS 1 linted in the committed contract file; ``modules`` + is what PASS 3's AST walk opened (packages both named in a contract and + present as top-level). PASS 2 delegates to lint-imports and reports neither, + so the line states each number separately rather than saying one covers the + other — a contract naming no top-level package genuinely lints clauses while + scanning zero modules, and that is not a hole, it is two surfaces. + """ + return f"— linted {clauses} contract clause(s); scanned {modules} module(s) for dynamic imports" + + def _missing_contract_verdict(root: Path) -> int: packages = _top_level_packages(resolve_source_root(root)) - measured = f"— linted 0 contract clause(s) over {len(packages)} top-level package(s)" - evidence = {"contract_clauses": 0, "top_level_packages": len(packages)} + measured = _measured(0, 0) + evidence = {"contract_clauses": 0, "modules_scanned": 0} if not packages: return print_verdict( "cf-import-contract", @@ -425,7 +438,7 @@ def _run_gate(root: Path) -> int: "cf-import-contract", violations, notices=notices, - measured=f"— linted {clauses} contract clause(s) over {modules} module(s) scanned", + measured=_measured(clauses, modules), evidence={"contract_clauses": clauses, "modules_scanned": modules}, clean_summary=_CLEAN_SUMMARY, ) diff --git a/src/cf_quality/no_bon_ref.py b/src/cf_quality/no_bon_ref.py index 4b51eb6..56347ba 100644 --- a/src/cf_quality/no_bon_ref.py +++ b/src/cf_quality/no_bon_ref.py @@ -91,14 +91,20 @@ def _scan_bytes(data: bytes) -> Iterator[tuple[int, list[str]]]: yield index, [m.decode("ascii") for m in matches] -def scan_file(path: Path, root: Path) -> list[GateViolation]: - """Scan one file; binary or unreadable files yield nothing (not a crash).""" +def scan_file(path: Path, root: Path) -> tuple[list[GateViolation], bool]: + """Scan one file — findings, and whether it was READ at all. + + A binary or unreadable file yields nothing (not a crash) and is reported as + unread: "no ticket refs found" and "never opened" are opposite worlds, and + only the second flag can keep the sweep's denominator from counting a file + it never looked inside. + """ try: data = path.read_bytes() except OSError: - return [] + return [], False if _is_binary(data): - return [] + return [], False rel = path.relative_to(root).as_posix() return [ GateViolation( @@ -112,11 +118,11 @@ def scan_file(path: Path, root: Path) -> list[GateViolation]: context={"refs": refs}, ) for line, refs in _scan_bytes(data) - ] + ], True def scan_tree(root: Path) -> tuple[list[GateViolation], int]: - """Sweep the code/config tree — findings and files swept; GateError when absent.""" + """Sweep the code/config tree — findings and files READ; GateError when absent.""" if not root.exists(): raise GateError( code="GATE_PATH_MISSING", @@ -126,8 +132,9 @@ def scan_tree(root: Path) -> tuple[list[GateViolation], int]: violations: list[GateViolation] = [] swept = 0 for path in iter_source_files(root): - swept += 1 - violations.extend(scan_file(path, root)) + found, was_read = scan_file(path, root) + swept += int(was_read) + violations.extend(found) return sorted(violations, key=lambda v: (v.path, v.line or 0)), swept @@ -246,8 +253,8 @@ def main(argv: list[str] | None = None) -> int: "cf-no-bon-ref", violations, notices=notices, - measured=f"— swept {swept} file(s) of the code/config tree for ticket refs", - evidence={"files_swept": swept}, + measured=f"— read {swept} text file(s) of the code/config tree for ticket refs", + evidence={"files_read": swept}, clean_summary="cf-no-bon-ref: OK (no ticket references in the code/config tree)", fail_summary=f"cf-no-bon-ref: FAIL ({len(violations)} ticket reference(s))", ) diff --git a/tests/test_file_budget.py b/tests/test_file_budget.py index 43045a3..b249907 100644 --- a/tests/test_file_budget.py +++ b/tests/test_file_budget.py @@ -332,7 +332,7 @@ def test_statement_joining_cannot_fake_a_shrink(self, tmp_path: Path, capsys: An assert main(["check", "--root", str(tmp_path)]) == 1 out = capsys.readouterr().out assert "ratchet the baseline down" not in out # no shrink notice was printed - assert "FILE_BUDGET_GREW" in out + assert "src/engine/handle.py: FILE_BUDGET_GREW:" in out # the composed line def test_joined_shrink_cannot_free_package_headroom( self, tmp_path: Path, capsys: Any, monkeypatch: Any diff --git a/tests/test_gate_denominator.py b/tests/test_gate_denominator.py index e01bd62..2d0b0ad 100644 --- a/tests/test_gate_denominator.py +++ b/tests/test_gate_denominator.py @@ -2,15 +2,19 @@ A board line reading ``PASS cf-no-bon-ref`` is a name, not an event: it reads the same whether the gate swept four hundred files or none at all. So every -gate carries ONE denominator line on its verdict (``notices``) and the same -measurement in machine form (``evidence``), and both ride the wire the runner -parses back — which is what makes the aggregated board line say what was -measured instead of merely that something passed. +cf-* gate carries ONE denominator line on its verdict (``notices``) and the +same measurement in machine form (``evidence``), and both ride the wire the +runner parses back — which is what makes the aggregated board line say what was +measured instead of merely that something passed. The external tools on the +board (ruff, mypy, complexipy, pytest) are deliberately out of scope: they own +their own output and the kit does not compute their denominators. The control rod is the ZERO case: a gate driven to examine nothing must render a visible ``0``, never an omitted line. A suppressed zero is exactly the vacuous PASS this contract exists to expose, so it is asserted per gate, in the rendered -output, on a tree built to be empty. +output. Each zero fixture is built to be the SHAPE that hides the defect — a +tree of one unreadable binary, a ``src/`` holding no Python — not merely an +empty directory, because an empty directory drives every counting bug to 0 too. """ from __future__ import annotations @@ -20,6 +24,9 @@ from pathlib import Path from typing import Any +from test_import_contract import _mount as mount_contract_repo +from test_import_contract import _run_main as contract_run_main + from cf_quality import gate_runner from cf_quality.errors import GateVerdict, GateViolation from cf_quality.exemptions import main as exemptions_main @@ -45,8 +52,8 @@ def test_wire_carries_populated_notices_and_evidence( verdict = json.loads(capsys.readouterr().out) assert verdict["passed"] is True - assert verdict["notices"] == ["— swept 2 file(s) of the code/config tree for ticket refs"] - assert verdict["evidence"] == {"files_swept": 2} + assert verdict["notices"] == ["— read 2 text file(s) of the code/config tree for ticket refs"] + assert verdict["evidence"] == {"files_read": 2} def test_file_budget_speaks_the_wire_form_like_every_other_gate( @@ -61,8 +68,33 @@ def test_file_budget_speaks_the_wire_form_like_every_other_gate( verdict = json.loads(capsys.readouterr().out) assert verdict["gate"] == "cf-file-budget" - assert verdict["notices"] == ["— measured 1 file(s) against 0 frozen entry(ies)"] - assert verdict["evidence"] == {"files_measured": 1, "frozen_files": 0} + assert verdict["notices"] == [ + "— measured 1 file(s) against 0 frozen file entry(ies) and 0 frozen package budget(s)" + ] + # BOTH baselines the gate enforces ride the evidence — the package budget is + # a second ceiling, and reporting only the file entries hid it. + assert verdict["evidence"] == { + "files_measured": 1, + "frozen_files": 0, + "frozen_packages": 0, + } + + +def test_file_budget_evidence_carries_both_frozen_baselines( + tmp_path: Path, capsys: Any, monkeypatch: Any +) -> None: + (tmp_path / "src").mkdir() + (tmp_path / "src" / "a.py").write_text("x = 1\n", encoding="utf-8") + (tmp_path / "file-budget.json").write_text( + json.dumps({"files": {}, "packages": {"src": 400}}), encoding="utf-8" + ) + monkeypatch.setenv("CF_QUALITY_JSON", "1") + + assert file_budget_main(["check", "--root", str(tmp_path)]) == 0 + verdict = json.loads(capsys.readouterr().out) + + assert verdict["evidence"]["frozen_packages"] == 1, "a package ceiling is a measured budget" + assert verdict["evidence"]["frozen_files"] == 0 def test_structured_verdict_round_trips_notices_and_evidence() -> None: @@ -93,16 +125,21 @@ def test_aggregated_board_line_carries_the_gate_denominator(tmp_path: Path, caps board = capsys.readouterr().out.splitlines()[0] assert board == ( - "PASS cf-no-bon-ref — swept 1 file(s) of the code/config tree for ticket refs" + "PASS cf-no-bon-ref — read 1 text file(s) of the code/config tree for ticket refs" ) # --- the control rod: a gate that examined nothing must SHOW the zero -------- -def test_zero_denominator_is_visible_for_no_bon_ref(tmp_path: Path, capsys: Any) -> None: +def test_zero_denominator_counts_files_read_not_files_offered(tmp_path: Path, capsys: Any) -> None: + # The shape that hides the defect: the walk OFFERS this file, scan_file + # bails on the NUL byte before reading a line of it. A denominator counting + # the offer would say 1 and certify a sweep that never looked inside. + (tmp_path / "asset.png").write_bytes(b"\x89PNG\x00\x00binary\x00") + assert no_bon_ref_main(["--root", str(tmp_path)]) == 0 - assert "— swept 0 file(s)" in capsys.readouterr().out + assert "— read 0 text file(s)" in capsys.readouterr().out def test_zero_denominator_is_visible_for_recursion_check(tmp_path: Path, capsys: Any) -> None: @@ -126,14 +163,60 @@ def test_zero_denominator_is_visible_for_sticky_check(tmp_path: Path, capsys: An assert "— examined 0 CLAUDE.md" in capsys.readouterr().out -def test_zero_denominator_is_visible_for_exemptions(tmp_path: Path, capsys: Any) -> None: +def test_zero_denominator_for_exemptions_counts_files_not_surface_bases( + tmp_path: Path, capsys: Any +) -> None: + # The shape that hides the defect (and the mexxa scar itself): a repo-root + # src/ that holds NO Python. The surface resolves to exactly one base either + # way, so a base count reads 1 and certifies a scanner that opened nothing. + (tmp_path / "src").mkdir() + (tmp_path / "src" / "app.js").write_text("const x = 1;\n", encoding="utf-8") + + assert exemptions_main(["--root", str(tmp_path)]) == 0 + assert "— read 0 Python file(s)" in capsys.readouterr().out + + +def test_exemptions_denominator_tracks_the_files_it_tokenized(tmp_path: Path, capsys: Any) -> None: + # The control rod's other half: the same repo with real Python must NOT + # report the same number. A base count reported both worlds identically. + (tmp_path / "src").mkdir() + for name in ("a.py", "b.py", "c.py"): + (tmp_path / "src" / name).write_text("x = 1\n", encoding="utf-8") + assert exemptions_main(["--root", str(tmp_path)]) == 0 - assert "— measured 0 suppression(s) over 0 scan path(s)" in capsys.readouterr().out + assert "— read 3 Python file(s)" in capsys.readouterr().out -def test_zero_denominator_is_visible_for_import_contract(tmp_path: Path, capsys: Any) -> None: +def test_zero_denominator_is_visible_for_import_contract_clauses( + tmp_path: Path, capsys: Any +) -> None: assert import_contract_main(["--root", str(tmp_path)]) == 0 - assert "— linted 0 contract clause(s)" in capsys.readouterr().out + out = capsys.readouterr().out + assert "— linted 0 contract clause(s); scanned 0 module(s)" in out + + +def test_import_contract_evidence_schema_is_identical_on_both_branches( + tmp_path: Path, capsys: Any, monkeypatch: Any +) -> None: + # One gate must speak ONE evidence schema. The no-contract branch and the + # linted branch shipped different keys, so a machine reading the field it + # knew got silence from the other half of the same gate. The mounted fixture + # is REUSED from the gate's own suite rather than pasted. + monkeypatch.setenv("CF_QUALITY_JSON", "1") + bare = tmp_path / "bare" + bare.mkdir() + assert contract_run_main(bare, monkeypatch) == 0 + missing = json.loads(capsys.readouterr().out)["evidence"] + + mounted = tmp_path / "mounted" + mount_contract_repo(mounted) + assert contract_run_main(mounted, monkeypatch) == 0 + linted = json.loads(capsys.readouterr().out)["evidence"] + + assert missing.keys() == linted.keys() == {"contract_clauses", "modules_scanned"} + assert missing == {"contract_clauses": 0, "modules_scanned": 0} + assert linted["contract_clauses"] == 1 + assert linted["modules_scanned"] == 3, "core/__init__ + tenants/__init__ + tenants/acme" def test_zero_denominator_is_visible_for_mirror_check(tmp_path: Path, capsys: Any) -> None: diff --git a/tests/test_no_bon_ref.py b/tests/test_no_bon_ref.py index 4965770..ef2a834 100644 --- a/tests/test_no_bon_ref.py +++ b/tests/test_no_bon_ref.py @@ -101,7 +101,11 @@ def test_vendored_and_vcs_and_caches_are_skipped(tmp_path: Path) -> None: def test_binary_files_are_skipped(tmp_path: Path) -> None: (tmp_path / "asset.png").write_bytes(b"\x89PNG\x00\x00" + _REF.encode() + b"\x00") - assert scan_tree(tmp_path)[0] == [] + violations, read = scan_tree(tmp_path) + assert violations == [] + # and the denominator says so: a skipped file was never READ, so counting it + # would certify a sweep of a file the gate did not open. + assert read == 0 # --- the reasoned, ratcheted exemption registry ----------------------------- From 910122f3661b0fa47e168b07c748f30fee5753c6 Mon Sep 17 00:00:00 2001 From: Antawari Date: Wed, 29 Jul 2026 18:58:35 -0600 Subject: [PATCH 3/3] Make the denominators themselves falsifiable, and stop one that lied MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A refuter lens claimed three denominators were vacuous. I verified it by mutation rather than by argument: hardcoding cf-recursion-check, cf-sticky-check and cf-mirror-check to report 0 left the whole suite GREEN, and the board then printed "examined 0 CLAUDE.md" on a repo that has one and "walked 0 Python file(s)" over a 19-file tree. Confident and false — the exact defect this branch exists to cure, reproduced inside it. Each of those three had a zero assertion and no non-zero counterpart, so the zero was proving nothing. Each now has the other half of the rod: a known count over real content. Re-running the same four mutants after the change kills all four. The previous commit message overstated its fixtures: it claimed every zero used "the SHAPE that hides the defect" when three were bare empty directories, which drive every counting bug to 0. cf-recursion-check and cf-file-budget now use a present tree holding no Python. cf-file-budget's denominator was also untrue: it called the whole baseline "frozen file entry(ies)", but a declared-not-banned entry carries a purpose and no line count, and _check_file treats it exactly like an undeclared file. It claimed a shrink-only ratchet over files that have none. The two kinds are now counted and named separately. Finally, the aggregated board itself had no guard: nothing failed if a cf-* gate reached it carrying no measurement, because every other denominator test drives one gate in-process. The full-battery test now asserts every cf-* verdict crosses the real subprocess wire with exactly one notice and a non-empty evidence dict, which also makes the runner's silent unparseable-output fallback fatal for a kit gate. --- src/cf_quality/file_budget.py | 11 ++++-- tests/test_gate_denominator.py | 54 +++++++++++++++++++++++++++++- tests/test_integration_consumer.py | 30 +++++++++++++++++ tests/test_mirror_check.py | 14 ++++++++ 4 files changed, 106 insertions(+), 3 deletions(-) diff --git a/src/cf_quality/file_budget.py b/src/cf_quality/file_budget.py index 7848959..4d496df 100644 --- a/src/cf_quality/file_budget.py +++ b/src/cf_quality/file_budget.py @@ -267,17 +267,24 @@ def init_tree(root: Path) -> dict[str, Any]: def _run_check(root: Path, config: Path) -> int: budget = load_budget(config) violations, notices, files = check_tree(root, budget) + # Only an entry carrying a line count is FROZEN (shrink-only). A + # declared-not-banned entry is a purpose with no ceiling — _check_file + # treats it exactly like an undeclared file — so counting the whole + # baseline as "frozen" would claim a ratchet that is not being enforced. + frozen = sum(1 for entry in budget.files.values() if entry.frozen_lines is not None) return print_verdict( "cf-file-budget", violations, notices=notices, measured=( - f"— measured {files} file(s) against {len(budget.files)} frozen file entry(ies) " + f"— measured {files} file(s) against {frozen} frozen file entry(ies), " + f"{len(budget.files) - frozen} declared-not-banned entry(ies) " f"and {len(budget.packages)} frozen package budget(s)" ), evidence={ "files_measured": files, - "frozen_files": len(budget.files), + "frozen_files": frozen, + "declared_files": len(budget.files) - frozen, "frozen_packages": len(budget.packages), }, clean_summary="cf-file-budget: clean", diff --git a/tests/test_gate_denominator.py b/tests/test_gate_denominator.py index 2d0b0ad..751ef40 100644 --- a/tests/test_gate_denominator.py +++ b/tests/test_gate_denominator.py @@ -36,6 +36,7 @@ from cf_quality.mirror_check import render_template from cf_quality.no_bon_ref import main as no_bon_ref_main from cf_quality.recursion_check import main as recursion_main +from cf_quality.sticky_check import canonical_text from cf_quality.sticky_check import main as sticky_main # --- the wire carries the measurement (real JSON, no mock) ------------------- @@ -69,17 +70,38 @@ def test_file_budget_speaks_the_wire_form_like_every_other_gate( assert verdict["gate"] == "cf-file-budget" assert verdict["notices"] == [ - "— measured 1 file(s) against 0 frozen file entry(ies) and 0 frozen package budget(s)" + "— measured 1 file(s) against 0 frozen file entry(ies), 0 declared-not-banned " + "entry(ies) and 0 frozen package budget(s)" ] # BOTH baselines the gate enforces ride the evidence — the package budget is # a second ceiling, and reporting only the file entries hid it. assert verdict["evidence"] == { "files_measured": 1, "frozen_files": 0, + "declared_files": 0, "frozen_packages": 0, } +def test_declared_not_banned_entries_are_not_counted_as_frozen( + tmp_path: Path, capsys: Any, monkeypatch: Any +) -> None: + # A purpose-only entry has NO line ceiling — _check_file treats it exactly + # like an undeclared file. Counting the whole baseline as "frozen" claimed a + # shrink-only ratchet over a file that has none. + (tmp_path / "a.py").write_text("x = 1\n", encoding="utf-8") + (tmp_path / "file-budget.json").write_text( + json.dumps({"files": {"a.py": {"purpose": "adapter"}}, "packages": {}}), encoding="utf-8" + ) + monkeypatch.setenv("CF_QUALITY_JSON", "1") + + assert file_budget_main(["check", "--root", str(tmp_path)]) == 0 + evidence = json.loads(capsys.readouterr().out)["evidence"] + + assert evidence["frozen_files"] == 0, "a purpose without a line count freezes nothing" + assert evidence["declared_files"] == 1 + + def test_file_budget_evidence_carries_both_frozen_baselines( tmp_path: Path, capsys: Any, monkeypatch: Any ) -> None: @@ -143,11 +165,30 @@ def test_zero_denominator_counts_files_read_not_files_offered(tmp_path: Path, ca def test_zero_denominator_is_visible_for_recursion_check(tmp_path: Path, capsys: Any) -> None: + # A present tree holding NO Python, not a bare empty dir: the walk offers a + # file and the gate must still say 0. An empty directory drives every + # counting bug to 0 and so proves nothing. + (tmp_path / "notes.txt").write_text("not python\n", encoding="utf-8") + assert recursion_main([str(tmp_path)]) == 0 assert "— walked 0 Python file(s)" in capsys.readouterr().out +def test_recursion_denominator_tracks_the_files_it_walked(tmp_path: Path, capsys: Any) -> None: + # The zero alone would survive a hardcoded 0. This is the other half of the + # rod: the same gate on real Python must report the count, not the constant. + for name in ("a.py", "b.py"): + (tmp_path / name).write_text("x = 1\n", encoding="utf-8") + + assert recursion_main([str(tmp_path)]) == 0 + assert "— walked 2 Python file(s)" in capsys.readouterr().out + + def test_zero_denominator_is_visible_for_file_budget(tmp_path: Path, capsys: Any) -> None: + # Present tree, no Python — the budget measures *.py only, so a denominator + # counting every file it was offered would read 1 here. + (tmp_path / "README.md").write_text("# prose\n", encoding="utf-8") + assert file_budget_main(["check", "--root", str(tmp_path)]) == 0 assert "— measured 0 file(s)" in capsys.readouterr().out @@ -163,6 +204,17 @@ def test_zero_denominator_is_visible_for_sticky_check(tmp_path: Path, capsys: An assert "— examined 0 CLAUDE.md" in capsys.readouterr().out +def test_sticky_denominator_reads_one_when_a_claude_md_was_examined( + tmp_path: Path, capsys: Any +) -> None: + # Without this, `int(present)` could be hardcoded to 0 and every board line + # would claim the gauge examined nothing while still grading the file. + (tmp_path / "CLAUDE.md").write_text(canonical_text(), encoding="utf-8") + + assert sticky_main(["check", str(tmp_path)]) == 0 + assert "— examined 1 CLAUDE.md" in capsys.readouterr().out + + def test_zero_denominator_for_exemptions_counts_files_not_surface_bases( tmp_path: Path, capsys: Any ) -> None: diff --git a/tests/test_integration_consumer.py b/tests/test_integration_consumer.py index afdc184..c3c4686 100644 --- a/tests/test_integration_consumer.py +++ b/tests/test_integration_consumer.py @@ -102,6 +102,36 @@ def test_clean_consumer_passes_the_full_battery( assert "complexipy" in by_gate, "the ratchet stage ran (snapshot watermark present)" +def test_every_cf_gate_reaches_the_board_carrying_a_denominator( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """The denominator of the denominators — asserted through the REAL wire. + + Every other denominator test drives one gate's ``main`` in-process. This one + is the only place the whole crossing is proven: gate subprocess -> JSON wire + -> ``_structured_verdict`` parse-back -> the aggregated board. That crossing + is where the measurement used to be thrown away, so a gate whose wire form + regressed — or an eighth cf-* stage added without ``measured=`` — would + otherwise land as a bare ``PASS`` with nothing going red. + + ``_verdict_from_proc`` FABRICATES an empty verdict for a cf-* gate whose + output it cannot parse (wire shape 3). That fallback is invisible on a green + board, which is exactly why the assertion is ``notices and evidence`` rather + than a passed flag: it makes the silent fallback fatal for a kit gate. + """ + consumer = _consumer_copy(tmp_path) + _stub_pytest(monkeypatch) + + verdicts = run_battery(consumer, os.environ) + cf_verdicts = [v for v in verdicts if v.gate.startswith("cf-") and v.error is None] + + assert cf_verdicts, _board(verdicts) # the gates ran at all — never a vacuous loop + for verdict in cf_verdicts: + assert verdict.notices, f"{verdict.gate} reached the board with no denominator" + assert verdict.evidence, f"{verdict.gate} carries no machine-form measurement" + assert len(verdict.notices) == 1, f"{verdict.gate}: ONE denominator line, not a blob" + + def test_injected_type_error_flips_the_battery_red( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: diff --git a/tests/test_mirror_check.py b/tests/test_mirror_check.py index bb0cc6b..20f5a46 100644 --- a/tests/test_mirror_check.py +++ b/tests/test_mirror_check.py @@ -191,6 +191,20 @@ def test_multiple_rows_collect_all_violations(self, tmp_path: Path) -> None: codes = sorted(v.code for v in check_mirrors(tmp_path, today=TODAY)[0]) assert codes == ["MIRROR_DIVERGED", "MIRROR_FILE_MISSING"] + def test_row_count_tracks_the_rows_checked(self, tmp_path: Path) -> None: + # The board's denominator. Asserted here against a KNOWN row count, + # because the zero case (a header with no rows) would equally survive a + # hardcoded 0 — and then every board line would claim 0 mirrors checked + # while the gate was really enforcing three. + digest = make_mirror(tmp_path, "data/good.md", b"fine\n") + rows = ( + row("data/good.md", digest) + + row("data/ghost.md", sha256_of(b"y"), artifact="ghost") + + row("data/good.md", sha256_of(b"other"), artifact="diverged") + ) + write_mirrors(tmp_path, rows) + assert check_mirrors(tmp_path, today=TODAY)[1] == 3 + class TestInit: def test_init_writes_template_that_parses_clean(self, tmp_path: Path) -> None: