From ea65b8f5f086ba960002ea2890dc41eff8927964 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 17 Oct 2026 06:24:46 +0000 Subject: [PATCH 1/2] ci: check the pinned threat-model baseline hash against its tag Fail when docs/threat-model.md at a baseline tag no longer matches the hash pinned in docs/audits/threat-model-baseline.md (or the tag is unreadable). Report, without failing, how many commits and lines the current document has drifted since the latest baseline. The pin and the baseline are untouched. Closes #366 Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01GqtiAsmkmDmpynCN35XTzg Signed-off-by: Claude --- .github/workflows/ci.yml | 22 +++ CHANGELOG.md | 9 + Makefile | 5 + scripts/check_threat_model_baseline.py | 193 +++++++++++++++++++ tests/test_threat_model_baseline_check.py | 220 ++++++++++++++++++++++ 5 files changed, 449 insertions(+) create mode 100644 scripts/check_threat_model_baseline.py create mode 100644 tests/test_threat_model_baseline_check.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 44eb430..5245ca1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -100,6 +100,28 @@ jobs: # (synthetic RFC 3161 tokens in committed demo/golden packets). Any # finding not fingerprinted there still fails this gate. + threat-model-baseline: + name: threat-model baseline (frozen tags must match their pins; drift is report-only) + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Harden Runner (audit mode) + uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 + with: + egress-policy: audit + - name: Checkout (full history and tags — the check reads the document at each baseline tag) + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + fetch-tags: true + persist-credentials: false + # Fails if docs/threat-model.md at a baseline tag no longer hashes to the value pinned in + # docs/audits/threat-model-baseline.md. Drift of the current file from the latest + # baseline is printed as a warning annotation and never fails the job (issue #366). + # Standard library only; no environment to build. + - name: Check the pinned baselines + run: python3 scripts/check_threat_model_baseline.py + verifier-portability: name: verifier subset compiles on older Pythons runs-on: ubuntu-latest diff --git a/CHANGELOG.md b/CHANGELOG.md index 9a05b86..711cf74 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,15 @@ follow [Semantic Versioning](https://semver.org/). The **packet format** and the ### Added +- **A CI check for the pinned threat-model baseline** (#366). + `scripts/check_threat_model_baseline.py` reads each baseline in + `docs/audits/threat-model-baseline.md` and fails if `docs/threat-model.md` at that + baseline's tag no longer hashes to the pinned value, or if the tag cannot be read. + It also reports, without failing, when the current document differs from the latest + baseline, naming the commits and lines changed since the freeze. It runs as the + `threat-model baseline` job in `ci.yml` (and `make threat-model-baseline`), with tests + in `tests/test_threat_model_baseline_check.py`. The pinned hash and the baseline + document are untouched; cutting B2 still needs a maintainer re-review. - **Google Analytics 4 on the documentation website, and only there** (owner decision, 2026-09-17: GA4 on every public site, with the privacy copy updated to match). The app, the CLI and the relay keep the no-telemetry rule unchanged. `site/analytics.js` diff --git a/Makefile b/Makefile index c2c9ba6..c46edd1 100644 --- a/Makefile +++ b/Makefile @@ -132,6 +132,11 @@ i18n-usage: ## Report app i18n keys with no rendering path (reports; never gates doc-links: ## Validate local Markdown links and capability-ledger evidence paths uv run python scripts/check_doc_links.py +threat-model-baseline: ## Fail if a frozen threat-model baseline tag no longer matches its pinned hash; report (never fail) drift since the freeze (issue #366) + # Not in `verify`: it needs the baseline tags, which a shallow or tagless clone does + # not have. CI runs it in its own job with `fetch-tags`. Drift is printed, not gated. + python3 scripts/check_threat_model_baseline.py + markers: ## No bare TODO/FIXME/HACK (must reference an issue, e.g. TODO(#142)); no un-issued noqa/type:ignore @bad=$$(grep -rnE '(TODO|FIXME|HACK)' --include='*.py' --include='*.js' src tests app scripts fuzz 2>/dev/null \ | grep -vE '\(#[0-9]+\)' || true); \ diff --git a/scripts/check_threat_model_baseline.py b/scripts/check_threat_model_baseline.py new file mode 100644 index 0000000..9d9ff7f --- /dev/null +++ b/scripts/check_threat_model_baseline.py @@ -0,0 +1,193 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later +# Copyright 2026 Chelsea Kelly-Reif +"""Check the frozen threat-model baselines, and report drift from the latest one. + +``docs/audits/threat-model-baseline.md`` pins a SHA-256 for each baseline (B1, ...) and names +the annotated git tag that marks its freeze commit. Two things follow from "immutable once +tagged", and this script treats them differently: + +1. **Fails** when the frozen document, read from a baseline's tag, no longer hashes to the value + pinned in the audit file. That is the immutability the baseline claims: the tag was moved or + the pin was edited. It also fails when a pinned tag cannot be found, because a check that + cannot read the tag has verified nothing (a shallow checkout must fetch tags). +2. **Reports, and does not fail,** when the current ``docs/threat-model.md`` differs from the + latest baseline, naming the commits and lines changed since the freeze. Drift is expected + between baselines; the point is that it stays visible until a new baseline is cut and + re-reviewed by a person. This script never writes the pin or the threat model. + +Standard library only, so it runs before project dependencies are installed. +""" + +from __future__ import annotations + +import argparse +import hashlib +import os +import re +import shutil +import subprocess +import sys +from dataclasses import dataclass +from pathlib import Path + +BASELINE_FILE = Path("docs/audits/threat-model-baseline.md") +_DEFAULT_DOCUMENT = "docs/threat-model.md" + +_HEADING = re.compile(r"^## Baseline (B\d+)\s*$", re.MULTILINE) +_NEXT_HEADING = re.compile(r"^## ", re.MULTILINE) +_HASH_ROW = re.compile(r"^\|\s*Document SHA-256\s*\|\s*`([0-9a-f]{64})`\s*\|", re.MULTILINE) +_TAG_ROW = re.compile(r"^\|\s*Git tag\s*\|\s*`([^`\s]+)`", re.MULTILINE) +_DOC_ROW = re.compile(r"^\|\s*Frozen document\s*\|\s*\[`([^`\s]+)`\]", re.MULTILINE) + + +@dataclass(frozen=True) +class Baseline: + """One pinned baseline as written in the audit file.""" + + baseline_id: str + document: str + sha256: str + tag: str + + +class BaselineError(Exception): + """The audit file does not carry a readable pin.""" + + +def parse_baselines(text: str) -> list[Baseline]: + """Return the baselines in file order. Raises if a section is missing its hash or tag.""" + headings = list(_HEADING.finditer(text)) + if not headings: + raise BaselineError("no '## Baseline B' section found") + baselines: list[Baseline] = [] + for heading in headings: + start = heading.end() + following = _NEXT_HEADING.search(text, start) + section = text[start : following.start() if following else len(text)] + digest = _HASH_ROW.search(section) + tag = _TAG_ROW.search(section) + if digest is None or tag is None: + raise BaselineError( + f"baseline {heading.group(1)} has no readable 'Document SHA-256' and 'Git tag' rows" + ) + document = _DOC_ROW.search(section) + baselines.append( + Baseline( + baseline_id=heading.group(1), + document=document.group(1) if document else _DEFAULT_DOCUMENT, + sha256=digest.group(1), + tag=tag.group(1), + ) + ) + return baselines + + +def _git(root: Path, *args: str) -> subprocess.CompletedProcess[bytes]: + git = shutil.which("git") + if git is None: + raise BaselineError("git is not installed; the baseline tags cannot be read") + return subprocess.run( # noqa: S603 - resolved git path and constant arguments + [git, "-C", str(root), *args], capture_output=True, check=False + ) + + +def _file_at_tag(root: Path, tag: str, document: str) -> bytes | None: + result = _git(root, "show", f"refs/tags/{tag}:{document}") + return result.stdout if result.returncode == 0 else None + + +def verify_immutable(root: Path, baselines: list[Baseline]) -> list[str]: + """Return one problem per baseline whose tagged document no longer matches its pin.""" + problems: list[str] = [] + for baseline in baselines: + frozen = _file_at_tag(root, baseline.tag, baseline.document) + if frozen is None: + problems.append( + f"{baseline.baseline_id}: cannot read {baseline.document} at tag " + f"{baseline.tag} (is the tag missing? a CI checkout needs " + "fetch-depth: 0 and fetch-tags: true)" + ) + continue + actual = hashlib.sha256(frozen).hexdigest() + if actual != baseline.sha256: + problems.append( + f"{baseline.baseline_id}: {baseline.document} at tag {baseline.tag} hashes to " + f"{actual}, but the audit file pins {baseline.sha256}. A baseline is immutable " + "once tagged; do not move the tag or edit the pin. Cut a new baseline instead." + ) + return problems + + +def drift_report(root: Path, baseline: Baseline) -> str | None: + """Describe how the working-tree document differs from ``baseline``, or ``None`` if it does not. + + Reports commits (touching the document, since the tag) and added/removed lines (tag against + the working tree, so uncommitted edits count too). Report-only: callers must not fail on it. + """ + path = root / baseline.document + if not path.is_file(): + return f"{baseline.document} is missing; the latest baseline is {baseline.baseline_id}" + if hashlib.sha256(path.read_bytes()).hexdigest() == baseline.sha256: + return None + + commits = "an unknown number of" + counted = _git( + root, "rev-list", "--count", f"refs/tags/{baseline.tag}..HEAD", "--", baseline.document + ) + if counted.returncode == 0: + commits = counted.stdout.decode().strip() + + lines = "an unknown number of lines changed" + numstat = _git(root, "diff", "--numstat", f"refs/tags/{baseline.tag}", "--", baseline.document) + if numstat.returncode == 0 and numstat.stdout.strip(): + added, removed, *_ = numstat.stdout.decode().split() + lines = f"{added} lines added, {removed} removed" + + return ( + f"{baseline.document} differs from baseline {baseline.baseline_id} " + f"(tag {baseline.tag}): {commits} commit(s) since the freeze, {lines}. " + "Report only: a maintainer re-review cuts the next baseline." + ) + + +def check(root: Path) -> tuple[list[str], str | None]: + """Return ``(problems, drift)``. Only ``problems`` should fail a build.""" + audit = root / BASELINE_FILE + try: + baselines = parse_baselines(audit.read_text(encoding="utf-8")) + except (OSError, BaselineError) as exc: + return [f"{BASELINE_FILE}: {exc}"], None + try: + problems = verify_immutable(root, baselines) + return problems, drift_report(root, baselines[-1]) + except BaselineError as exc: + return [str(exc)], None + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--root", + type=Path, + default=Path(__file__).resolve().parent.parent, + help="repository root (defaults to the parent of scripts/)", + ) + args = parser.parse_args() + problems, drift = check(args.root) + + # GitHub renders ::warning:: as an annotation; on a terminal it reads as a plain line. + on_actions = os.environ.get("GITHUB_ACTIONS") == "true" + if drift is not None: + prefix = "::warning title=Threat model drift::" if on_actions else "DRIFT (report only): " + print(f"{prefix}{drift}") + if problems: + print("threat-model baseline gate failed:") + for problem in problems: + print(f"- {problem}") + return 1 + print("threat-model baselines: every pinned tag matches its pinned hash") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_threat_model_baseline_check.py b/tests/test_threat_model_baseline_check.py new file mode 100644 index 0000000..c498949 --- /dev/null +++ b/tests/test_threat_model_baseline_check.py @@ -0,0 +1,220 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later +# Copyright 2026 Chelsea Kelly-Reif +"""The threat-model baseline gate: immutability fails, drift only reports (issue #366).""" + +from __future__ import annotations + +import hashlib +import importlib.util +import subprocess +import sys +from pathlib import Path +from types import ModuleType + +import pytest + +_ROOT = Path(__file__).resolve().parent.parent +_SCRIPT = _ROOT / "scripts" / "check_threat_model_baseline.py" + +_TAG = "threat-model-baseline-B1" +_FROZEN = "line one\nline two\nline three\n" + + +def _load() -> ModuleType: + spec = importlib.util.spec_from_file_location("check_threat_model_baseline", _SCRIPT) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +gate = _load() + + +def _git(root: Path, *args: str) -> None: + subprocess.run( + [ + "git", + "-c", + "user.name=Test", + "-c", + "user.email=test@example.invalid", + "-c", + "commit.gpgsign=false", + "-c", + "tag.gpgsign=false", + "-C", + str(root), + *args, + ], + check=True, + capture_output=True, + ) + + +def _audit_text(sha: str, *, tag: str = _TAG) -> str: + return ( + "# Threat-model baseline\n\n" + "## Baseline B1\n\n" + "| Field | Value |\n| --- | --- |\n" + "| Baseline id | **B1** |\n" + "| Frozen document | [`docs/threat-model.md`](../threat-model.md) |\n" + f"| Document SHA-256 | `{sha}` |\n" + f"| Git tag | `{tag}` (annotated) |\n\n" + '## What "frozen" means\n\nImmutable.\n' + ) + + +def _sha(text: str) -> str: + return hashlib.sha256(text.encode()).hexdigest() + + +@pytest.fixture +def repo(tmp_path: Path) -> Path: + """A throwaway repository with the model frozen and tagged at ``_FROZEN``.""" + _git(tmp_path, "init", "-q") + (tmp_path / "docs" / "audits").mkdir(parents=True) + (tmp_path / "docs" / "threat-model.md").write_text(_FROZEN) + (tmp_path / "docs" / "audits" / "threat-model-baseline.md").write_text( + _audit_text(_sha(_FROZEN)) + ) + _git(tmp_path, "add", "-A") + _git(tmp_path, "commit", "-q", "-m", "freeze") + _git(tmp_path, "tag", "-a", _TAG, "-m", "B1") + return tmp_path + + +def _run(root: Path) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [sys.executable, str(_SCRIPT), "--root", str(root)], + capture_output=True, + check=False, + text=True, + ) + + +def test_the_real_audit_file_pins_a_hash_and_tag_for_b1() -> None: + """The parser reads the file this gate guards; the tag itself is checked by the CI job.""" + text = (_ROOT / gate.BASELINE_FILE).read_text(encoding="utf-8") + baselines = gate.parse_baselines(text) + assert baselines[0].baseline_id == "B1" + assert baselines[0].document == "docs/threat-model.md" + assert baselines[0].tag == _TAG + assert len(baselines[0].sha256) == 64 + + +def test_a_file_with_no_baseline_section_is_an_error() -> None: + with pytest.raises(gate.BaselineError): + gate.parse_baselines("# nothing pinned here\n") + + +def test_a_section_missing_its_hash_is_an_error() -> None: + with pytest.raises(gate.BaselineError, match="B1"): + gate.parse_baselines("## Baseline B1\n\n| Git tag | `t` |\n") + + +def test_an_untouched_baseline_passes_with_no_drift(repo: Path) -> None: + problems, drift = gate.check(repo) + assert problems == [] + assert drift is None + result = _run(repo) + assert result.returncode == 0 + assert "DRIFT" not in result.stdout + + +def test_a_tagged_file_that_no_longer_matches_the_pin_fails(repo: Path) -> None: + """Someone edited the pin (or moved the tag): the immutability B1 claims is gone.""" + audit = repo / "docs" / "audits" / "threat-model-baseline.md" + audit.write_text(_audit_text(_sha("a different document\n"))) + problems, _ = gate.check(repo) + assert len(problems) == 1 + assert "immutable" in problems[0] + result = _run(repo) + assert result.returncode == 1 + assert "baseline gate failed" in result.stdout + + +def test_a_moved_tag_fails(repo: Path) -> None: + (repo / "docs" / "threat-model.md").write_text(_FROZEN + "added after the freeze\n") + _git(repo, "commit", "-q", "-am", "edit") + _git(repo, "tag", "-f", "-a", _TAG, "-m", "moved") + problems, _ = gate.check(repo) + assert len(problems) == 1 + assert _sha(_FROZEN + "added after the freeze\n") in problems[0] + assert _run(repo).returncode == 1 + + +def test_a_missing_tag_fails_rather_than_passing_unchecked(repo: Path) -> None: + _git(repo, "tag", "-d", _TAG) + problems, _ = gate.check(repo) + assert len(problems) == 1 + assert "fetch-tags" in problems[0] + assert _run(repo).returncode == 1 + + +def test_drift_is_reported_with_commits_and_lines_but_does_not_fail(repo: Path) -> None: + model = repo / "docs" / "threat-model.md" + model.write_text(_FROZEN + "four\n") + _git(repo, "commit", "-q", "-am", "one") + model.write_text("line one\nline three\nfour\nfive\n") + _git(repo, "commit", "-q", "-am", "two") + + problems, drift = gate.check(repo) + assert problems == [] + assert drift is not None + assert "baseline B1" in drift + assert "2 commit(s)" in drift + assert "2 lines added, 1 removed" in drift + + result = _run(repo) + assert result.returncode == 0 + assert "DRIFT (report only)" in result.stdout + assert "every pinned tag matches" in result.stdout + + +def test_uncommitted_edits_count_as_drift(repo: Path) -> None: + (repo / "docs" / "threat-model.md").write_text(_FROZEN + "unsaved thought\n") + _, drift = gate.check(repo) + assert drift is not None + assert "0 commit(s)" in drift + assert "1 lines added, 0 removed" in drift + + +def test_github_actions_gets_a_warning_annotation( + repo: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + (repo / "docs" / "threat-model.md").write_text(_FROZEN + "more\n") + monkeypatch.setenv("GITHUB_ACTIONS", "true") + result = _run(repo) + assert result.returncode == 0 + assert "::warning title=Threat model drift::" in result.stdout + + +def test_the_latest_baseline_is_the_one_drift_is_measured_against(repo: Path) -> None: + """Appending B2 does not rewrite B1: both are checked, drift is reported against B2.""" + model = repo / "docs" / "threat-model.md" + second = _FROZEN + "second baseline\n" + model.write_text(second) + _git(repo, "commit", "-q", "-am", "b2 text") + _git(repo, "tag", "-a", "threat-model-baseline-B2", "-m", "B2") + audit = repo / "docs" / "audits" / "threat-model-baseline.md" + b2_section = ( + "## Baseline B2\n\n" + "| Field | Value |\n| --- | --- |\n" + "| Frozen document | [`docs/threat-model.md`](../threat-model.md) |\n" + f"| Document SHA-256 | `{_sha(second)}` |\n" + "| Git tag | `threat-model-baseline-B2` (annotated) |\n\n" + ) + audit.write_text( + audit.read_text().replace('## What "frozen" means', b2_section + '## What "frozen" means') + ) + problems, drift = gate.check(repo) + assert problems == [] + assert drift is None + + model.write_text(second + "after b2\n") + problems, drift = gate.check(repo) + assert problems == [] + assert drift is not None + assert "baseline B2" in drift From b6a19a30965ed3ae579a11df74dcbbffef27f8e9 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 17 Oct 2026 06:29:20 +0000 Subject: [PATCH 2/2] test: do not inherit GITHUB_ACTIONS in the baseline-check tests On the CI runner the script emitted a ::warning:: annotation instead of the plain DRIFT line the test expected. Set the variable explicitly per run. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01GqtiAsmkmDmpynCN35XTzg Signed-off-by: Claude --- tests/test_threat_model_baseline_check.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/tests/test_threat_model_baseline_check.py b/tests/test_threat_model_baseline_check.py index c498949..554c8b0 100644 --- a/tests/test_threat_model_baseline_check.py +++ b/tests/test_threat_model_baseline_check.py @@ -6,6 +6,7 @@ import hashlib import importlib.util +import os import subprocess import sys from pathlib import Path @@ -85,11 +86,16 @@ def repo(tmp_path: Path) -> Path: return tmp_path -def _run(root: Path) -> subprocess.CompletedProcess[str]: +def _run(root: Path, *, actions: bool = False) -> subprocess.CompletedProcess[str]: + """Run the script with GITHUB_ACTIONS set deliberately, not inherited from the CI runner.""" + env = {key: value for key, value in os.environ.items() if key != "GITHUB_ACTIONS"} + if actions: + env["GITHUB_ACTIONS"] = "true" return subprocess.run( [sys.executable, str(_SCRIPT), "--root", str(root)], capture_output=True, check=False, + env=env, text=True, ) @@ -181,12 +187,9 @@ def test_uncommitted_edits_count_as_drift(repo: Path) -> None: assert "1 lines added, 0 removed" in drift -def test_github_actions_gets_a_warning_annotation( - repo: Path, monkeypatch: pytest.MonkeyPatch -) -> None: +def test_github_actions_gets_a_warning_annotation(repo: Path) -> None: (repo / "docs" / "threat-model.md").write_text(_FROZEN + "more\n") - monkeypatch.setenv("GITHUB_ACTIONS", "true") - result = _run(repo) + result = _run(repo, actions=True) assert result.returncode == 0 assert "::warning title=Threat model drift::" in result.stdout