From a78dacbd608de5ac6a3102704f21f63123377138 Mon Sep 17 00:00:00 2001 From: kegouro <141108917+kegouro@users.noreply.github.com> Date: Mon, 3 Aug 2026 05:00:55 -0400 Subject: [PATCH] fix(docs): use content-addressed manual provenance --- .github/workflows/docs.yml | 2 - docs/manual/artifacts-manifest.json | 26 +++ docs/manual/downloads.md | 10 +- scripts/check_docs_sync.py | 58 ++----- scripts/update_manual_provenance.py | 192 +++++++++++++++++++++ tests/validation/test_manual_provenance.py | 102 +++++++++++ 6 files changed, 336 insertions(+), 54 deletions(-) create mode 100644 docs/manual/artifacts-manifest.json create mode 100644 scripts/update_manual_provenance.py create mode 100644 tests/validation/test_manual_provenance.py diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 82ab7e8..e041473 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -20,7 +20,6 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 with: - fetch-depth: 0 persist-credentials: false - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 @@ -65,7 +64,6 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 with: - fetch-depth: 0 persist-credentials: false - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 diff --git a/docs/manual/artifacts-manifest.json b/docs/manual/artifacts-manifest.json new file mode 100644 index 0000000..7400d4b --- /dev/null +++ b/docs/manual/artifacts-manifest.json @@ -0,0 +1,26 @@ +{ + "artifacts": [ + { + "bytes": 56415, + "path": "docs/user-guide.md", + "sha256": "dbbfa1fddffd5f25b3b880c80b33f6b8b1e37b0c15dd85af25c7aacef9f8fe08" + }, + { + "bytes": 39873, + "path": "docs/user-guide.tex", + "sha256": "d11a27d1889510c6beaed2f1055216b54366e737cbfbfc4bbbaaa5e6617bd6db" + }, + { + "bytes": 117821, + "path": "docs/user-guide.pdf", + "sha256": "1ec76b89c84151b2f75621bf3f37eb0b86c887430130835fc7bffd2d796b79e9" + } + ], + "policy": { + "encoding": "UTF-8", + "identity": "content-addressed-manual-artifacts", + "newline": "LF", + "updater": "scripts/update_manual_provenance.py" + }, + "schema_version": 1 +} diff --git a/docs/manual/downloads.md b/docs/manual/downloads.md index a858f64..4963503 100644 --- a/docs/manual/downloads.md +++ b/docs/manual/downloads.md @@ -9,11 +9,11 @@ **Documented source:** `0.1.5.dev0` · **GitHub release:** `0.1.4` · **PyPI distribution:** `0.1.2` · **Build date:** 2026-07-29 -**Source commit:** -[`06da8895ef9d7dfb5978f97f8283695deb02f870`](https://github.com/kegouro/spmkit/commit/06da8895ef9d7dfb5978f97f8283695deb02f870) - -**PDF SHA-256:** -`1ec76b89c84151b2f75621bf3f37eb0b86c887430130835fc7bffd2d796b79e9` +**Artifact identity:** Exact byte identities for the published manual artifacts +are recorded in the committed [content-addressed manifest](artifacts-manifest.json). +CI verifies the current files against that manifest. Git commits and release +tags identify repository revisions; the manifest proves exact artifact identity +only, not semantic correctness or reproducible PDF generation. To compile the PDF from source: diff --git a/scripts/check_docs_sync.py b/scripts/check_docs_sync.py index 0ab030c..38f1fcf 100644 --- a/scripts/check_docs_sync.py +++ b/scripts/check_docs_sync.py @@ -11,6 +11,8 @@ from pathlib import Path from typing import Any +from update_manual_provenance import check_manifest + try: import yaml except ImportError as exc: @@ -271,52 +273,14 @@ def main() -> int: pdf = DOCS / "user-guide.pdf" pages = pdf_pages(pdf) checks.require(pages == 19, "committed PDF has 19 pages") - actual_pdf_hash = sha256(pdf) downloads = text(DOCS / "manual/downloads.md") - published_hash = re.search(r"PDF SHA-256:\*\*\s*\n`([0-9a-f]{64})`", downloads) - checks.require( - bool(published_hash and published_hash.group(1) == actual_pdf_hash), - "download metadata matches committed PDF SHA-256", - ) - source_match = re.search( - r"Source commit:\*\*\s*\n\[`([0-9a-f]{40})`\]" - r"\(https://github\.com/kegouro/spmkit/commit/([0-9a-f]{40})\)", - downloads, - ) - source_commit_ok = False - if source_match and source_match.group(1) == source_match.group(2): - source_commit = source_match.group(1) - ancestor = subprocess.run( - ["git", "merge-base", "--is-ancestor", source_commit, "HEAD"], - cwd=REPO, - check=False, - ) - source_commit_ok = ancestor.returncode == 0 - for manual in required_manual_files: - relative = manual.relative_to(REPO).as_posix() - committed = subprocess.run( - ["git", "rev-parse", f"{source_commit}:{relative}"], - cwd=REPO, - capture_output=True, - check=False, - text=True, - ) - current = subprocess.run( - ["git", "hash-object", relative], - cwd=REPO, - capture_output=True, - check=False, - text=True, - ) - source_commit_ok = source_commit_ok and ( - committed.returncode == 0 - and current.returncode == 0 - and committed.stdout.strip() == current.stdout.strip() - ) + provenance_errors = check_manifest(REPO) checks.require( - source_commit_ok, - "source commit is an ancestor containing the published manual artifacts", + not provenance_errors, + "content-addressed manual artifact manifest matches published files", ) + for error in provenance_errors: + print(f"[FAIL] manual provenance: {error}") checks.require("115 KiB" in downloads and "19-page" in downloads, "PDF size/page metadata") viewer_dir = DOCS / "assets/pdf-viewer" @@ -324,14 +288,14 @@ def main() -> int: checks.require(viewer.stat().st_size > 0, "embedded PDF reader is present") pdfjs_hashes = { "vendor/pdf.min.mjs": "343b4166b06716a55a8f87175b83223cb1a9ab701eb8a96b2577509d47fbaf4a", - "vendor/pdf.worker.min.mjs": "dbcae78a691b3c501508f74b774c6066a57a14a76cefdc9e25ad86b651bb75d5", + "vendor/pdf.worker.min.mjs": ( + "dbcae78a691b3c501508f74b774c6066a57a14a76cefdc9e25ad86b651bb75d5" + ), } for relative, expected_hash in pdfjs_hashes.items(): asset = viewer_dir / relative checks.require( - asset.is_file() - and asset.stat().st_size > 100_000 - and sha256(asset) == expected_hash, + asset.is_file() and asset.stat().st_size > 100_000 and sha256(asset) == expected_hash, f"pinned PDF.js asset: {relative}", ) pdfjs_notice = text(viewer_dir / "NOTICE.txt") diff --git a/scripts/update_manual_provenance.py b/scripts/update_manual_provenance.py new file mode 100644 index 0000000..92c3c91 --- /dev/null +++ b/scripts/update_manual_provenance.py @@ -0,0 +1,192 @@ +#!/usr/bin/env python3 +"""Create and verify the content-addressed manual artifact manifest.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import tempfile +from pathlib import Path, PurePosixPath +from typing import Any + +SCHEMA_VERSION = 1 +MANIFEST_RELATIVE = "docs/manual/artifacts-manifest.json" +ARTIFACT_PATHS = ( + "docs/user-guide.md", + "docs/user-guide.tex", + "docs/user-guide.pdf", +) +EXPECTED_POLICY = { + "identity": "content-addressed-manual-artifacts", + "encoding": "UTF-8", + "newline": "LF", + "updater": "scripts/update_manual_provenance.py", +} +FORBIDDEN_PATH_PARTS = {"__pycache__", ".cache", ".git", "tmp"} + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _artifact_record(root: Path, relative: str) -> dict[str, Any]: + path = root / relative + if not path.is_file(): + raise ValueError(f"missing manual artifact: {relative}") + return {"bytes": path.stat().st_size, "path": relative, "sha256": _sha256(path)} + + +def build_manifest(root: Path) -> dict[str, Any]: + return { + "artifacts": [_artifact_record(root, relative) for relative in ARTIFACT_PATHS], + "policy": EXPECTED_POLICY, + "schema_version": SCHEMA_VERSION, + } + + +def render_manifest(manifest: dict[str, Any]) -> bytes: + return (json.dumps(manifest, indent=2, sort_keys=True) + "\n").encode("utf-8") + + +def manifest_path(root: Path) -> Path: + return root / MANIFEST_RELATIVE + + +def write_manifest(root: Path) -> None: + target = manifest_path(root) + target.parent.mkdir(parents=True, exist_ok=True) + payload = render_manifest(build_manifest(root)) + with tempfile.NamedTemporaryFile( + dir=target.parent, + prefix=f".{target.name}.", + suffix=".tmp", + delete=False, + ) as handle: + temporary = Path(handle.name) + handle.write(payload) + handle.flush() + os.fsync(handle.fileno()) + os.replace(temporary, target) + + +def _validate_relative_path(value: Any, field: str) -> list[str]: + errors: list[str] = [] + if not isinstance(value, str): + return [f"{field} must be a string"] + parsed = PurePosixPath(value) + if parsed.is_absolute() or "\\" in value or any(part == ".." for part in parsed.parts): + errors.append(f"{field} must be repository-relative") + if any(part in FORBIDDEN_PATH_PARTS for part in parsed.parts): + errors.append(f"{field} contains a forbidden local/cache path component") + return errors + + +def _validate_manifest_shape(manifest: Any) -> list[str]: + if not isinstance(manifest, dict): + return ["manifest root must be an object"] + errors: list[str] = [] + if manifest.get("schema_version") != SCHEMA_VERSION: + errors.append("unsupported schema_version") + if manifest.get("policy") != EXPECTED_POLICY: + errors.append("policy does not match the canonical updater policy") + artifacts = manifest.get("artifacts") + if not isinstance(artifacts, list): + return [*errors, "artifacts must be a list"] + seen: set[str] = set() + for index, artifact in enumerate(artifacts): + prefix = f"artifacts[{index}]" + if not isinstance(artifact, dict): + errors.append(f"{prefix} must be an object") + continue + errors.extend(_validate_relative_path(artifact.get("path"), f"{prefix}.path")) + relative = artifact.get("path") + if isinstance(relative, str): + if relative in seen: + errors.append(f"duplicate artifact path: {relative}") + seen.add(relative) + if not isinstance(artifact.get("bytes"), int) or artifact.get("bytes", -1) < 0: + errors.append(f"{prefix}.bytes must be a non-negative integer") + digest = artifact.get("sha256") + if ( + not isinstance(digest, str) + or len(digest) != 64 + or any(character not in "0123456789abcdef" for character in digest) + ): + errors.append(f"{prefix}.sha256 must be lowercase hexadecimal SHA-256") + if seen != set(ARTIFACT_PATHS): + errors.append("manifest artifact set does not match the canonical published set") + return errors + + +def check_manifest(root: Path) -> list[str]: + target = manifest_path(root) + if not target.is_file(): + return [f"missing provenance manifest: {MANIFEST_RELATIVE}"] + try: + raw = target.read_bytes() + manifest = json.loads(raw.decode("utf-8")) + except (OSError, UnicodeError, json.JSONDecodeError) as exc: + return [f"cannot parse provenance manifest: {exc}"] + errors = _validate_manifest_shape(manifest) + if errors: + return errors + if render_manifest(manifest) != raw: + errors.append("manifest is not canonical UTF-8 JSON with an LF terminator") + try: + expected = build_manifest(root) + except (OSError, ValueError) as exc: + return [str(exc)] + if manifest != expected: + for expected_artifact, actual_artifact in zip( + expected["artifacts"], manifest["artifacts"], strict=True + ): + if expected_artifact["path"] != actual_artifact["path"]: + continue + if expected_artifact["bytes"] != actual_artifact["bytes"]: + errors.append(f"size mismatch: {expected_artifact['path']}") + if expected_artifact["sha256"] != actual_artifact["sha256"]: + errors.append(f"SHA-256 mismatch: {expected_artifact['path']}") + if render_manifest(manifest) != render_manifest(expected): + errors.append("manifest content is stale or not canonical JSON") + return errors + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + mode = parser.add_mutually_exclusive_group(required=True) + mode.add_argument("--write", action="store_true", help="write the canonical manifest") + mode.add_argument("--check", action="store_true", help="verify the committed manifest") + parser.add_argument( + "--root", + type=Path, + default=Path(__file__).resolve().parents[1], + help=argparse.SUPPRESS, + ) + args = parser.parse_args(argv) + root = args.root.resolve() + try: + if args.write: + write_manifest(root) + print(f"WROTE {manifest_path(root).relative_to(root)}") + return 0 + errors = check_manifest(root) + except OSError as exc: + print(f"MANUAL PROVENANCE FAILED: {exc}") + return 1 + if errors: + print(f"MANUAL PROVENANCE FAILED: {len(errors)} problem(s)") + for error in errors: + print(f" - {error}") + return 1 + print("MANUAL PROVENANCE OK") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/validation/test_manual_provenance.py b/tests/validation/test_manual_provenance.py new file mode 100644 index 0000000..fb00786 --- /dev/null +++ b/tests/validation/test_manual_provenance.py @@ -0,0 +1,102 @@ +from __future__ import annotations + +import importlib.util +import json +import subprocess +import sys +from pathlib import Path + +import pytest + +SCRIPT = Path(__file__).parents[2] / "scripts/update_manual_provenance.py" +ROOT = SCRIPT.parents[1] + + +def _module(): + spec = importlib.util.spec_from_file_location("update_manual_provenance", SCRIPT) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _workspace(tmp_path: Path) -> Path: + root = tmp_path / "repo" + for relative, content in ( + ("docs/user-guide.md", "manual\n"), + ("docs/user-guide.tex", "tex\n"), + ("docs/user-guide.pdf", b"pdf\n"), + ): + path = root / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(content if isinstance(content, bytes) else content.encode()) + return root + + +def _run(root: Path, *args: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [sys.executable, str(SCRIPT), "--root", str(root), *args], + check=False, + capture_output=True, + text=True, + ) + + +def test_write_is_deterministic_and_check_is_clean(tmp_path: Path) -> None: + root = _workspace(tmp_path) + first = _run(root, "--write") + first_bytes = (root / "docs/manual/artifacts-manifest.json").read_bytes() + second = _run(root, "--write") + assert first.returncode == second.returncode == 0 + assert first_bytes == (root / "docs/manual/artifacts-manifest.json").read_bytes() + assert _run(root, "--check").returncode == 0 + + +@pytest.mark.parametrize("mutation", ["bytes", "size"]) +def test_changed_artifact_fails(tmp_path: Path, mutation: str) -> None: + root = _workspace(tmp_path) + assert _run(root, "--write").returncode == 0 + artifact = root / "docs/user-guide.md" + artifact.write_bytes(b"changed\n" if mutation == "bytes" else b"manual\nextra") + result = _run(root, "--check") + assert result.returncode == 1 + assert "mismatch" in result.stdout + + +def test_missing_and_stale_manifest_fail(tmp_path: Path) -> None: + root = _workspace(tmp_path) + assert _run(root, "--check").returncode == 1 + assert _run(root, "--write").returncode == 0 + (root / "docs/user-guide.tex").unlink() + assert _run(root, "--check").returncode == 1 + + +def test_malformed_and_extra_manifest_fail(tmp_path: Path) -> None: + root = _workspace(tmp_path) + manifest = root / "docs/manual/artifacts-manifest.json" + manifest.parent.mkdir(parents=True, exist_ok=True) + manifest.write_text("not json\n", encoding="utf-8") + assert _run(root, "--check").returncode == 1 + assert _run(root, "--write").returncode == 0 + data = json.loads(manifest.read_text(encoding="utf-8")) + manifest.write_text(json.dumps(data), encoding="utf-8") + assert _run(root, "--check").returncode == 1 + data["artifacts"].append({"path": "/absolute/secret", "bytes": 1, "sha256": "0" * 64}) + manifest.write_text(json.dumps(data), encoding="utf-8") + result = _run(root, "--check") + assert result.returncode == 1 + assert "repository-relative" in result.stdout + + +def test_check_does_not_modify_manifest(tmp_path: Path) -> None: + root = _workspace(tmp_path) + assert _run(root, "--write").returncode == 0 + manifest = root / "docs/manual/artifacts-manifest.json" + before = manifest.stat().st_mtime_ns, manifest.read_bytes() + assert _run(root, "--check").returncode == 0 + assert before == (manifest.stat().st_mtime_ns, manifest.read_bytes()) + + +def test_repository_manifest_matches_real_artifacts() -> None: + module = _module() + assert module.check_manifest(ROOT) == []