From a07561620467b711939408c2f52341d27b735637 Mon Sep 17 00:00:00 2001 From: codeasier Date: Mon, 7 Sep 2026 01:53:39 +0800 Subject: [PATCH] feat(tests): name the offending files when the fixture provenance guard aborts The session-start guard in tests/conftest.py intentionally stops pytest before collection whenever tests/fixtures/snapshots/ contains a pickle that SHA256SUMS does not list, so an unreviewed object can never be deserialized by accident. Its only message, however, was "Executable fixture set does not match the reviewed manifest": it did not say which file was unexpected or missing, nor what to do about it, and the directory's .gitignore rule for *.pickle makes it look like a safe place for local captures. verify_snapshot_fixtures() now reports the sorted unexpected and missing names separately, points at the fix (move the file out, e.g. to .tmp/, or complete provenance review), and reports SHA256SUMS/size-table drift as its own error. The guard still never reads pickle content. The function takes the fixture directory, manifest, and size table as defaulted parameters so the diagnostics are unit-tested against a temporary directory without touching the real allowlist. tests/AGENTS.md and tests/fixtures/AGENTS.md document that any stray pickle aborts the whole session and that there is deliberately no override. Fixes #118 --- tests/AGENTS.md | 1 + tests/_fixture_provenance.py | 43 ++++++++++++--- tests/fixtures/AGENTS.md | 1 + tests/test_fixture_provenance.py | 92 ++++++++++++++++++++++++++++++++ 4 files changed, 129 insertions(+), 8 deletions(-) diff --git a/tests/AGENTS.md b/tests/AGENTS.md index 92af0e1..a6845c6 100644 --- a/tests/AGENTS.md +++ b/tests/AGENTS.md @@ -55,6 +55,7 @@ - CLI tests exercise Typer commands through test runners and assert printed output. - Query tests build minimal SQLite schemas and YAML-like config objects around the template pipeline. - Snapshot/import tests may deserialize only objects accepted by `fixtures/snapshots/PROVENANCE.md` and `SHA256SUMS`; never load a new or changed pickle before review. +- `conftest.py` aborts the whole pytest session before collection when any `.pkl`/`.pickle` in `tests/fixtures/snapshots/` is not listed in `SHA256SUMS` (or a listed one is absent); the exit message names the offending files. Keep local analysis snapshots outside that directory, for example under `.tmp/`. There is deliberately no override flag. ## Dependencies diff --git a/tests/_fixture_provenance.py b/tests/_fixture_provenance.py index c84ca1a..86066b5 100644 --- a/tests/_fixture_provenance.py +++ b/tests/_fixture_provenance.py @@ -28,9 +28,9 @@ class FixtureProvenanceError(RuntimeError): pass -def load_manifest() -> dict[str, str]: +def load_manifest(manifest: Path = MANIFEST) -> dict[str, str]: entries: dict[str, str] = {} - for line in MANIFEST.read_text().splitlines(): + for line in manifest.read_text().splitlines(): digest, filename = line.split(" ", 1) if filename in entries: raise FixtureProvenanceError(f"Duplicate fixture manifest entry: {filename}") @@ -46,13 +46,40 @@ def _sha256(path: Path) -> str: return digest.hexdigest() -def verify_snapshot_fixtures() -> None: - expected = load_manifest() +def _format_names(names: list[str]) -> str: + return ", ".join(names) if names else "" + + +def _describe_set_mismatch(fixture_dir: Path, unexpected: list[str], missing: list[str]) -> str: + # Only file names are reported; the guard never reads pickle content. + return ( + f"Executable fixture set does not match the reviewed manifest in {fixture_dir}.\n" + f" unexpected (not in SHA256SUMS): {_format_names(unexpected)}\n" + f" missing (in SHA256SUMS but absent): {_format_names(missing)}\n" + f"Move unreviewed pickles out of {fixture_dir}/ (for example into .tmp/), or complete " + "the provenance review in PROVENANCE.md and update SHA256SUMS before running the suite." + ) + + +def verify_snapshot_fixtures( + fixture_dir: Path = FIXTURE_DIR, + manifest: Path = MANIFEST, + expected_sizes: dict[str, int] = EXPECTED_SIZES, +) -> None: + expected = load_manifest(manifest) fixtures = { - path.name: path for path in FIXTURE_DIR.iterdir() if path.suffix.lower() in PICKLE_SUFFIXES + path.name: path for path in fixture_dir.iterdir() if path.suffix.lower() in PICKLE_SUFFIXES } - if set(fixtures) != set(expected) or set(fixtures) != set(EXPECTED_SIZES): - raise FixtureProvenanceError("Executable fixture set does not match the reviewed manifest") + unexpected = sorted(set(fixtures) - set(expected)) + missing = sorted(set(expected) - set(fixtures)) + if unexpected or missing: + raise FixtureProvenanceError(_describe_set_mismatch(fixture_dir, unexpected, missing)) + if set(expected) != set(expected_sizes): + raise FixtureProvenanceError( + "SHA256SUMS and the reviewed size table disagree; update both together.\n" + f" only in SHA256SUMS: {_format_names(sorted(set(expected) - set(expected_sizes)))}\n" + f" only in size table: {_format_names(sorted(set(expected_sizes) - set(expected)))}" + ) for filename, path in fixtures.items(): pointer = LFS_POINTER.fullmatch(path.read_bytes()) if path.stat().st_size < 1024 else None @@ -60,5 +87,5 @@ def verify_snapshot_fixtures() -> None: actual_size = int(pointer.group("size")) if pointer else path.stat().st_size if actual_digest != expected[filename]: raise FixtureProvenanceError(f"Fixture checksum mismatch: {filename}") - if actual_size != EXPECTED_SIZES[filename]: + if actual_size != expected_sizes[filename]: raise FixtureProvenanceError(f"Fixture size mismatch: {filename}") diff --git a/tests/fixtures/AGENTS.md b/tests/fixtures/AGENTS.md index ba764ea..dbc5d03 100644 --- a/tests/fixtures/AGENTS.md +++ b/tests/fixtures/AGENTS.md @@ -17,4 +17,5 @@ they are tracked by Git. ## Verification - Run `pytest tests/test_fixture_provenance.py` before import, split, benchmark, or snapshot suites that load committed pickles. +- Any `.pkl`/`.pickle` present in `snapshots/` but missing from `SHA256SUMS` stops the entire pytest session at startup (`tests/conftest.py`), and the message lists the unexpected and missing names. The `.gitignore` rule for `*.pickle` only keeps such files out of Git; it does not make them acceptable here. Store local snapshots elsewhere (e.g. `.tmp/`). - Changes to fixture trust records require `git diff --check` and review of `.gitattributes`/`.gitignore` when Git LFS or sensitive-data rules are affected. diff --git a/tests/test_fixture_provenance.py b/tests/test_fixture_provenance.py index 18e101c..44f44d2 100644 --- a/tests/test_fixture_provenance.py +++ b/tests/test_fixture_provenance.py @@ -1,5 +1,11 @@ +import hashlib +from pathlib import Path + +import pytest + from tests._fixture_provenance import ( PROVENANCE, + FixtureProvenanceError, load_manifest, verify_snapshot_fixtures, ) @@ -14,3 +20,89 @@ def test_fixture_provenance_covers_every_manifest_object() -> None: for filename, digest in load_manifest().items(): assert filename in text assert digest in text + + +def _write_fixture(directory: Path, name: str, payload: bytes) -> tuple[str, int]: + """Write a stand-in fixture and return its (sha256, size). + + The payload is arbitrary bytes, never a real pickle, so these tests keep the + guard's no-deserialization property. + """ + path = directory / name + path.write_bytes(payload) + return hashlib.sha256(payload).hexdigest(), len(payload) + + +def _write_manifest(directory: Path, entries: dict[str, str]) -> Path: + manifest = directory / "SHA256SUMS" + manifest.write_text("".join(f"{digest} {name}\n" for name, digest in entries.items())) + return manifest + + +def test_unexpected_local_pickle_is_named_with_guidance(tmp_path: Path) -> None: + """Regression for issue #118: the failure must say which file is unreviewed.""" + digest, size = _write_fixture(tmp_path, "reviewed.pkl", b"reviewed") + _write_fixture(tmp_path, "memory0.pickle", b"local capture") + _write_fixture(tmp_path, "notes.txt", b"not a pickle, ignored") + manifest = _write_manifest(tmp_path, {"reviewed.pkl": digest}) + + with pytest.raises(FixtureProvenanceError) as exc_info: + verify_snapshot_fixtures(tmp_path, manifest, {"reviewed.pkl": size}) + + message = str(exc_info.value) + assert f"does not match the reviewed manifest in {tmp_path}" in message + assert "unexpected (not in SHA256SUMS): memory0.pickle" in message + assert "missing (in SHA256SUMS but absent): " in message + assert "Move unreviewed pickles out of" in message + assert "notes.txt" not in message + + +def test_missing_reviewed_pickle_is_named(tmp_path: Path) -> None: + digest, size = _write_fixture(tmp_path, "present.pkl", b"present") + manifest = _write_manifest(tmp_path, {"present.pkl": digest, "absent.pkl": "0" * 64}) + + with pytest.raises(FixtureProvenanceError) as exc_info: + verify_snapshot_fixtures(tmp_path, manifest, {"present.pkl": size, "absent.pkl": 1}) + + message = str(exc_info.value) + assert "unexpected (not in SHA256SUMS): " in message + assert "missing (in SHA256SUMS but absent): absent.pkl" in message + + +def test_unexpected_names_are_sorted_and_complete(tmp_path: Path) -> None: + digest, size = _write_fixture(tmp_path, "reviewed.pkl", b"reviewed") + _write_fixture(tmp_path, "zeta.pickle", b"z") + _write_fixture(tmp_path, "alpha.PKL", b"a") + manifest = _write_manifest(tmp_path, {"reviewed.pkl": digest}) + + with pytest.raises( + FixtureProvenanceError, match=r"not in SHA256SUMS\): alpha\.PKL, zeta\.pickle" + ): + verify_snapshot_fixtures(tmp_path, manifest, {"reviewed.pkl": size}) + + +def test_manifest_and_size_table_disagreement_is_reported(tmp_path: Path) -> None: + digest, _ = _write_fixture(tmp_path, "reviewed.pkl", b"reviewed") + manifest = _write_manifest(tmp_path, {"reviewed.pkl": digest}) + + with pytest.raises(FixtureProvenanceError) as exc_info: + verify_snapshot_fixtures(tmp_path, manifest, {}) + + message = str(exc_info.value) + assert "SHA256SUMS and the reviewed size table disagree" in message + assert "only in SHA256SUMS: reviewed.pkl" in message + assert "only in size table: " in message + + +def test_checksum_and_size_mismatches_still_fail(tmp_path: Path) -> None: + digest, size = _write_fixture(tmp_path, "reviewed.pkl", b"reviewed") + + wrong_digest = _write_manifest(tmp_path, {"reviewed.pkl": "f" * 64}) + with pytest.raises(FixtureProvenanceError, match="Fixture checksum mismatch: reviewed.pkl"): + verify_snapshot_fixtures(tmp_path, wrong_digest, {"reviewed.pkl": size}) + + right_digest = _write_manifest(tmp_path, {"reviewed.pkl": digest}) + with pytest.raises(FixtureProvenanceError, match="Fixture size mismatch: reviewed.pkl"): + verify_snapshot_fixtures(tmp_path, right_digest, {"reviewed.pkl": size + 1}) + + verify_snapshot_fixtures(tmp_path, right_digest, {"reviewed.pkl": size})