diff --git a/tests/AGENTS.md b/tests/AGENTS.md index 92af0e1..a6845c6 100644 --- a/tests/AGENTS.md +++ b/tests/AGENTS.md @@ -55,6 +55,7 @@ - CLI tests exercise Typer commands through test runners and assert printed output. - Query tests build minimal SQLite schemas and YAML-like config objects around the template pipeline. - Snapshot/import tests may deserialize only objects accepted by `fixtures/snapshots/PROVENANCE.md` and `SHA256SUMS`; never load a new or changed pickle before review. +- `conftest.py` aborts the whole pytest session before collection when any `.pkl`/`.pickle` in `tests/fixtures/snapshots/` is not listed in `SHA256SUMS` (or a listed one is absent); the exit message names the offending files. Keep local analysis snapshots outside that directory, for example under `.tmp/`. There is deliberately no override flag. ## Dependencies diff --git a/tests/_fixture_provenance.py b/tests/_fixture_provenance.py index c84ca1a..86066b5 100644 --- a/tests/_fixture_provenance.py +++ b/tests/_fixture_provenance.py @@ -28,9 +28,9 @@ class FixtureProvenanceError(RuntimeError): pass -def load_manifest() -> dict[str, str]: +def load_manifest(manifest: Path = MANIFEST) -> dict[str, str]: entries: dict[str, str] = {} - for line in MANIFEST.read_text().splitlines(): + for line in manifest.read_text().splitlines(): digest, filename = line.split(" ", 1) if filename in entries: raise FixtureProvenanceError(f"Duplicate fixture manifest entry: {filename}") @@ -46,13 +46,40 @@ def _sha256(path: Path) -> str: return digest.hexdigest() -def verify_snapshot_fixtures() -> None: - expected = load_manifest() +def _format_names(names: list[str]) -> str: + return ", ".join(names) if names else "" + + +def _describe_set_mismatch(fixture_dir: Path, unexpected: list[str], missing: list[str]) -> str: + # Only file names are reported; the guard never reads pickle content. + return ( + f"Executable fixture set does not match the reviewed manifest in {fixture_dir}.\n" + f" unexpected (not in SHA256SUMS): {_format_names(unexpected)}\n" + f" missing (in SHA256SUMS but absent): {_format_names(missing)}\n" + f"Move unreviewed pickles out of {fixture_dir}/ (for example into .tmp/), or complete " + "the provenance review in PROVENANCE.md and update SHA256SUMS before running the suite." + ) + + +def verify_snapshot_fixtures( + fixture_dir: Path = FIXTURE_DIR, + manifest: Path = MANIFEST, + expected_sizes: dict[str, int] = EXPECTED_SIZES, +) -> None: + expected = load_manifest(manifest) fixtures = { - path.name: path for path in FIXTURE_DIR.iterdir() if path.suffix.lower() in PICKLE_SUFFIXES + path.name: path for path in fixture_dir.iterdir() if path.suffix.lower() in PICKLE_SUFFIXES } - if set(fixtures) != set(expected) or set(fixtures) != set(EXPECTED_SIZES): - raise FixtureProvenanceError("Executable fixture set does not match the reviewed manifest") + unexpected = sorted(set(fixtures) - set(expected)) + missing = sorted(set(expected) - set(fixtures)) + if unexpected or missing: + raise FixtureProvenanceError(_describe_set_mismatch(fixture_dir, unexpected, missing)) + if set(expected) != set(expected_sizes): + raise FixtureProvenanceError( + "SHA256SUMS and the reviewed size table disagree; update both together.\n" + f" only in SHA256SUMS: {_format_names(sorted(set(expected) - set(expected_sizes)))}\n" + f" only in size table: {_format_names(sorted(set(expected_sizes) - set(expected)))}" + ) for filename, path in fixtures.items(): pointer = LFS_POINTER.fullmatch(path.read_bytes()) if path.stat().st_size < 1024 else None @@ -60,5 +87,5 @@ def verify_snapshot_fixtures() -> None: actual_size = int(pointer.group("size")) if pointer else path.stat().st_size if actual_digest != expected[filename]: raise FixtureProvenanceError(f"Fixture checksum mismatch: {filename}") - if actual_size != EXPECTED_SIZES[filename]: + if actual_size != expected_sizes[filename]: raise FixtureProvenanceError(f"Fixture size mismatch: {filename}") diff --git a/tests/fixtures/AGENTS.md b/tests/fixtures/AGENTS.md index ba764ea..dbc5d03 100644 --- a/tests/fixtures/AGENTS.md +++ b/tests/fixtures/AGENTS.md @@ -17,4 +17,5 @@ they are tracked by Git. ## Verification - Run `pytest tests/test_fixture_provenance.py` before import, split, benchmark, or snapshot suites that load committed pickles. +- Any `.pkl`/`.pickle` present in `snapshots/` but missing from `SHA256SUMS` stops the entire pytest session at startup (`tests/conftest.py`), and the message lists the unexpected and missing names. The `.gitignore` rule for `*.pickle` only keeps such files out of Git; it does not make them acceptable here. Store local snapshots elsewhere (e.g. `.tmp/`). - Changes to fixture trust records require `git diff --check` and review of `.gitattributes`/`.gitignore` when Git LFS or sensitive-data rules are affected. diff --git a/tests/test_fixture_provenance.py b/tests/test_fixture_provenance.py index 18e101c..44f44d2 100644 --- a/tests/test_fixture_provenance.py +++ b/tests/test_fixture_provenance.py @@ -1,5 +1,11 @@ +import hashlib +from pathlib import Path + +import pytest + from tests._fixture_provenance import ( PROVENANCE, + FixtureProvenanceError, load_manifest, verify_snapshot_fixtures, ) @@ -14,3 +20,89 @@ def test_fixture_provenance_covers_every_manifest_object() -> None: for filename, digest in load_manifest().items(): assert filename in text assert digest in text + + +def _write_fixture(directory: Path, name: str, payload: bytes) -> tuple[str, int]: + """Write a stand-in fixture and return its (sha256, size). + + The payload is arbitrary bytes, never a real pickle, so these tests keep the + guard's no-deserialization property. + """ + path = directory / name + path.write_bytes(payload) + return hashlib.sha256(payload).hexdigest(), len(payload) + + +def _write_manifest(directory: Path, entries: dict[str, str]) -> Path: + manifest = directory / "SHA256SUMS" + manifest.write_text("".join(f"{digest} {name}\n" for name, digest in entries.items())) + return manifest + + +def test_unexpected_local_pickle_is_named_with_guidance(tmp_path: Path) -> None: + """Regression for issue #118: the failure must say which file is unreviewed.""" + digest, size = _write_fixture(tmp_path, "reviewed.pkl", b"reviewed") + _write_fixture(tmp_path, "memory0.pickle", b"local capture") + _write_fixture(tmp_path, "notes.txt", b"not a pickle, ignored") + manifest = _write_manifest(tmp_path, {"reviewed.pkl": digest}) + + with pytest.raises(FixtureProvenanceError) as exc_info: + verify_snapshot_fixtures(tmp_path, manifest, {"reviewed.pkl": size}) + + message = str(exc_info.value) + assert f"does not match the reviewed manifest in {tmp_path}" in message + assert "unexpected (not in SHA256SUMS): memory0.pickle" in message + assert "missing (in SHA256SUMS but absent): " in message + assert "Move unreviewed pickles out of" in message + assert "notes.txt" not in message + + +def test_missing_reviewed_pickle_is_named(tmp_path: Path) -> None: + digest, size = _write_fixture(tmp_path, "present.pkl", b"present") + manifest = _write_manifest(tmp_path, {"present.pkl": digest, "absent.pkl": "0" * 64}) + + with pytest.raises(FixtureProvenanceError) as exc_info: + verify_snapshot_fixtures(tmp_path, manifest, {"present.pkl": size, "absent.pkl": 1}) + + message = str(exc_info.value) + assert "unexpected (not in SHA256SUMS): " in message + assert "missing (in SHA256SUMS but absent): absent.pkl" in message + + +def test_unexpected_names_are_sorted_and_complete(tmp_path: Path) -> None: + digest, size = _write_fixture(tmp_path, "reviewed.pkl", b"reviewed") + _write_fixture(tmp_path, "zeta.pickle", b"z") + _write_fixture(tmp_path, "alpha.PKL", b"a") + manifest = _write_manifest(tmp_path, {"reviewed.pkl": digest}) + + with pytest.raises( + FixtureProvenanceError, match=r"not in SHA256SUMS\): alpha\.PKL, zeta\.pickle" + ): + verify_snapshot_fixtures(tmp_path, manifest, {"reviewed.pkl": size}) + + +def test_manifest_and_size_table_disagreement_is_reported(tmp_path: Path) -> None: + digest, _ = _write_fixture(tmp_path, "reviewed.pkl", b"reviewed") + manifest = _write_manifest(tmp_path, {"reviewed.pkl": digest}) + + with pytest.raises(FixtureProvenanceError) as exc_info: + verify_snapshot_fixtures(tmp_path, manifest, {}) + + message = str(exc_info.value) + assert "SHA256SUMS and the reviewed size table disagree" in message + assert "only in SHA256SUMS: reviewed.pkl" in message + assert "only in size table: " in message + + +def test_checksum_and_size_mismatches_still_fail(tmp_path: Path) -> None: + digest, size = _write_fixture(tmp_path, "reviewed.pkl", b"reviewed") + + wrong_digest = _write_manifest(tmp_path, {"reviewed.pkl": "f" * 64}) + with pytest.raises(FixtureProvenanceError, match="Fixture checksum mismatch: reviewed.pkl"): + verify_snapshot_fixtures(tmp_path, wrong_digest, {"reviewed.pkl": size}) + + right_digest = _write_manifest(tmp_path, {"reviewed.pkl": digest}) + with pytest.raises(FixtureProvenanceError, match="Fixture size mismatch: reviewed.pkl"): + verify_snapshot_fixtures(tmp_path, right_digest, {"reviewed.pkl": size + 1}) + + verify_snapshot_fixtures(tmp_path, right_digest, {"reviewed.pkl": size})