Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions tests/AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@
- CLI tests exercise Typer commands through test runners and assert printed output.
- Query tests build minimal SQLite schemas and YAML-like config objects around the template pipeline.
- Snapshot/import tests may deserialize only objects accepted by `fixtures/snapshots/PROVENANCE.md` and `SHA256SUMS`; never load a new or changed pickle before review.
- `conftest.py` aborts the whole pytest session before collection when any `.pkl`/`.pickle` in `tests/fixtures/snapshots/` is not listed in `SHA256SUMS` (or a listed one is absent); the exit message names the offending files. Keep local analysis snapshots outside that directory, for example under `.tmp/`. There is deliberately no override flag.

## Dependencies

Expand Down
43 changes: 35 additions & 8 deletions tests/_fixture_provenance.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,9 +28,9 @@ class FixtureProvenanceError(RuntimeError):
pass


def load_manifest() -> dict[str, str]:
def load_manifest(manifest: Path = MANIFEST) -> dict[str, str]:
entries: dict[str, str] = {}
for line in MANIFEST.read_text().splitlines():
for line in manifest.read_text().splitlines():
digest, filename = line.split(" ", 1)
if filename in entries:
raise FixtureProvenanceError(f"Duplicate fixture manifest entry: {filename}")
Expand All @@ -46,19 +46,46 @@ def _sha256(path: Path) -> str:
return digest.hexdigest()


def verify_snapshot_fixtures() -> None:
expected = load_manifest()
def _format_names(names: list[str]) -> str:
return ", ".join(names) if names else "<none>"


def _describe_set_mismatch(fixture_dir: Path, unexpected: list[str], missing: list[str]) -> str:
# Only file names are reported; the guard never reads pickle content.
return (
f"Executable fixture set does not match the reviewed manifest in {fixture_dir}.\n"
f" unexpected (not in SHA256SUMS): {_format_names(unexpected)}\n"
f" missing (in SHA256SUMS but absent): {_format_names(missing)}\n"
f"Move unreviewed pickles out of {fixture_dir}/ (for example into .tmp/), or complete "
"the provenance review in PROVENANCE.md and update SHA256SUMS before running the suite."
)


def verify_snapshot_fixtures(
fixture_dir: Path = FIXTURE_DIR,
manifest: Path = MANIFEST,
expected_sizes: dict[str, int] = EXPECTED_SIZES,
) -> None:
expected = load_manifest(manifest)
fixtures = {
path.name: path for path in FIXTURE_DIR.iterdir() if path.suffix.lower() in PICKLE_SUFFIXES
path.name: path for path in fixture_dir.iterdir() if path.suffix.lower() in PICKLE_SUFFIXES
}
if set(fixtures) != set(expected) or set(fixtures) != set(EXPECTED_SIZES):
raise FixtureProvenanceError("Executable fixture set does not match the reviewed manifest")
unexpected = sorted(set(fixtures) - set(expected))
missing = sorted(set(expected) - set(fixtures))
if unexpected or missing:
raise FixtureProvenanceError(_describe_set_mismatch(fixture_dir, unexpected, missing))
if set(expected) != set(expected_sizes):
raise FixtureProvenanceError(
"SHA256SUMS and the reviewed size table disagree; update both together.\n"
f" only in SHA256SUMS: {_format_names(sorted(set(expected) - set(expected_sizes)))}\n"
f" only in size table: {_format_names(sorted(set(expected_sizes) - set(expected)))}"
)

for filename, path in fixtures.items():
pointer = LFS_POINTER.fullmatch(path.read_bytes()) if path.stat().st_size < 1024 else None
actual_digest = pointer.group("digest").decode() if pointer else _sha256(path)
actual_size = int(pointer.group("size")) if pointer else path.stat().st_size
if actual_digest != expected[filename]:
raise FixtureProvenanceError(f"Fixture checksum mismatch: {filename}")
if actual_size != EXPECTED_SIZES[filename]:
if actual_size != expected_sizes[filename]:
raise FixtureProvenanceError(f"Fixture size mismatch: {filename}")
1 change: 1 addition & 0 deletions tests/fixtures/AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -17,4 +17,5 @@ they are tracked by Git.

## Verification
- Run `pytest tests/test_fixture_provenance.py` before import, split, benchmark, or snapshot suites that load committed pickles.
- Any `.pkl`/`.pickle` present in `snapshots/` but missing from `SHA256SUMS` stops the entire pytest session at startup (`tests/conftest.py`), and the message lists the unexpected and missing names. The `.gitignore` rule for `*.pickle` only keeps such files out of Git; it does not make them acceptable here. Store local snapshots elsewhere (e.g. `.tmp/`).
- Changes to fixture trust records require `git diff --check` and review of `.gitattributes`/`.gitignore` when Git LFS or sensitive-data rules are affected.
92 changes: 92 additions & 0 deletions tests/test_fixture_provenance.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,11 @@
import hashlib
from pathlib import Path

import pytest

from tests._fixture_provenance import (
PROVENANCE,
FixtureProvenanceError,
load_manifest,
verify_snapshot_fixtures,
)
Expand All @@ -14,3 +20,89 @@ def test_fixture_provenance_covers_every_manifest_object() -> None:
for filename, digest in load_manifest().items():
assert filename in text
assert digest in text


def _write_fixture(directory: Path, name: str, payload: bytes) -> tuple[str, int]:
"""Write a stand-in fixture and return its (sha256, size).

The payload is arbitrary bytes, never a real pickle, so these tests keep the
guard's no-deserialization property.
"""
path = directory / name
path.write_bytes(payload)
return hashlib.sha256(payload).hexdigest(), len(payload)


def _write_manifest(directory: Path, entries: dict[str, str]) -> Path:
manifest = directory / "SHA256SUMS"
manifest.write_text("".join(f"{digest} {name}\n" for name, digest in entries.items()))
return manifest


def test_unexpected_local_pickle_is_named_with_guidance(tmp_path: Path) -> None:
"""Regression for issue #118: the failure must say which file is unreviewed."""
digest, size = _write_fixture(tmp_path, "reviewed.pkl", b"reviewed")
_write_fixture(tmp_path, "memory0.pickle", b"local capture")
_write_fixture(tmp_path, "notes.txt", b"not a pickle, ignored")
manifest = _write_manifest(tmp_path, {"reviewed.pkl": digest})

with pytest.raises(FixtureProvenanceError) as exc_info:
verify_snapshot_fixtures(tmp_path, manifest, {"reviewed.pkl": size})

message = str(exc_info.value)
assert f"does not match the reviewed manifest in {tmp_path}" in message
assert "unexpected (not in SHA256SUMS): memory0.pickle" in message
assert "missing (in SHA256SUMS but absent): <none>" in message
assert "Move unreviewed pickles out of" in message
assert "notes.txt" not in message


def test_missing_reviewed_pickle_is_named(tmp_path: Path) -> None:
digest, size = _write_fixture(tmp_path, "present.pkl", b"present")
manifest = _write_manifest(tmp_path, {"present.pkl": digest, "absent.pkl": "0" * 64})

with pytest.raises(FixtureProvenanceError) as exc_info:
verify_snapshot_fixtures(tmp_path, manifest, {"present.pkl": size, "absent.pkl": 1})

message = str(exc_info.value)
assert "unexpected (not in SHA256SUMS): <none>" in message
assert "missing (in SHA256SUMS but absent): absent.pkl" in message


def test_unexpected_names_are_sorted_and_complete(tmp_path: Path) -> None:
digest, size = _write_fixture(tmp_path, "reviewed.pkl", b"reviewed")
_write_fixture(tmp_path, "zeta.pickle", b"z")
_write_fixture(tmp_path, "alpha.PKL", b"a")
manifest = _write_manifest(tmp_path, {"reviewed.pkl": digest})

with pytest.raises(
FixtureProvenanceError, match=r"not in SHA256SUMS\): alpha\.PKL, zeta\.pickle"
):
verify_snapshot_fixtures(tmp_path, manifest, {"reviewed.pkl": size})


def test_manifest_and_size_table_disagreement_is_reported(tmp_path: Path) -> None:
digest, _ = _write_fixture(tmp_path, "reviewed.pkl", b"reviewed")
manifest = _write_manifest(tmp_path, {"reviewed.pkl": digest})

with pytest.raises(FixtureProvenanceError) as exc_info:
verify_snapshot_fixtures(tmp_path, manifest, {})

message = str(exc_info.value)
assert "SHA256SUMS and the reviewed size table disagree" in message
assert "only in SHA256SUMS: reviewed.pkl" in message
assert "only in size table: <none>" in message


def test_checksum_and_size_mismatches_still_fail(tmp_path: Path) -> None:
digest, size = _write_fixture(tmp_path, "reviewed.pkl", b"reviewed")

wrong_digest = _write_manifest(tmp_path, {"reviewed.pkl": "f" * 64})
with pytest.raises(FixtureProvenanceError, match="Fixture checksum mismatch: reviewed.pkl"):
verify_snapshot_fixtures(tmp_path, wrong_digest, {"reviewed.pkl": size})

right_digest = _write_manifest(tmp_path, {"reviewed.pkl": digest})
with pytest.raises(FixtureProvenanceError, match="Fixture size mismatch: reviewed.pkl"):
verify_snapshot_fixtures(tmp_path, right_digest, {"reviewed.pkl": size + 1})

verify_snapshot_fixtures(tmp_path, right_digest, {"reviewed.pkl": size})
Loading