diff --git a/README.md b/README.md index 880baca..7961666 100644 --- a/README.md +++ b/README.md @@ -177,6 +177,37 @@ Run the dedicated pivot capability artifact with: uv run excelbench cross-language-pivot-context --fixture fixtures/excel/tier2/15_pivot_tables.xlsx --output results-cross-language-pivots ``` + +## Exact Evidence Manifests + +A benchmark directory can be bound to its exact source and artifact identities with +a deterministic, path-free manifest: + +```bash +uv run excelbench evidence-manifest \ + --root results-release-2026-08-31 \ + --snapshot-id wolfxl-2.1-linux-x86_64 \ + --source-sha 0123456789abcdef0123456789abcdef01234567 \ + --observed-at 2026-08-31T00:00:00Z \ + --subject wolfxl-wheel@2.1.0= + +uv run excelbench verify-evidence \ + --root results-release-2026-08-31 \ + --expected-source-sha 0123456789abcdef0123456789abcdef01234567 +``` + +The v1 contract inventories every regular file, hashes a canonical sorted file set, +rejects symlinks and cross-platform path collisions, and refuses undeclared, missing, +or changed files. It excludes only the manifest itself. The observation timestamp is +explicit so identical inputs produce identical manifest bytes. + +The manifest is the subject to sign or attest in release CI. Successful verification +does not make an evidence lane current by itself: public claims must still name the +snapshot date, source commit, tested package subjects, platform, and workload. + +Schema: [\`schemas/evidence-manifest-v1.schema.json\`](schemas/evidence-manifest-v1.schema.json) + + ## How It Works 1. **Generate reference files** -- [xlwings](https://www.xlwings.org/) drives real Excel to produce canonical `.xlsx`/`.xls` test files with known features. diff --git a/schemas/evidence-manifest-v1.schema.json b/schemas/evidence-manifest-v1.schema.json new file mode 100644 index 0000000..3db76a5 --- /dev/null +++ b/schemas/evidence-manifest-v1.schema.json @@ -0,0 +1,118 @@ +{ + "$id": "https://excelbench.dev/schemas/evidence-manifest/v1", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "artifact_count": { + "minimum": 1, + "type": "integer" + }, + "artifact_set_sha256": { + "pattern": "^[0-9a-f]{64}$", + "type": "string" + }, + "artifacts": { + "items": { + "additionalProperties": false, + "properties": { + "path": { + "minLength": 1, + "type": "string" + }, + "sha256": { + "pattern": "^[0-9a-f]{64}$", + "type": "string" + }, + "size_bytes": { + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "path", + "sha256", + "size_bytes" + ], + "type": "object" + }, + "minItems": 1, + "type": "array" + }, + "observed_at": { + "format": "date-time", + "pattern": "Z$", + "type": "string" + }, + "schema": { + "const": "https://excelbench.dev/schemas/evidence-manifest/v1" + }, + "schema_version": { + "const": 1 + }, + "snapshot_id": { + "minLength": 1, + "type": "string" + }, + "source": { + "additionalProperties": false, + "properties": { + "commit": { + "pattern": "^[0-9a-f]{40}$", + "type": "string" + }, + "repository": { + "minLength": 1, + "type": "string" + } + }, + "required": [ + "repository", + "commit" + ], + "type": "object" + }, + "subjects": { + "items": { + "additionalProperties": false, + "properties": { + "name": { + "minLength": 1, + "type": "string" + }, + "sha256": { + "pattern": "^[0-9a-f]{64}$", + "type": "string" + }, + "version": { + "minLength": 1, + "type": "string" + } + }, + "required": [ + "name", + "sha256" + ], + "type": "object" + }, + "type": "array" + }, + "total_size_bytes": { + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "schema", + "schema_version", + "snapshot_id", + "observed_at", + "source", + "subjects", + "artifacts", + "artifact_count", + "total_size_bytes", + "artifact_set_sha256" + ], + "title": "ExcelBench evidence manifest v1", + "type": "object" +} diff --git a/src/excelbench/cli.py b/src/excelbench/cli.py index 4c8292b..160e287 100644 --- a/src/excelbench/cli.py +++ b/src/excelbench/cli.py @@ -2001,5 +2001,120 @@ def show_summary(results: "BenchmarkResults") -> None: console.print(table) +@app.command("evidence-manifest") +def evidence_manifest( + root: Path = typer.Option( + Path("results"), + "--root", + help="Evidence directory to inventory exactly.", + ), + snapshot_id: str = typer.Option( + ..., + "--snapshot-id", + help="Stable identity for this evidence snapshot.", + ), + source_sha: str = typer.Option( + ..., + "--source-sha", + help="Exact 40-character source commit represented by the evidence.", + ), + observed_at: str = typer.Option( + ..., + "--observed-at", + help="Explicit timezone-aware observation timestamp.", + ), + repository: str = typer.Option( + "SynthGL/ExcelBench", + "--repository", + help="Source repository identity.", + ), + subject: list[str] | None = typer.Option( + None, + "--subject", + help="Bound artifact as NAME=SHA256 or NAME@VERSION=SHA256; repeatable.", + ), + manifest_name: str = typer.Option( + "excelbench-evidence.json", + "--manifest-name", + help="Manifest filename created directly below the evidence root.", + ), + replace: bool = typer.Option( + False, + "--replace", + help="Atomically replace an existing manifest.", + ), +) -> None: + """Create a deterministic exact-file evidence manifest.""" + from excelbench.evidence_manifest import ( + EvidenceManifestError, + build_evidence_manifest, + parse_subject, + verify_evidence_manifest, + write_evidence_manifest, + ) + + try: + subjects = [parse_subject(value) for value in subject or []] + manifest = build_evidence_manifest( + root, + snapshot_id=snapshot_id, + repository=repository, + source_sha=source_sha, + observed_at=observed_at, + subjects=subjects, + manifest_name=manifest_name, + ) + output = root.resolve() / manifest_name + write_evidence_manifest(output, manifest, replace=replace) + verify_evidence_manifest(root, manifest, manifest_name=manifest_name) + except (EvidenceManifestError, OSError) as exc: + console.print(f"[red]Evidence manifest refused: {exc}[/red]") + raise typer.Exit(1) from exc + console.print(f"[green]✓ Evidence manifest: {output}[/green]") + console.print(f" Artifacts: {manifest['artifact_count']}") + console.print(f" Artifact set: {manifest['artifact_set_sha256']}") + + +@app.command("verify-evidence") +def verify_evidence( + root: Path = typer.Option( + Path("results"), + "--root", + help="Evidence directory covered by the manifest.", + ), + manifest_name: str = typer.Option( + "excelbench-evidence.json", + "--manifest-name", + help="Manifest filename directly below the evidence root.", + ), + expected_source_sha: str | None = typer.Option( + None, + "--expected-source-sha", + help="Optional exact source commit required by the caller.", + ), +) -> None: + """Fail unless a manifest exactly covers the current evidence directory.""" + from excelbench.evidence_manifest import ( + EvidenceManifestError, + read_evidence_manifest, + verify_evidence_manifest, + ) + + path = root.resolve() / manifest_name + try: + manifest = read_evidence_manifest(path) + verify_evidence_manifest( + root, + manifest, + expected_source_sha=expected_source_sha, + manifest_name=manifest_name, + ) + except (EvidenceManifestError, OSError) as exc: + console.print(f"[red]Evidence verification failed: {exc}[/red]") + raise typer.Exit(1) from exc + console.print(f"[green]✓ Evidence verified: {path}[/green]") + console.print(f" Artifact set: {manifest['artifact_set_sha256']}") + + if __name__ == "__main__": app() diff --git a/src/excelbench/evidence_manifest.py b/src/excelbench/evidence_manifest.py new file mode 100644 index 0000000..8fd3dd3 --- /dev/null +++ b/src/excelbench/evidence_manifest.py @@ -0,0 +1,745 @@ +"""Deterministic, exact-file manifests for benchmark evidence snapshots.""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +import stat +import tempfile +import unicodedata +from collections.abc import Iterable, Mapping, Sequence +from dataclasses import dataclass +from datetime import UTC, datetime +from pathlib import Path, PurePosixPath +from typing import Any + +SCHEMA_VERSION = 1 +DEFAULT_MANIFEST_NAME = "excelbench-evidence.json" +MAX_FILES = 10_000 +MAX_FILE_BYTES = 512 * 1024 * 1024 +MAX_TOTAL_BYTES = 2 * 1024 * 1024 * 1024 +MAX_MANIFEST_BYTES = 4 * 1024 * 1024 +_SHA256_RE = re.compile(r"^[0-9a-f]{64}$") +_GIT_SHA_RE = re.compile(r"^[0-9a-f]{40}$") +_WINDOWS_RESERVED_BASENAMES = { + "aux", + "clock$", + "con", + "conin$", + "conout$", + "nul", + "prn", + *(f"com{suffix}" for suffix in (*range(1, 10), "¹", "²", "³")), + *(f"lpt{suffix}" for suffix in (*range(1, 10), "¹", "²", "³")), +} +_WINDOWS_FORBIDDEN_CHARACTERS = frozenset('<>:"|?*') + + +class EvidenceManifestError(ValueError): + """The evidence snapshot or manifest violates its fail-closed contract.""" + + +@dataclass(frozen=True, order=True) +class EvidenceSubject: + """An external artifact or source identity bound into a snapshot.""" + + name: str + sha256: str + version: str | None = None + + def to_dict(self) -> dict[str, str]: + value = {"name": self.name, "sha256": self.sha256} + if self.version is not None: + value["version"] = self.version + return value + + +@dataclass(frozen=True, order=True) +class EvidenceArtifact: + """One immutable regular file in an evidence snapshot.""" + + path: str + sha256: str + size_bytes: int + + def to_dict(self) -> dict[str, str | int]: + return { + "path": self.path, + "sha256": self.sha256, + "size_bytes": self.size_bytes, + } + + +def canonical_json_bytes(value: object) -> bytes: + """Return the one wire representation used for hashing and publication.""" + try: + return json.dumps( + value, + ensure_ascii=False, + allow_nan=False, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8", errors="strict") + except UnicodeEncodeError as exc: + raise EvidenceManifestError( + "manifest contains text that is not valid UTF-8" + ) from exc + + +def _bounded_manifest_bytes(value: object) -> bytes: + contents = canonical_json_bytes(value) + b"\n" + if len(contents) > MAX_MANIFEST_BYTES: + raise EvidenceManifestError("manifest exceeds the 4 MiB size limit") + return contents + + +def build_evidence_manifest( + root: Path, + *, + snapshot_id: str, + repository: str, + source_sha: str, + observed_at: str, + subjects: Sequence[EvidenceSubject] = (), + manifest_name: str = DEFAULT_MANIFEST_NAME, +) -> dict[str, Any]: + """Inventory ``root`` exactly and return a path-free deterministic manifest. + + ``observed_at`` is explicit rather than read from the clock so rerunning with + identical inputs produces identical bytes. The output manifest itself is + excluded when it lives directly below ``root``. + """ + root = root.resolve(strict=True) + if not root.is_dir(): + raise EvidenceManifestError("evidence root must be a directory") + snapshot_id = _required_text(snapshot_id, "snapshot_id") + repository = _required_text(repository, "repository") + source_sha = source_sha.strip().lower() + if _GIT_SHA_RE.fullmatch(source_sha) is None: + raise EvidenceManifestError("source_sha must be a full lowercase 40-character Git SHA") + observed_at = _canonical_utc_timestamp(observed_at) + manifest_name = _safe_manifest_name(manifest_name) + _read_existing_manifest_destination(root / manifest_name) + normalized_subjects = _validate_subjects(subjects) + artifacts = _inventory(root, excluded_root_name=manifest_name) + artifact_dicts = [artifact.to_dict() for artifact in artifacts] + artifact_set_sha256 = hashlib.sha256(canonical_json_bytes(artifact_dicts)).hexdigest() + manifest = { + "schema": "https://excelbench.dev/schemas/evidence-manifest/v1", + "schema_version": SCHEMA_VERSION, + "snapshot_id": snapshot_id, + "observed_at": observed_at, + "source": {"repository": repository, "commit": source_sha}, + "subjects": [subject.to_dict() for subject in normalized_subjects], + "artifacts": artifact_dicts, + "artifact_count": len(artifacts), + "total_size_bytes": sum(artifact.size_bytes for artifact in artifacts), + "artifact_set_sha256": artifact_set_sha256, + } + _bounded_manifest_bytes(manifest) + return manifest + + +def verify_evidence_manifest( + root: Path, + manifest: Mapping[str, Any], + *, + expected_source_sha: str | None = None, + manifest_name: str = DEFAULT_MANIFEST_NAME, +) -> None: + """Fail unless ``manifest`` exactly describes every regular file in ``root``.""" + manifest_name = _safe_manifest_name(manifest_name) + expected = _validate_manifest_document( + manifest, + reserved_manifest_name=manifest_name, + ) + source = _mapping(manifest["source"], "source") + source_sha = _string(source.get("commit"), "source.commit") + if expected_source_sha is not None and source_sha != expected_source_sha: + raise EvidenceManifestError("manifest source commit does not match expected source SHA") + root = root.resolve(strict=True) + destination_manifest = _read_existing_manifest_destination(root / manifest_name) + if ( + destination_manifest is not None + and canonical_json_bytes(destination_manifest) != canonical_json_bytes(dict(manifest)) + ): + raise EvidenceManifestError("manifest file does not match the manifest being verified") + actual = _inventory( + root, + excluded_root_name=manifest_name, + require_artifact=False, + ) + final_destination_manifest = _read_existing_manifest_destination(root / manifest_name) + if (destination_manifest is None) != (final_destination_manifest is None) or ( + final_destination_manifest is not None + and canonical_json_bytes(final_destination_manifest) + != canonical_json_bytes(dict(manifest)) + ): + raise EvidenceManifestError("manifest file changed while verifying evidence") + if actual != expected: + actual_by_path = {artifact.path: artifact for artifact in actual} + expected_by_path = {artifact.path: artifact for artifact in expected} + missing = sorted(expected_by_path.keys() - actual_by_path.keys()) + extra = sorted(actual_by_path.keys() - expected_by_path.keys()) + changed = sorted( + path + for path in actual_by_path.keys() & expected_by_path.keys() + if actual_by_path[path] != expected_by_path[path] + ) + raise EvidenceManifestError( + f"evidence mismatch: missing={missing!r}, extra={extra!r}, changed={changed!r}" + ) + + +def read_evidence_manifest(path: Path) -> dict[str, Any]: + """Read a bounded UTF-8 manifest and reject duplicate JSON keys.""" + contents = _read_manifest_bytes(path) + + def no_duplicates(pairs: Iterable[tuple[str, Any]]) -> dict[str, Any]: + value: dict[str, Any] = {} + for key, item in pairs: + if key in value: + raise EvidenceManifestError(f"manifest contains duplicate key {key!r}") + value[key] = item + return value + + try: + decoded = json.loads( + contents.decode("utf-8", errors="strict"), + object_pairs_hook=no_duplicates, + ) + except EvidenceManifestError: + raise + except (RecursionError, UnicodeDecodeError, ValueError) as exc: + raise EvidenceManifestError("manifest is not valid UTF-8 JSON") from exc + if not isinstance(decoded, dict): + raise EvidenceManifestError("manifest root must be an object") + _validate_serialized_text(decoded) + return decoded + + +def _read_manifest_bytes(path: Path) -> bytes: + """Read one stable regular file through a bounded no-follow descriptor.""" + flags = os.O_RDONLY | getattr(os, "O_BINARY", 0) + flags |= getattr(os, "O_NOFOLLOW", 0) + try: + descriptor = os.open(path, flags) + except OSError as exc: + raise EvidenceManifestError( + "manifest must be a regular non-symlink file" + ) from exc + + try: + before = os.fstat(descriptor) + if not stat.S_ISREG(before.st_mode): + raise EvidenceManifestError( + "manifest must be a regular non-symlink file" + ) + if before.st_size > MAX_MANIFEST_BYTES: + raise EvidenceManifestError("manifest exceeds the 4 MiB size limit") + chunks: list[bytes] = [] + remaining = MAX_MANIFEST_BYTES + 1 + while remaining: + chunk = os.read(descriptor, min(1024 * 1024, remaining)) + if not chunk: + break + chunks.append(chunk) + remaining -= len(chunk) + contents = b"".join(chunks) + after = os.fstat(descriptor) + except EvidenceManifestError: + raise + except OSError as exc: + raise EvidenceManifestError("manifest changed while reading") from exc + finally: + os.close(descriptor) + + if len(contents) > MAX_MANIFEST_BYTES: + raise EvidenceManifestError("manifest exceeds the 4 MiB size limit") + try: + current = os.stat(path, follow_symlinks=False) + except OSError as exc: + raise EvidenceManifestError("manifest changed while reading") from exc + if ( + stat.S_ISLNK(current.st_mode) + or not stat.S_ISREG(current.st_mode) + or len(contents) != before.st_size + or _file_signature(before) != _file_signature(after) + or _file_signature(after) != _file_signature(current) + ): + raise EvidenceManifestError("manifest changed while reading") + return contents + + +def write_evidence_manifest( + path: Path, manifest: Mapping[str, Any], *, replace: bool = False +) -> None: + """Publish one canonical manifest without exposing a partially written file.""" + if path.is_symlink(): + raise EvidenceManifestError("manifest destination must not be a symlink") + name = _safe_manifest_name(path.name) + path.parent.mkdir(parents=True, exist_ok=True) + path = path.parent.resolve(strict=True) / name + if path.is_symlink(): + raise EvidenceManifestError("manifest destination must not be a symlink") + if replace: + _read_existing_manifest_destination(path) + _validate_manifest_document(manifest, reserved_manifest_name=name) + contents = _bounded_manifest_bytes(dict(manifest)) + descriptor, temporary_name = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent) + temporary = Path(temporary_name) + try: + with os.fdopen(descriptor, "wb") as output: + output.write(contents) + output.flush() + os.fsync(output.fileno()) + if replace: + _read_existing_manifest_destination(path) + os.replace(temporary, path) + else: + try: + os.link(temporary, path) + except FileExistsError as exc: + raise EvidenceManifestError("manifest already exists; pass replace=True") from exc + temporary.unlink() + if os.name != "nt": + directory = os.open(path.parent, os.O_RDONLY) + try: + os.fsync(directory) + finally: + os.close(directory) + finally: + temporary.unlink(missing_ok=True) + + +def parse_subject(value: str) -> EvidenceSubject: + """Parse ``NAME=SHA256`` or ``NAME@VERSION=SHA256`` CLI syntax.""" + identity, separator, digest = value.partition("=") + if not separator: + raise EvidenceManifestError("subject must use NAME=SHA256 or NAME@VERSION=SHA256") + name, version_separator, version = identity.partition("@") + return EvidenceSubject( + name=_required_text(name, "subject.name"), + version=_required_text(version, "subject.version") if version_separator else None, + sha256=digest.strip().lower(), + ) + + +def _inventory( + root: Path, *, excluded_root_name: str, require_artifact: bool = True +) -> list[EvidenceArtifact]: + artifacts: list[EvidenceArtifact] = [] + entries = _inventory_entries(root, excluded_root_name=excluded_root_name) + hashed_signatures: dict[str, tuple[int, int, int, int, int]] = {} + total_size = 0 + for relative, path, initial_signature in entries: + digest, size, signature = _sha256_stable_file(path) + if signature != initial_signature: + raise EvidenceManifestError( + f"evidence file changed after the initial inventory: {path}" + ) + hashed_signatures[relative] = signature + total_size += size + if total_size > MAX_TOTAL_BYTES: + raise EvidenceManifestError("evidence snapshot exceeds total size limit") + artifacts.append(EvidenceArtifact(path=relative, sha256=digest, size_bytes=size)) + if len(artifacts) > MAX_FILES: + raise EvidenceManifestError("evidence snapshot exceeds file-count limit") + + # Hashing can be long-running. Re-scan the complete namespace so a producer + # cannot add, remove, replace, or mutate an artifact after the first walk + # and still receive a successful exact-snapshot manifest. + final_entries = _inventory_entries(root, excluded_root_name=excluded_root_name) + final_signatures = { + relative: signature for relative, _path, signature in final_entries + } + if final_signatures != hashed_signatures: + raise EvidenceManifestError( + "evidence file set or metadata changed while inventorying" + ) + if require_artifact and not artifacts: + raise EvidenceManifestError("evidence snapshot must contain at least one artifact") + return artifacts + + +def _inventory_entries( + root: Path, *, excluded_root_name: str +) -> list[tuple[str, Path, tuple[int, int, int, int, int]]]: + entries: list[tuple[str, Path, tuple[int, int, int, int, int]]] = [] + # The manifest is excluded from its own inventory, but its portable name is + # still reserved so a differently-cased artifact cannot alias it on Windows. + seen_casefolded: set[str] = {_portable_path_key(excluded_root_name)} + for path in sorted(root.rglob("*"), key=lambda item: item.as_posix()): + if path.parent == root and path.name == excluded_root_name: + continue + try: + metadata = os.stat(path, follow_symlinks=False) + except OSError as exc: + raise EvidenceManifestError( + f"evidence entry changed while inventorying: {path.relative_to(root)}" + ) from exc + if stat.S_ISLNK(metadata.st_mode): + raise EvidenceManifestError( + f"symlinks are not allowed in evidence: {path.relative_to(root)}" + ) + if stat.S_ISDIR(metadata.st_mode): + continue + if not stat.S_ISREG(metadata.st_mode): + raise EvidenceManifestError(f"non-regular evidence entry: {path.relative_to(root)}") + relative = _canonical_relative_path(path.relative_to(root)) + portable_key = _portable_path_key(relative) + if portable_key in seen_casefolded: + raise EvidenceManifestError(f"case-insensitive evidence path collision: {relative}") + seen_casefolded.add(portable_key) + entries.append((relative, path, _file_signature(metadata))) + if len(entries) > MAX_FILES: + raise EvidenceManifestError("evidence snapshot exceeds file-count limit") + return entries + + +def _canonical_relative_path(path: Path) -> str: + raw = path.as_posix() + try: + raw.encode("utf-8", errors="strict") + except UnicodeEncodeError as exc: + raise EvidenceManifestError( + f"evidence path is not valid UTF-8: {raw!r}" + ) from exc + if "\\" in raw: + raise EvidenceManifestError(f"evidence path contains a backslash: {raw!r}") + normalized = unicodedata.normalize("NFC", raw) + if raw != normalized: + raise EvidenceManifestError(f"evidence path is not Unicode NFC-normalized: {raw!r}") + pure = PurePosixPath(normalized) + if pure.is_absolute() or ".." in pure.parts or not pure.parts: + raise EvidenceManifestError(f"unsafe evidence path: {raw!r}") + canonical = pure.as_posix() + _portable_path_key(canonical) + return canonical + + +def _portable_path_key(path: str) -> str: + key_parts: list[str] = [] + for segment in PurePosixPath(path).parts: + if segment.endswith((".", " ")): + raise EvidenceManifestError( + f"evidence path is not portable to Windows: {path!r}" + ) + if any( + ord(character) < 32 or character in _WINDOWS_FORBIDDEN_CHARACTERS + for character in segment + ): + raise EvidenceManifestError( + f"evidence path contains a Windows-forbidden character: {path!r}" + ) + basename = segment.split(".", maxsplit=1)[0].casefold() + if basename in _WINDOWS_RESERVED_BASENAMES: + raise EvidenceManifestError( + f"evidence path uses a Windows-reserved basename: {path!r}" + ) + key_parts.append(segment.casefold()) + return "/".join(key_parts) + + +def _file_signature(metadata: os.stat_result) -> tuple[int, int, int, int, int]: + return ( + metadata.st_dev, + metadata.st_ino, + metadata.st_size, + metadata.st_mtime_ns, + metadata.st_ctime_ns, + ) + + +def _sha256_stable_file( + path: Path, +) -> tuple[str, int, tuple[int, int, int, int, int]]: + flags = os.O_RDONLY | getattr(os, "O_BINARY", 0) + flags |= getattr(os, "O_NOFOLLOW", 0) + try: + descriptor = os.open(path, flags) + except OSError as exc: + raise EvidenceManifestError(f"cannot open evidence file safely: {path}") from exc + + digest = hashlib.sha256() + with os.fdopen(descriptor, "rb") as source: + before = os.fstat(source.fileno()) + if not stat.S_ISREG(before.st_mode): + raise EvidenceManifestError(f"evidence entry is not a regular file: {path}") + if before.st_size > MAX_FILE_BYTES: + raise EvidenceManifestError(f"evidence file exceeds size limit: {path}") + + bytes_read = 0 + for chunk in iter(lambda: source.read(1024 * 1024), b""): + bytes_read += len(chunk) + if bytes_read > MAX_FILE_BYTES: + raise EvidenceManifestError(f"evidence file exceeds size limit: {path}") + digest.update(chunk) + after = os.fstat(source.fileno()) + + try: + current = os.stat(path, follow_symlinks=False) + except OSError as exc: + raise EvidenceManifestError(f"evidence file changed while hashing: {path}") from exc + if ( + stat.S_ISLNK(current.st_mode) + or bytes_read != before.st_size + or _file_signature(before) != _file_signature(after) + or _file_signature(after) != _file_signature(current) + ): + raise EvidenceManifestError(f"evidence file changed while hashing: {path}") + return digest.hexdigest(), before.st_size, _file_signature(current) + + +def _canonical_utc_timestamp(value: str) -> str: + text = _required_text(value, "observed_at") + try: + parsed = datetime.fromisoformat(text.replace("Z", "+00:00")) + except ValueError as exc: + raise EvidenceManifestError("observed_at must be an ISO-8601 timestamp") from exc + if parsed.tzinfo is None: + raise EvidenceManifestError("observed_at must include a timezone") + return parsed.astimezone(UTC).isoformat().replace("+00:00", "Z") + + +def _validate_subjects(subjects: Sequence[EvidenceSubject]) -> list[EvidenceSubject]: + normalized: list[EvidenceSubject] = [] + identities: set[tuple[str, str | None]] = set() + for subject in subjects: + name = _required_text(subject.name, "subject.name") + version = ( + _required_text(subject.version, "subject.version") + if subject.version is not None + else None + ) + digest = subject.sha256.strip().lower() + if _SHA256_RE.fullmatch(digest) is None: + raise EvidenceManifestError("subject.sha256 must be a lowercase SHA-256 digest") + identity = (name.casefold(), version) + if identity in identities: + raise EvidenceManifestError(f"duplicate evidence subject: {name!r}") + identities.add(identity) + normalized.append(EvidenceSubject(name=name, version=version, sha256=digest)) + return sorted(normalized, key=lambda item: (item.name.casefold(), item.version or "")) + + +def _validate_manifest_shape(manifest: Mapping[str, Any]) -> None: + required = { + "schema", + "schema_version", + "snapshot_id", + "observed_at", + "source", + "subjects", + "artifacts", + "artifact_count", + "total_size_bytes", + "artifact_set_sha256", + } + if set(manifest) != required: + raise EvidenceManifestError( + f"manifest fields must be exactly {sorted(required)!r}" + ) + schema_version = manifest["schema_version"] + if ( + not isinstance(schema_version, int) + or isinstance(schema_version, bool) + or schema_version != SCHEMA_VERSION + ): + raise EvidenceManifestError(f"unsupported schema_version: {schema_version!r}") + if manifest["schema"] != "https://excelbench.dev/schemas/evidence-manifest/v1": + raise EvidenceManifestError("unsupported manifest schema URI") + snapshot_id = _string(manifest["snapshot_id"], "snapshot_id") + if snapshot_id != _required_text(snapshot_id, "snapshot_id"): + raise EvidenceManifestError("snapshot_id must use its canonical text form") + observed_at = _string(manifest["observed_at"], "observed_at") + if observed_at != _canonical_utc_timestamp(observed_at): + raise EvidenceManifestError("observed_at must use canonical UTC form") + source = _mapping(manifest["source"], "source") + if set(source) != {"repository", "commit"}: + raise EvidenceManifestError("source must contain exactly repository and commit") + repository = _string(source["repository"], "source.repository") + if repository != _required_text(repository, "source.repository"): + raise EvidenceManifestError("source.repository must use its canonical text form") + commit = _string(source["commit"], "source.commit") + if _GIT_SHA_RE.fullmatch(commit) is None: + raise EvidenceManifestError("source.commit must be a full lowercase Git SHA") + if not isinstance(manifest["subjects"], list) or not isinstance(manifest["artifacts"], list): + raise EvidenceManifestError("subjects and artifacts must be arrays") + artifact_count = manifest["artifact_count"] + if ( + not isinstance(artifact_count, int) + or isinstance(artifact_count, bool) + or artifact_count < 1 + ): + raise EvidenceManifestError("artifact_count must be a positive integer") + if artifact_count > MAX_FILES: + raise EvidenceManifestError("artifact_count exceeds the file-count limit") + total_size_bytes = manifest["total_size_bytes"] + if ( + not isinstance(total_size_bytes, int) + or isinstance(total_size_bytes, bool) + or total_size_bytes < 0 + ): + raise EvidenceManifestError("total_size_bytes must be a non-negative integer") + if total_size_bytes > MAX_TOTAL_BYTES: + raise EvidenceManifestError("total_size_bytes exceeds the total size limit") + artifact_set_sha256 = _string( + manifest["artifact_set_sha256"], "artifact_set_sha256" + ) + if _SHA256_RE.fullmatch(artifact_set_sha256) is None: + raise EvidenceManifestError("artifact_set_sha256 must be a lowercase SHA-256 digest") + + +def _validate_manifest_document( + manifest: Mapping[str, Any], + *, + reserved_manifest_name: str | None = None, +) -> list[EvidenceArtifact]: + _validate_serialized_text(manifest) + _validate_manifest_shape(manifest) + subjects = [ + _subject_from_mapping(value, index) + for index, value in enumerate(manifest["subjects"]) + ] + if subjects != _validate_subjects(subjects): + raise EvidenceManifestError("subjects must be sorted by canonical identity") + artifacts = [ + _artifact_from_mapping(value, index) + for index, value in enumerate(manifest["artifacts"]) + ] + if len(artifacts) > MAX_FILES: + raise EvidenceManifestError("manifest exceeds the file-count limit") + if any(artifact.size_bytes > MAX_FILE_BYTES for artifact in artifacts): + raise EvidenceManifestError("manifest artifact exceeds the per-file size limit") + total_size = sum(artifact.size_bytes for artifact in artifacts) + if total_size > MAX_TOTAL_BYTES: + raise EvidenceManifestError("manifest exceeds the total size limit") + if artifacts != sorted(artifacts): + raise EvidenceManifestError("artifacts must be sorted by canonical path") + if len({_portable_path_key(artifact.path) for artifact in artifacts}) != len(artifacts): + raise EvidenceManifestError("manifest contains case-insensitive path collisions") + if reserved_manifest_name is not None: + reserved_key = _portable_path_key(_safe_manifest_name(reserved_manifest_name)) + if any(_portable_path_key(artifact.path) == reserved_key for artifact in artifacts): + raise EvidenceManifestError( + "manifest artifact collides with the manifest destination" + ) + artifact_dicts = [artifact.to_dict() for artifact in artifacts] + digest = hashlib.sha256(canonical_json_bytes(artifact_dicts)).hexdigest() + if manifest["artifact_set_sha256"] != digest: + raise EvidenceManifestError("artifact_set_sha256 does not match the artifact inventory") + if manifest["artifact_count"] != len(artifacts): + raise EvidenceManifestError("artifact_count does not match the artifact inventory") + if manifest["total_size_bytes"] != total_size: + raise EvidenceManifestError("total_size_bytes does not match the artifact inventory") + return artifacts + + +def _read_existing_manifest_destination(path: Path) -> dict[str, Any] | None: + if path.is_symlink(): + raise EvidenceManifestError("manifest destination must not be a symlink") + if not path.exists(): + return None + if not path.is_file(): + raise EvidenceManifestError("manifest destination must be a regular file") + try: + manifest = read_evidence_manifest(path) + _validate_manifest_document( + manifest, + reserved_manifest_name=path.name, + ) + except EvidenceManifestError as exc: + raise EvidenceManifestError( + "manifest destination exists but is not a valid evidence manifest" + ) from exc + return manifest + + +def _artifact_from_mapping(value: Any, index: int) -> EvidenceArtifact: + item = _mapping(value, f"artifacts[{index}]") + if set(item) != {"path", "sha256", "size_bytes"}: + raise EvidenceManifestError(f"artifacts[{index}] has unexpected fields") + path = _string(item["path"], f"artifacts[{index}].path") + if _canonical_relative_path(Path(path)) != path: + raise EvidenceManifestError(f"artifacts[{index}].path is not canonical") + digest = _string(item["sha256"], f"artifacts[{index}].sha256") + if _SHA256_RE.fullmatch(digest) is None: + raise EvidenceManifestError(f"artifacts[{index}].sha256 is invalid") + size = item["size_bytes"] + if not isinstance(size, int) or isinstance(size, bool) or size < 0: + raise EvidenceManifestError(f"artifacts[{index}].size_bytes is invalid") + return EvidenceArtifact(path=path, sha256=digest, size_bytes=size) + + +def _subject_from_mapping(value: Any, index: int) -> EvidenceSubject: + item = _mapping(value, f"subjects[{index}]") + if set(item) not in ({"name", "sha256"}, {"name", "sha256", "version"}): + raise EvidenceManifestError(f"subjects[{index}] has unexpected fields") + return EvidenceSubject( + name=_string(item["name"], f"subjects[{index}].name"), + sha256=_string(item["sha256"], f"subjects[{index}].sha256"), + version=( + _string(item["version"], f"subjects[{index}].version") + if "version" in item + else None + ), + ) + + +def _safe_manifest_name(value: str) -> str: + name = _required_text(value, "manifest_name") + if Path(name).name != name or name in {".", ".."}: + raise EvidenceManifestError("manifest_name must be one filename") + if _canonical_relative_path(Path(name)) != name: + raise EvidenceManifestError("manifest_name must use its canonical portable form") + return name + + +def _required_text(value: str, label: str) -> str: + if not isinstance(value, str): + raise EvidenceManifestError(f"{label} must be a string") + _validate_utf8_text(value, label) + stripped = value.strip() + if not stripped or any(character in stripped for character in "\r\n\x00"): + raise EvidenceManifestError(f"{label} must be non-empty single-line text") + return stripped + + +def _mapping(value: Any, label: str) -> Mapping[str, Any]: + if not isinstance(value, Mapping) or not all(isinstance(key, str) for key in value): + raise EvidenceManifestError(f"{label} must be an object with string keys") + for key in value: + _validate_utf8_text(key, f"{label} key") + return value + + +def _string(value: Any, label: str) -> str: + if not isinstance(value, str): + raise EvidenceManifestError(f"{label} must be a string") + _validate_utf8_text(value, label) + return value + + +def _validate_utf8_text(value: str, label: str) -> None: + try: + value.encode("utf-8", errors="strict") + except UnicodeEncodeError as exc: + raise EvidenceManifestError(f"{label} must be valid UTF-8 text") from exc + + +def _validate_serialized_text(value: object) -> None: + pending = [value] + while pending: + item = pending.pop() + if isinstance(item, str): + _validate_utf8_text(item, "manifest text") + elif isinstance(item, Mapping): + for key, nested in item.items(): + if isinstance(key, str): + _validate_utf8_text(key, "manifest object key") + pending.append(nested) + elif isinstance(item, (list, tuple)): + pending.extend(item) diff --git a/tests/test_evidence_manifest.py b/tests/test_evidence_manifest.py new file mode 100644 index 0000000..fcb4cd8 --- /dev/null +++ b/tests/test_evidence_manifest.py @@ -0,0 +1,619 @@ +import hashlib +import json +import os +from pathlib import Path +from typing import Any + +import pytest +from typer.testing import CliRunner + +import excelbench.evidence_manifest as evidence_manifest +from excelbench.cli import app +from excelbench.evidence_manifest import ( + EvidenceManifestError, + EvidenceSubject, + build_evidence_manifest, + canonical_json_bytes, + parse_subject, + read_evidence_manifest, + verify_evidence_manifest, + write_evidence_manifest, +) + +SOURCE_SHA = "a" * 40 +OBSERVED_AT = "2026-08-31T00:00:00Z" +RUNNER = CliRunner() + + +def _manifest(root: Path) -> dict[str, Any]: + return build_evidence_manifest( + root, + snapshot_id="wolfxl-2.1-linux-x86_64", + repository="SynthGL/ExcelBench", + source_sha=SOURCE_SHA, + observed_at=OBSERVED_AT, + subjects=[EvidenceSubject("wolfxl-wheel", "b" * 64, "2.1.0")], + ) + + +def test_manifest_is_deterministic_path_free_and_exact(tmp_path: Path) -> None: + root = tmp_path / "results" + (root / "nested").mkdir(parents=True) + (root / "nested" / "matrix.csv").write_text("feature,score\ncell,3\n") + (root / "results.json").write_text('{"passed":true}\n') + + first = _manifest(root) + second = _manifest(root) + + assert canonical_json_bytes(first) == canonical_json_bytes(second) + serialized = canonical_json_bytes(first).decode() + assert str(tmp_path) not in serialized + assert [item["path"] for item in first["artifacts"]] == [ + "nested/matrix.csv", + "results.json", + ] + assert first["artifact_count"] == 2 + verify_evidence_manifest(root, first, expected_source_sha=SOURCE_SHA) + + +def test_verification_rejects_missing_extra_and_changed_files(tmp_path: Path) -> None: + root = tmp_path / "results" + root.mkdir() + artifact = root / "results.json" + artifact.write_text("before") + manifest = _manifest(root) + + artifact.write_text("after") + with pytest.raises(EvidenceManifestError, match="changed=.*results.json"): + verify_evidence_manifest(root, manifest) + + artifact.write_text("before") + (root / "extra.txt").write_text("unexpected") + with pytest.raises(EvidenceManifestError, match="extra=.*extra.txt"): + verify_evidence_manifest(root, manifest) + + (root / "extra.txt").unlink() + artifact.unlink() + with pytest.raises(EvidenceManifestError, match="missing=.*results.json"): + verify_evidence_manifest(root, manifest) + + +def test_verification_rechecks_manifest_after_inventory( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + root = tmp_path / "results" + root.mkdir() + (root / "results.json").write_text("{}") + manifest = _manifest(root) + path = root / "excelbench-evidence.json" + write_evidence_manifest(path, manifest) + replacement = json.loads(json.dumps(manifest)) + replacement["snapshot_id"] = "replacement-snapshot" + real_inventory = evidence_manifest._inventory + + def replace_after_inventory( + inventory_root: Path, + *, + excluded_root_name: str, + require_artifact: bool = True, + ) -> list[evidence_manifest.EvidenceArtifact]: + artifacts = real_inventory( + inventory_root, + excluded_root_name=excluded_root_name, + require_artifact=require_artifact, + ) + path.write_bytes(canonical_json_bytes(replacement) + b"\n") + return artifacts + + monkeypatch.setattr(evidence_manifest, "_inventory", replace_after_inventory) + with pytest.raises(EvidenceManifestError, match="changed while verifying evidence"): + verify_evidence_manifest(root, manifest) + + +def test_manifest_file_is_excluded_and_atomic_no_clobber_is_default(tmp_path: Path) -> None: + root = tmp_path / "results" + root.mkdir() + (root / "results.json").write_text("{}") + manifest = _manifest(root) + path = root / "excelbench-evidence.json" + + write_evidence_manifest(path, manifest) + assert read_evidence_manifest(path) == manifest + verify_evidence_manifest(root, read_evidence_manifest(path)) + + with pytest.raises(EvidenceManifestError, match="already exists"): + write_evidence_manifest(path, manifest) + + +def test_manifest_destination_rejects_symlinks_and_unrelated_artifacts( + tmp_path: Path, +) -> None: + root = tmp_path / "results" + root.mkdir() + (root / "artifact.txt").write_text("evidence") + external = tmp_path / "external.json" + external.write_text("do not replace") + destination = root / "excelbench-evidence.json" + try: + destination.symlink_to(external) + except OSError: + pytest.skip("symlinks unavailable") + + with pytest.raises(EvidenceManifestError, match="must not be a symlink"): + _manifest(root) + with pytest.raises(EvidenceManifestError, match="must not be a symlink"): + write_evidence_manifest(destination, {"irrelevant": True}, replace=True) + assert external.read_text() == "do not replace" + + destination.unlink() + destination.write_text("benchmark output") + with pytest.raises(EvidenceManifestError, match="not a valid evidence manifest"): + _manifest(root) + with pytest.raises(EvidenceManifestError, match="not a valid evidence manifest"): + write_evidence_manifest(destination, {"irrelevant": True}, replace=True) + assert destination.read_text() == "benchmark output" + + +def test_read_rejects_duplicate_json_keys(tmp_path: Path) -> None: + path = tmp_path / "manifest.json" + path.write_text('{"schema_version":1,"schema_version":1}') + + with pytest.raises(EvidenceManifestError, match="duplicate key"): + read_evidence_manifest(path) + + +def test_read_wraps_oversized_json_integer_errors(tmp_path: Path) -> None: + path = tmp_path / "manifest.json" + path.write_text('{"artifact_count":' + "9" * 5_000 + "}") + + with pytest.raises(EvidenceManifestError, match="not valid UTF-8 JSON"): + read_evidence_manifest(path) + + +@pytest.mark.parametrize( + "field", + ["snapshot_id", "source.repository", "subjects[0].name"], +) +def test_lone_surrogate_metadata_is_a_controlled_utf8_refusal( + tmp_path: Path, + field: str, +) -> None: + root = tmp_path / "results" + root.mkdir() + (root / "results.json").write_text("{}") + manifest = _manifest(root) + if field == "snapshot_id": + manifest["snapshot_id"] = "invalid-\ud800" + elif field == "source.repository": + manifest["source"]["repository"] = "invalid-\ud800" + else: + manifest["subjects"][0]["name"] = "invalid-\ud800" + + with pytest.raises(EvidenceManifestError, match="valid UTF-8"): + canonical_json_bytes(manifest) + with pytest.raises(EvidenceManifestError, match="valid UTF-8"): + write_evidence_manifest(tmp_path / "refused.json", manifest) + + path = root / "excelbench-evidence.json" + path.write_text(json.dumps(manifest), encoding="utf-8") + with pytest.raises(EvidenceManifestError, match="valid UTF-8"): + read_evidence_manifest(path) + + verified = RUNNER.invoke(app, ["verify-evidence", "--root", str(root)]) + assert verified.exit_code == 1 + assert "Evidence verification failed" in verified.output + assert "valid UTF-8" in verified.output + + +def test_read_rejects_oversized_descriptor_content( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + path = tmp_path / "manifest.json" + path.write_bytes(b"{" + b" " * 32) + monkeypatch.setattr(evidence_manifest, "MAX_MANIFEST_BYTES", 16) + + with pytest.raises(EvidenceManifestError, match="4 MiB size limit"): + read_evidence_manifest(path) + + +def test_read_detects_path_replacement_after_open( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + path = tmp_path / "manifest.json" + path.write_text("{}") + replacement = tmp_path / "replacement.json" + replacement.write_text('{"replacement":true}') + real_fstat = os.fstat + calls = 0 + + def replace_after_read(descriptor: int) -> os.stat_result: + nonlocal calls + metadata = real_fstat(descriptor) + calls += 1 + if calls == 2: + os.replace(replacement, path) + return metadata + + monkeypatch.setattr(os, "fstat", replace_after_read) + with pytest.raises(EvidenceManifestError, match="changed while reading"): + read_evidence_manifest(path) + + +def test_read_rejects_manifest_symlink(tmp_path: Path) -> None: + target = tmp_path / "target.json" + target.write_text("{}") + path = tmp_path / "manifest.json" + try: + path.symlink_to(target) + except OSError: + pytest.skip("symlinks unavailable") + + with pytest.raises(EvidenceManifestError, match="regular non-symlink"): + read_evidence_manifest(path) + + +def test_deeply_nested_json_is_a_controlled_refusal(tmp_path: Path) -> None: + root = tmp_path / "results" + root.mkdir() + path = root / "excelbench-evidence.json" + path.write_text('{"unexpected":' + "[" * 2_000 + "null" + "]" * 2_000 + "}") + + verified = RUNNER.invoke(app, ["verify-evidence", "--root", str(root)]) + assert verified.exit_code == 1 + assert "Evidence verification failed" in verified.output + assert "Traceback" not in verified.output + assert ( + "manifest is not valid UTF-8 JSON" in verified.output + or "manifest fields must be exactly" in verified.output + ) + + +def test_generated_and_written_manifests_share_the_reader_size_limit( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + root = tmp_path / "results" + root.mkdir() + (root / "results.json").write_text("{}") + manifest = _manifest(root) + limit = len(canonical_json_bytes(manifest)) + monkeypatch.setattr(evidence_manifest, "MAX_MANIFEST_BYTES", limit) + + with pytest.raises(EvidenceManifestError, match="4 MiB size limit"): + _manifest(root) + with pytest.raises(EvidenceManifestError, match="4 MiB size limit"): + write_evidence_manifest(root / "excelbench-evidence.json", manifest) + + +@pytest.mark.parametrize( + ("limit_name", "limit", "message"), + [ + ("MAX_FILES", 1, "file-count limit"), + ("MAX_FILE_BYTES", 4, "per-file size limit"), + ("MAX_TOTAL_BYTES", 8, "total size limit"), + ], +) +def test_writer_rejects_manifest_inventory_beyond_verifier_limits( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + limit_name: str, + limit: int, + message: str, +) -> None: + root = tmp_path / "results" + root.mkdir() + (root / "a.json").write_text("four") + (root / "b.json").write_text("five!") + manifest = _manifest(root) + monkeypatch.setattr(evidence_manifest, limit_name, limit) + + with pytest.raises(EvidenceManifestError, match=message): + write_evidence_manifest(root / "excelbench-evidence.json", manifest) + + +@pytest.mark.parametrize( + "artifact_path", + ["excelbench-evidence.json", "ExcelBench-Evidence.json"], +) +def test_writer_rejects_manifest_destination_artifact_alias( + tmp_path: Path, + artifact_path: str, +) -> None: + root = tmp_path / "results" + root.mkdir() + (root / "artifact.json").write_text("{}") + manifest = _manifest(root) + manifest["artifacts"][0]["path"] = artifact_path + manifest["artifact_set_sha256"] = hashlib.sha256( + canonical_json_bytes(manifest["artifacts"]) + ).hexdigest() + + destination = root / "excelbench-evidence.json" + with pytest.raises(EvidenceManifestError, match="collides with the manifest destination"): + write_evidence_manifest(destination, manifest) + assert not destination.exists() + + +def test_symlinks_and_case_collisions_fail_closed(tmp_path: Path) -> None: + root = tmp_path / "results" + root.mkdir() + target = root / "target.json" + target.write_text("{}") + link = root / "link.json" + try: + link.symlink_to(target) + except OSError: + pytest.skip("symlinks unavailable") + + with pytest.raises(EvidenceManifestError, match="symlinks are not allowed"): + _manifest(root) + + link.unlink() + (root / "A.json").write_text("a") + (root / "a.json").write_text("b") + with pytest.raises(EvidenceManifestError, match="case-insensitive"): + _manifest(root) + + +def test_windows_aliases_and_reserved_names_fail_closed(tmp_path: Path) -> None: + root = tmp_path / "results" + root.mkdir() + (root / "artifact.json").write_text("{}") + trailing_dot = root / "report." + try: + trailing_dot.write_text("alias") + except OSError: + pytest.skip("host filesystem rejects Windows-aliased names") + + with pytest.raises(EvidenceManifestError, match="not portable to Windows"): + _manifest(root) + + trailing_dot.unlink() + reserved = root / "CON.txt" + try: + reserved.write_text("reserved") + except OSError: + pytest.skip("host filesystem rejects Windows-reserved names") + with pytest.raises(EvidenceManifestError, match="Windows-reserved basename"): + _manifest(root) + + +@pytest.mark.parametrize("manifest_name", ["CON.json", "report.", r"nested\\manifest.json"]) +def test_manifest_name_must_be_portable( + tmp_path: Path, manifest_name: str +) -> None: + root = tmp_path / "results" + root.mkdir() + (root / "artifact.json").write_text("{}") + + with pytest.raises(EvidenceManifestError): + build_evidence_manifest( + root, + snapshot_id="snapshot", + repository="SynthGL/ExcelBench", + source_sha=SOURCE_SHA, + observed_at=OBSERVED_AT, + manifest_name=manifest_name, + ) + + +def test_manifest_name_portable_key_is_reserved(tmp_path: Path) -> None: + root = tmp_path / "results" + root.mkdir() + (root / "artifact.json").write_text("{}") + (root / "ExcelBench-Evidence.json").write_text("collision") + + with pytest.raises(EvidenceManifestError, match="case-insensitive"): + _manifest(root) + + +def test_artifact_metadata_changes_during_hashing_fail_closed( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + root = tmp_path / "results" + root.mkdir() + (root / "artifact.json").write_text("{}") + real_signature = evidence_manifest._file_signature + calls = 0 + + def changing_signature(metadata: Any) -> tuple[int, int, int, int, int]: + nonlocal calls + calls += 1 + signature = real_signature(metadata) + if calls == 2: + return (*signature[:-1], signature[-1] + 1) + return signature + + monkeypatch.setattr(evidence_manifest, "_file_signature", changing_signature) + with pytest.raises(EvidenceManifestError, match="changed while hashing"): + _manifest(root) + + +def test_artifact_created_during_hashing_fails_closed( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + root = tmp_path / "results" + root.mkdir() + (root / "artifact.json").write_text("{}") + real_hash = evidence_manifest._sha256_stable_file + created = False + + def create_late_artifact(path: Path) -> tuple[str, int, tuple[int, int, int, int, int]]: + nonlocal created + result = real_hash(path) + if not created: + (root / "late.json").write_text("late") + created = True + return result + + monkeypatch.setattr(evidence_manifest, "_sha256_stable_file", create_late_artifact) + with pytest.raises(EvidenceManifestError, match="changed while inventorying"): + _manifest(root) + + +def test_later_artifact_replaced_before_hashing_fails_closed( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + root = tmp_path / "results" + root.mkdir() + (root / "a.json").write_text("first") + later = root / "z.json" + later.write_text("original") + replacement = tmp_path / "replacement.json" + replacement.write_text("replacement") + real_hash = evidence_manifest._sha256_stable_file + + def replace_later(path: Path) -> tuple[str, int, tuple[int, int, int, int, int]]: + result = real_hash(path) + if path.name == "a.json": + os.replace(replacement, later) + return result + + monkeypatch.setattr(evidence_manifest, "_sha256_stable_file", replace_later) + with pytest.raises(EvidenceManifestError, match="after the initial inventory"): + _manifest(root) + + +@pytest.mark.skipif(os.name == "nt", reason="requires POSIX byte filenames") +def test_non_utf8_artifact_name_fails_closed(tmp_path: Path) -> None: + root = tmp_path / "results" + root.mkdir() + byte_path = os.path.join(os.fsencode(root), b"invalid-\xff.json") + descriptor = os.open(byte_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + try: + os.write(descriptor, b"{}") + finally: + os.close(descriptor) + + with pytest.raises(EvidenceManifestError, match="not valid UTF-8"): + _manifest(root) + + +def test_timestamp_source_and_subject_contracts_are_strict(tmp_path: Path) -> None: + root = tmp_path / "results" + root.mkdir() + (root / "results.json").write_text("{}") + + with pytest.raises(EvidenceManifestError, match="40-character Git SHA"): + build_evidence_manifest( + root, + snapshot_id="snapshot", + repository="SynthGL/ExcelBench", + source_sha="main", + observed_at=OBSERVED_AT, + ) + with pytest.raises(EvidenceManifestError, match="include a timezone"): + build_evidence_manifest( + root, + snapshot_id="snapshot", + repository="SynthGL/ExcelBench", + source_sha=SOURCE_SHA, + observed_at="2026-08-31T00:00:00", + ) + assert parse_subject(f"wolfxl@2.1.0={'b' * 64}") == EvidenceSubject( + "wolfxl", "b" * 64, "2.1.0" + ) + with pytest.raises(EvidenceManifestError, match="lowercase SHA-256"): + build_evidence_manifest( + root, + snapshot_id="snapshot", + repository="SynthGL/ExcelBench", + source_sha=SOURCE_SHA, + observed_at=OBSERVED_AT, + subjects=[EvidenceSubject("wolfxl", "invalid")], + ) + + +def test_tampered_aggregate_fields_are_rejected(tmp_path: Path) -> None: + root = tmp_path / "results" + root.mkdir() + (root / "results.json").write_text("{}") + manifest = _manifest(root) + tampered = json.loads(json.dumps(manifest)) + tampered["artifact_count"] = 99 + + with pytest.raises(EvidenceManifestError, match="artifact_count"): + verify_evidence_manifest(root, tampered) + + +def test_boolean_schema_version_is_rejected(tmp_path: Path) -> None: + root = tmp_path / "results" + root.mkdir() + (root / "results.json").write_text("{}") + manifest = _manifest(root) + manifest["schema_version"] = True + + with pytest.raises(EvidenceManifestError, match="unsupported schema_version"): + verify_evidence_manifest(root, manifest) + + +def test_cli_builds_and_verifies_exact_snapshot(tmp_path: Path) -> None: + root = tmp_path / "results" + root.mkdir() + (root / "results.json").write_text("{}") + + built = RUNNER.invoke( + app, + [ + "evidence-manifest", + "--root", + str(root), + "--snapshot-id", + "release-linux", + "--source-sha", + SOURCE_SHA, + "--observed-at", + OBSERVED_AT, + "--subject", + f"wolfxl@2.1.0={'b' * 64}", + ], + ) + assert built.exit_code == 0, built.output + assert (root / "excelbench-evidence.json").exists() + + verified = RUNNER.invoke( + app, + [ + "verify-evidence", + "--root", + str(root), + "--expected-source-sha", + SOURCE_SHA, + ], + ) + assert verified.exit_code == 0, verified.output + + +def test_cli_refuses_success_if_snapshot_changes_after_publication( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + root = tmp_path / "results" + root.mkdir() + artifact = root / "results.json" + artifact.write_text("before") + real_write = evidence_manifest.write_evidence_manifest + + def write_then_mutate( + path: Path, manifest: dict[str, Any], *, replace: bool = False + ) -> None: + real_write(path, manifest, replace=replace) + artifact.write_text("after") + + monkeypatch.setattr(evidence_manifest, "write_evidence_manifest", write_then_mutate) + built = RUNNER.invoke( + app, + [ + "evidence-manifest", + "--root", + str(root), + "--snapshot-id", + "release-linux", + "--source-sha", + SOURCE_SHA, + "--observed-at", + OBSERVED_AT, + ], + ) + + assert built.exit_code == 1 + assert "Evidence manifest refused" in built.output + assert "changed=['results.json']" in built.output