From 4fea8bfcd1a2d85af8cbce0cea685a4a21841fa1 Mon Sep 17 00:00:00 2001 From: Wolfie Date: Sun, 30 Aug 2026 17:17:49 -0700 Subject: [PATCH 01/21] feat(evidence): add deterministic exact snapshot manifests --- README.md | 31 ++ schemas/evidence-manifest-v1.schema.json | 118 ++++++ src/excelbench/cli.py | 113 ++++++ src/excelbench/evidence_manifest.py | 439 +++++++++++++++++++++++ tests/test_evidence_manifest.py | 202 +++++++++++ 5 files changed, 903 insertions(+) create mode 100644 schemas/evidence-manifest-v1.schema.json create mode 100644 src/excelbench/evidence_manifest.py create mode 100644 tests/test_evidence_manifest.py diff --git a/README.md b/README.md index 880baca..359ad33 100644 --- a/README.md +++ b/README.md @@ -177,6 +177,37 @@ Run the dedicated pivot capability artifact with: uv run excelbench cross-language-pivot-context --fixture fixtures/excel/tier2/15_pivot_tables.xlsx --output results-cross-language-pivots ``` + +## Exact Evidence Manifests + +A benchmark directory can be bound to its exact source and artifact identities with +a deterministic, path-free manifest: + +\`\`\`bash +uv run excelbench evidence-manifest \ + --root results-release-2026-08-31 \ + --snapshot-id wolfxl-2.1-linux-x86_64 \ + --source-sha 0123456789abcdef0123456789abcdef01234567 \ + --observed-at 2026-08-31T00:00:00Z \ + --subject wolfxl-wheel@2.1.0= + +uv run excelbench verify-evidence \ + --root results-release-2026-08-31 \ + --expected-source-sha 0123456789abcdef0123456789abcdef01234567 +\`\`\` + +The v1 contract inventories every regular file, hashes a canonical sorted file set, +rejects symlinks and cross-platform path collisions, and refuses undeclared, missing, +or changed files. It excludes only the manifest itself. The observation timestamp is +explicit so identical inputs produce identical manifest bytes. + +The manifest is the subject to sign or attest in release CI. Successful verification +does not make an evidence lane current by itself: public claims must still name the +snapshot date, source commit, tested package subjects, platform, and workload. + +Schema: [\`schemas/evidence-manifest-v1.schema.json\`](schemas/evidence-manifest-v1.schema.json) + + ## How It Works 1. **Generate reference files** -- [xlwings](https://www.xlwings.org/) drives real Excel to produce canonical `.xlsx`/`.xls` test files with known features. diff --git a/schemas/evidence-manifest-v1.schema.json b/schemas/evidence-manifest-v1.schema.json new file mode 100644 index 0000000..3db76a5 --- /dev/null +++ b/schemas/evidence-manifest-v1.schema.json @@ -0,0 +1,118 @@ +{ + "$id": "https://excelbench.dev/schemas/evidence-manifest/v1", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "artifact_count": { + "minimum": 1, + "type": "integer" + }, + "artifact_set_sha256": { + "pattern": "^[0-9a-f]{64}$", + "type": "string" + }, + "artifacts": { + "items": { + "additionalProperties": false, + "properties": { + "path": { + "minLength": 1, + "type": "string" + }, + "sha256": { + "pattern": "^[0-9a-f]{64}$", + "type": "string" + }, + "size_bytes": { + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "path", + "sha256", + "size_bytes" + ], + "type": "object" + }, + "minItems": 1, + "type": "array" + }, + "observed_at": { + "format": "date-time", + "pattern": "Z$", + "type": "string" + }, + "schema": { + "const": "https://excelbench.dev/schemas/evidence-manifest/v1" + }, + "schema_version": { + "const": 1 + }, + "snapshot_id": { + "minLength": 1, + "type": "string" + }, + "source": { + "additionalProperties": false, + "properties": { + "commit": { + "pattern": "^[0-9a-f]{40}$", + "type": "string" + }, + "repository": { + "minLength": 1, + "type": "string" + } + }, + "required": [ + "repository", + "commit" + ], + "type": "object" + }, + "subjects": { + "items": { + "additionalProperties": false, + "properties": { + "name": { + "minLength": 1, + "type": "string" + }, + "sha256": { + "pattern": "^[0-9a-f]{64}$", + "type": "string" + }, + "version": { + "minLength": 1, + "type": "string" + } + }, + "required": [ + "name", + "sha256" + ], + "type": "object" + }, + "type": "array" + }, + "total_size_bytes": { + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "schema", + "schema_version", + "snapshot_id", + "observed_at", + "source", + "subjects", + "artifacts", + "artifact_count", + "total_size_bytes", + "artifact_set_sha256" + ], + "title": "ExcelBench evidence manifest v1", + "type": "object" +} diff --git a/src/excelbench/cli.py b/src/excelbench/cli.py index 4c8292b..ee40a02 100644 --- a/src/excelbench/cli.py +++ b/src/excelbench/cli.py @@ -2001,5 +2001,118 @@ def show_summary(results: "BenchmarkResults") -> None: console.print(table) +@app.command("evidence-manifest") +def evidence_manifest( + root: Path = typer.Option( + Path("results"), + "--root", + help="Evidence directory to inventory exactly.", + ), + snapshot_id: str = typer.Option( + ..., + "--snapshot-id", + help="Stable identity for this evidence snapshot.", + ), + source_sha: str = typer.Option( + ..., + "--source-sha", + help="Exact 40-character source commit represented by the evidence.", + ), + observed_at: str = typer.Option( + ..., + "--observed-at", + help="Explicit timezone-aware observation timestamp.", + ), + repository: str = typer.Option( + "SynthGL/ExcelBench", + "--repository", + help="Source repository identity.", + ), + subject: list[str] | None = typer.Option( + None, + "--subject", + help="Bound artifact as NAME=SHA256 or NAME@VERSION=SHA256; repeatable.", + ), + manifest_name: str = typer.Option( + "excelbench-evidence.json", + "--manifest-name", + help="Manifest filename created directly below the evidence root.", + ), + replace: bool = typer.Option( + False, + "--replace", + help="Atomically replace an existing manifest.", + ), +) -> None: + """Create a deterministic exact-file evidence manifest.""" + from excelbench.evidence_manifest import ( + EvidenceManifestError, + build_evidence_manifest, + parse_subject, + write_evidence_manifest, + ) + + try: + subjects = [parse_subject(value) for value in subject or []] + manifest = build_evidence_manifest( + root, + snapshot_id=snapshot_id, + repository=repository, + source_sha=source_sha, + observed_at=observed_at, + subjects=subjects, + manifest_name=manifest_name, + ) + output = root.resolve() / manifest_name + write_evidence_manifest(output, manifest, replace=replace) + except (EvidenceManifestError, OSError) as exc: + console.print(f"[red]Evidence manifest refused: {exc}[/red]") + raise typer.Exit(1) from exc + console.print(f"[green]✓ Evidence manifest: {output}[/green]") + console.print(f" Artifacts: {manifest['artifact_count']}") + console.print(f" Artifact set: {manifest['artifact_set_sha256']}") + + +@app.command("verify-evidence") +def verify_evidence( + root: Path = typer.Option( + Path("results"), + "--root", + help="Evidence directory covered by the manifest.", + ), + manifest_name: str = typer.Option( + "excelbench-evidence.json", + "--manifest-name", + help="Manifest filename directly below the evidence root.", + ), + expected_source_sha: str | None = typer.Option( + None, + "--expected-source-sha", + help="Optional exact source commit required by the caller.", + ), +) -> None: + """Fail unless a manifest exactly covers the current evidence directory.""" + from excelbench.evidence_manifest import ( + EvidenceManifestError, + read_evidence_manifest, + verify_evidence_manifest, + ) + + path = root.resolve() / manifest_name + try: + manifest = read_evidence_manifest(path) + verify_evidence_manifest( + root, + manifest, + expected_source_sha=expected_source_sha, + manifest_name=manifest_name, + ) + except (EvidenceManifestError, OSError) as exc: + console.print(f"[red]Evidence verification failed: {exc}[/red]") + raise typer.Exit(1) from exc + console.print(f"[green]✓ Evidence verified: {path}[/green]") + console.print(f" Artifact set: {manifest['artifact_set_sha256']}") + + if __name__ == "__main__": app() diff --git a/src/excelbench/evidence_manifest.py b/src/excelbench/evidence_manifest.py new file mode 100644 index 0000000..11ab840 --- /dev/null +++ b/src/excelbench/evidence_manifest.py @@ -0,0 +1,439 @@ +"""Deterministic, exact-file manifests for benchmark evidence snapshots.""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +import tempfile +import unicodedata +from dataclasses import dataclass +from datetime import UTC, datetime +from pathlib import Path, PurePosixPath +from typing import Any, Iterable, Mapping, Sequence + +SCHEMA_VERSION = 1 +DEFAULT_MANIFEST_NAME = "excelbench-evidence.json" +MAX_FILES = 10_000 +MAX_FILE_BYTES = 512 * 1024 * 1024 +MAX_TOTAL_BYTES = 2 * 1024 * 1024 * 1024 +_SHA256_RE = re.compile(r"^[0-9a-f]{64}$") +_GIT_SHA_RE = re.compile(r"^[0-9a-f]{40}$") + + +class EvidenceManifestError(ValueError): + """The evidence snapshot or manifest violates its fail-closed contract.""" + + +@dataclass(frozen=True, order=True) +class EvidenceSubject: + """An external artifact or source identity bound into a snapshot.""" + + name: str + sha256: str + version: str | None = None + + def to_dict(self) -> dict[str, str]: + value = {"name": self.name, "sha256": self.sha256} + if self.version is not None: + value["version"] = self.version + return value + + +@dataclass(frozen=True, order=True) +class EvidenceArtifact: + """One immutable regular file in an evidence snapshot.""" + + path: str + sha256: str + size_bytes: int + + def to_dict(self) -> dict[str, str | int]: + return { + "path": self.path, + "sha256": self.sha256, + "size_bytes": self.size_bytes, + } + + +def canonical_json_bytes(value: object) -> bytes: + """Return the one wire representation used for hashing and publication.""" + return json.dumps( + value, + ensure_ascii=False, + allow_nan=False, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + + +def build_evidence_manifest( + root: Path, + *, + snapshot_id: str, + repository: str, + source_sha: str, + observed_at: str, + subjects: Sequence[EvidenceSubject] = (), + manifest_name: str = DEFAULT_MANIFEST_NAME, +) -> dict[str, Any]: + """Inventory ``root`` exactly and return a path-free deterministic manifest. + + ``observed_at`` is explicit rather than read from the clock so rerunning with + identical inputs produces identical bytes. The output manifest itself is + excluded when it lives directly below ``root``. + """ + root = root.resolve(strict=True) + if not root.is_dir(): + raise EvidenceManifestError("evidence root must be a directory") + snapshot_id = _required_text(snapshot_id, "snapshot_id") + repository = _required_text(repository, "repository") + source_sha = source_sha.strip().lower() + if _GIT_SHA_RE.fullmatch(source_sha) is None: + raise EvidenceManifestError("source_sha must be a full lowercase 40-character Git SHA") + observed_at = _canonical_utc_timestamp(observed_at) + manifest_name = _safe_manifest_name(manifest_name) + normalized_subjects = _validate_subjects(subjects) + artifacts = _inventory(root, excluded_root_name=manifest_name) + artifact_dicts = [artifact.to_dict() for artifact in artifacts] + artifact_set_sha256 = hashlib.sha256(canonical_json_bytes(artifact_dicts)).hexdigest() + return { + "schema": "https://excelbench.dev/schemas/evidence-manifest/v1", + "schema_version": SCHEMA_VERSION, + "snapshot_id": snapshot_id, + "observed_at": observed_at, + "source": {"repository": repository, "commit": source_sha}, + "subjects": [subject.to_dict() for subject in normalized_subjects], + "artifacts": artifact_dicts, + "artifact_count": len(artifacts), + "total_size_bytes": sum(artifact.size_bytes for artifact in artifacts), + "artifact_set_sha256": artifact_set_sha256, + } + + +def verify_evidence_manifest( + root: Path, + manifest: Mapping[str, Any], + *, + expected_source_sha: str | None = None, + manifest_name: str = DEFAULT_MANIFEST_NAME, +) -> None: + """Fail unless ``manifest`` exactly describes every regular file in ``root``.""" + _validate_manifest_shape(manifest) + source = _mapping(manifest["source"], "source") + source_sha = _string(source.get("commit"), "source.commit") + if expected_source_sha is not None and source_sha != expected_source_sha: + raise EvidenceManifestError("manifest source commit does not match expected source SHA") + subjects = [ + _subject_from_mapping(value, index) + for index, value in enumerate(manifest["subjects"]) + ] + if subjects != _validate_subjects(subjects): + raise EvidenceManifestError("subjects must be sorted by canonical identity") + actual = _inventory( + root.resolve(strict=True), excluded_root_name=_safe_manifest_name(manifest_name) + ) + expected = [ + _artifact_from_mapping(value, index) + for index, value in enumerate(manifest["artifacts"]) + ] + if expected != sorted(expected): + raise EvidenceManifestError("artifacts must be sorted by canonical path") + if len({artifact.path.casefold() for artifact in expected}) != len(expected): + raise EvidenceManifestError("manifest contains case-insensitive path collisions") + if actual != expected: + actual_by_path = {artifact.path: artifact for artifact in actual} + expected_by_path = {artifact.path: artifact for artifact in expected} + missing = sorted(expected_by_path.keys() - actual_by_path.keys()) + extra = sorted(actual_by_path.keys() - expected_by_path.keys()) + changed = sorted( + path + for path in actual_by_path.keys() & expected_by_path.keys() + if actual_by_path[path] != expected_by_path[path] + ) + raise EvidenceManifestError( + f"evidence mismatch: missing={missing!r}, extra={extra!r}, changed={changed!r}" + ) + artifact_dicts = [artifact.to_dict() for artifact in expected] + digest = hashlib.sha256(canonical_json_bytes(artifact_dicts)).hexdigest() + if manifest["artifact_set_sha256"] != digest: + raise EvidenceManifestError("artifact_set_sha256 does not match the artifact inventory") + if manifest["artifact_count"] != len(expected): + raise EvidenceManifestError("artifact_count does not match the artifact inventory") + if manifest["total_size_bytes"] != sum(artifact.size_bytes for artifact in expected): + raise EvidenceManifestError("total_size_bytes does not match the artifact inventory") + + +def read_evidence_manifest(path: Path) -> dict[str, Any]: + """Read a bounded UTF-8 manifest and reject duplicate JSON keys.""" + if path.is_symlink() or not path.is_file(): + raise EvidenceManifestError("manifest must be a regular non-symlink file") + if path.stat().st_size > 4 * 1024 * 1024: + raise EvidenceManifestError("manifest exceeds the 4 MiB size limit") + + def no_duplicates(pairs: Iterable[tuple[str, Any]]) -> dict[str, Any]: + value: dict[str, Any] = {} + for key, item in pairs: + if key in value: + raise EvidenceManifestError(f"manifest contains duplicate key {key!r}") + value[key] = item + return value + + try: + decoded = json.loads(path.read_text(encoding="utf-8"), object_pairs_hook=no_duplicates) + except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc: + raise EvidenceManifestError("manifest is not valid UTF-8 JSON") from exc + if not isinstance(decoded, dict): + raise EvidenceManifestError("manifest root must be an object") + return decoded + + +def write_evidence_manifest( + path: Path, manifest: Mapping[str, Any], *, replace: bool = False +) -> None: + """Publish one canonical manifest without exposing a partially written file.""" + path = path.resolve() + path.parent.mkdir(parents=True, exist_ok=True) + contents = canonical_json_bytes(dict(manifest)) + b"\n" + descriptor, temporary_name = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent) + temporary = Path(temporary_name) + try: + with os.fdopen(descriptor, "wb") as output: + output.write(contents) + output.flush() + os.fsync(output.fileno()) + if replace: + os.replace(temporary, path) + else: + try: + os.link(temporary, path) + except FileExistsError as exc: + raise EvidenceManifestError("manifest already exists; pass replace=True") from exc + temporary.unlink() + if os.name != "nt": + directory = os.open(path.parent, os.O_RDONLY) + try: + os.fsync(directory) + finally: + os.close(directory) + finally: + temporary.unlink(missing_ok=True) + + +def parse_subject(value: str) -> EvidenceSubject: + """Parse ``NAME=SHA256`` or ``NAME@VERSION=SHA256`` CLI syntax.""" + identity, separator, digest = value.partition("=") + if not separator: + raise EvidenceManifestError("subject must use NAME=SHA256 or NAME@VERSION=SHA256") + name, version_separator, version = identity.partition("@") + return EvidenceSubject( + name=_required_text(name, "subject.name"), + version=_required_text(version, "subject.version") if version_separator else None, + sha256=digest.strip().lower(), + ) + + +def _inventory(root: Path, *, excluded_root_name: str) -> list[EvidenceArtifact]: + artifacts: list[EvidenceArtifact] = [] + seen_casefolded: set[str] = set() + total_size = 0 + for path in sorted(root.rglob("*"), key=lambda item: item.as_posix()): + if path.parent == root and path.name == excluded_root_name: + continue + if path.is_symlink(): + raise EvidenceManifestError( + f"symlinks are not allowed in evidence: {path.relative_to(root)}" + ) + if path.is_dir(): + continue + if not path.is_file(): + raise EvidenceManifestError(f"non-regular evidence entry: {path.relative_to(root)}") + relative = _canonical_relative_path(path.relative_to(root)) + folded = relative.casefold() + if folded in seen_casefolded: + raise EvidenceManifestError(f"case-insensitive evidence path collision: {relative}") + seen_casefolded.add(folded) + size = path.stat().st_size + if size > MAX_FILE_BYTES: + raise EvidenceManifestError(f"evidence file exceeds size limit: {relative}") + total_size += size + if total_size > MAX_TOTAL_BYTES: + raise EvidenceManifestError("evidence snapshot exceeds total size limit") + artifacts.append( + EvidenceArtifact(path=relative, sha256=_sha256_file(path), size_bytes=size) + ) + if len(artifacts) > MAX_FILES: + raise EvidenceManifestError("evidence snapshot exceeds file-count limit") + if not artifacts: + raise EvidenceManifestError("evidence snapshot must contain at least one artifact") + return artifacts + + +def _canonical_relative_path(path: Path) -> str: + raw = path.as_posix() + if "\\" in raw: + raise EvidenceManifestError(f"evidence path contains a backslash: {raw!r}") + normalized = unicodedata.normalize("NFC", raw) + if raw != normalized: + raise EvidenceManifestError(f"evidence path is not Unicode NFC-normalized: {raw!r}") + pure = PurePosixPath(normalized) + if pure.is_absolute() or ".." in pure.parts or not pure.parts: + raise EvidenceManifestError(f"unsafe evidence path: {raw!r}") + return pure.as_posix() + + +def _sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as source: + for chunk in iter(lambda: source.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _canonical_utc_timestamp(value: str) -> str: + text = _required_text(value, "observed_at") + try: + parsed = datetime.fromisoformat(text.replace("Z", "+00:00")) + except ValueError as exc: + raise EvidenceManifestError("observed_at must be an ISO-8601 timestamp") from exc + if parsed.tzinfo is None: + raise EvidenceManifestError("observed_at must include a timezone") + return parsed.astimezone(UTC).isoformat().replace("+00:00", "Z") + + +def _validate_subjects(subjects: Sequence[EvidenceSubject]) -> list[EvidenceSubject]: + normalized: list[EvidenceSubject] = [] + identities: set[tuple[str, str | None]] = set() + for subject in subjects: + name = _required_text(subject.name, "subject.name") + version = ( + _required_text(subject.version, "subject.version") + if subject.version is not None + else None + ) + digest = subject.sha256.strip().lower() + if _SHA256_RE.fullmatch(digest) is None: + raise EvidenceManifestError("subject.sha256 must be a lowercase SHA-256 digest") + identity = (name.casefold(), version) + if identity in identities: + raise EvidenceManifestError(f"duplicate evidence subject: {name!r}") + identities.add(identity) + normalized.append(EvidenceSubject(name=name, version=version, sha256=digest)) + return sorted(normalized, key=lambda item: (item.name.casefold(), item.version or "")) + + +def _validate_manifest_shape(manifest: Mapping[str, Any]) -> None: + required = { + "schema", + "schema_version", + "snapshot_id", + "observed_at", + "source", + "subjects", + "artifacts", + "artifact_count", + "total_size_bytes", + "artifact_set_sha256", + } + if set(manifest) != required: + raise EvidenceManifestError( + f"manifest fields must be exactly {sorted(required)!r}" + ) + if manifest["schema_version"] != SCHEMA_VERSION: + raise EvidenceManifestError(f"unsupported schema_version: {manifest['schema_version']!r}") + if manifest["schema"] != "https://excelbench.dev/schemas/evidence-manifest/v1": + raise EvidenceManifestError("unsupported manifest schema URI") + snapshot_id = _string(manifest["snapshot_id"], "snapshot_id") + if snapshot_id != _required_text(snapshot_id, "snapshot_id"): + raise EvidenceManifestError("snapshot_id must use its canonical text form") + observed_at = _string(manifest["observed_at"], "observed_at") + if observed_at != _canonical_utc_timestamp(observed_at): + raise EvidenceManifestError("observed_at must use canonical UTC form") + source = _mapping(manifest["source"], "source") + if set(source) != {"repository", "commit"}: + raise EvidenceManifestError("source must contain exactly repository and commit") + repository = _string(source["repository"], "source.repository") + if repository != _required_text(repository, "source.repository"): + raise EvidenceManifestError("source.repository must use its canonical text form") + commit = _string(source["commit"], "source.commit") + if _GIT_SHA_RE.fullmatch(commit) is None: + raise EvidenceManifestError("source.commit must be a full lowercase Git SHA") + if not isinstance(manifest["subjects"], list) or not isinstance(manifest["artifacts"], list): + raise EvidenceManifestError("subjects and artifacts must be arrays") + artifact_count = manifest["artifact_count"] + if ( + not isinstance(artifact_count, int) + or isinstance(artifact_count, bool) + or artifact_count < 1 + ): + raise EvidenceManifestError("artifact_count must be a positive integer") + total_size_bytes = manifest["total_size_bytes"] + if ( + not isinstance(total_size_bytes, int) + or isinstance(total_size_bytes, bool) + or total_size_bytes < 0 + ): + raise EvidenceManifestError("total_size_bytes must be a non-negative integer") + artifact_set_sha256 = _string( + manifest["artifact_set_sha256"], "artifact_set_sha256" + ) + if _SHA256_RE.fullmatch(artifact_set_sha256) is None: + raise EvidenceManifestError("artifact_set_sha256 must be a lowercase SHA-256 digest") + + +def _artifact_from_mapping(value: Any, index: int) -> EvidenceArtifact: + item = _mapping(value, f"artifacts[{index}]") + if set(item) != {"path", "sha256", "size_bytes"}: + raise EvidenceManifestError(f"artifacts[{index}] has unexpected fields") + path = _string(item["path"], f"artifacts[{index}].path") + if _canonical_relative_path(Path(path)) != path: + raise EvidenceManifestError(f"artifacts[{index}].path is not canonical") + digest = _string(item["sha256"], f"artifacts[{index}].sha256") + if _SHA256_RE.fullmatch(digest) is None: + raise EvidenceManifestError(f"artifacts[{index}].sha256 is invalid") + size = item["size_bytes"] + if not isinstance(size, int) or isinstance(size, bool) or size < 0 or size > MAX_FILE_BYTES: + raise EvidenceManifestError(f"artifacts[{index}].size_bytes is invalid") + return EvidenceArtifact(path=path, sha256=digest, size_bytes=size) + + +def _subject_from_mapping(value: Any, index: int) -> EvidenceSubject: + item = _mapping(value, f"subjects[{index}]") + if set(item) not in ({"name", "sha256"}, {"name", "sha256", "version"}): + raise EvidenceManifestError(f"subjects[{index}] has unexpected fields") + return EvidenceSubject( + name=_string(item["name"], f"subjects[{index}].name"), + sha256=_string(item["sha256"], f"subjects[{index}].sha256"), + version=( + _string(item["version"], f"subjects[{index}].version") + if "version" in item + else None + ), + ) + + +def _safe_manifest_name(value: str) -> str: + name = _required_text(value, "manifest_name") + if Path(name).name != name or name in {".", ".."}: + raise EvidenceManifestError("manifest_name must be one filename") + return name + + +def _required_text(value: str, label: str) -> str: + stripped = value.strip() + if not stripped or any(character in stripped for character in "\r\n\x00"): + raise EvidenceManifestError(f"{label} must be non-empty single-line text") + return stripped + + +def _mapping(value: Any, label: str) -> Mapping[str, Any]: + if not isinstance(value, Mapping) or not all(isinstance(key, str) for key in value): + raise EvidenceManifestError(f"{label} must be an object with string keys") + return value + + +def _string(value: Any, label: str) -> str: + if not isinstance(value, str): + raise EvidenceManifestError(f"{label} must be a string") + return value diff --git a/tests/test_evidence_manifest.py b/tests/test_evidence_manifest.py new file mode 100644 index 0000000..fd87a25 --- /dev/null +++ b/tests/test_evidence_manifest.py @@ -0,0 +1,202 @@ +import json +from pathlib import Path + +import pytest +from typer.testing import CliRunner + +from excelbench.cli import app +from excelbench.evidence_manifest import ( + EvidenceManifestError, + EvidenceSubject, + build_evidence_manifest, + canonical_json_bytes, + parse_subject, + read_evidence_manifest, + verify_evidence_manifest, + write_evidence_manifest, +) + +SOURCE_SHA = "a" * 40 +OBSERVED_AT = "2026-08-31T00:00:00Z" +RUNNER = CliRunner() + + +def _manifest(root: Path) -> dict[str, object]: + return build_evidence_manifest( + root, + snapshot_id="wolfxl-2.1-linux-x86_64", + repository="SynthGL/ExcelBench", + source_sha=SOURCE_SHA, + observed_at=OBSERVED_AT, + subjects=[EvidenceSubject("wolfxl-wheel", "b" * 64, "2.1.0")], + ) + + +def test_manifest_is_deterministic_path_free_and_exact(tmp_path: Path) -> None: + root = tmp_path / "results" + (root / "nested").mkdir(parents=True) + (root / "nested" / "matrix.csv").write_text("feature,score\ncell,3\n") + (root / "results.json").write_text('{"passed":true}\n') + + first = _manifest(root) + second = _manifest(root) + + assert canonical_json_bytes(first) == canonical_json_bytes(second) + serialized = canonical_json_bytes(first).decode() + assert str(tmp_path) not in serialized + assert [item["path"] for item in first["artifacts"]] == [ + "nested/matrix.csv", + "results.json", + ] + assert first["artifact_count"] == 2 + verify_evidence_manifest(root, first, expected_source_sha=SOURCE_SHA) + + +def test_verification_rejects_missing_extra_and_changed_files(tmp_path: Path) -> None: + root = tmp_path / "results" + root.mkdir() + artifact = root / "results.json" + artifact.write_text("before") + manifest = _manifest(root) + + artifact.write_text("after") + with pytest.raises(EvidenceManifestError, match="changed=.*results.json"): + verify_evidence_manifest(root, manifest) + + artifact.write_text("before") + (root / "extra.txt").write_text("unexpected") + with pytest.raises(EvidenceManifestError, match="extra=.*extra.txt"): + verify_evidence_manifest(root, manifest) + + (root / "extra.txt").unlink() + artifact.unlink() + with pytest.raises(EvidenceManifestError, match="missing=.*results.json"): + verify_evidence_manifest(root, manifest) + + +def test_manifest_file_is_excluded_and_atomic_no_clobber_is_default(tmp_path: Path) -> None: + root = tmp_path / "results" + root.mkdir() + (root / "results.json").write_text("{}") + manifest = _manifest(root) + path = root / "excelbench-evidence.json" + + write_evidence_manifest(path, manifest) + assert read_evidence_manifest(path) == manifest + verify_evidence_manifest(root, read_evidence_manifest(path)) + + with pytest.raises(EvidenceManifestError, match="already exists"): + write_evidence_manifest(path, manifest) + + +def test_read_rejects_duplicate_json_keys(tmp_path: Path) -> None: + path = tmp_path / "manifest.json" + path.write_text('{"schema_version":1,"schema_version":1}') + + with pytest.raises(EvidenceManifestError, match="duplicate key"): + read_evidence_manifest(path) + + +def test_symlinks_and_case_collisions_fail_closed(tmp_path: Path) -> None: + root = tmp_path / "results" + root.mkdir() + target = root / "target.json" + target.write_text("{}") + link = root / "link.json" + try: + link.symlink_to(target) + except OSError: + pytest.skip("symlinks unavailable") + + with pytest.raises(EvidenceManifestError, match="symlinks are not allowed"): + _manifest(root) + + link.unlink() + (root / "A.json").write_text("a") + (root / "a.json").write_text("b") + with pytest.raises(EvidenceManifestError, match="case-insensitive"): + _manifest(root) + + +def test_timestamp_source_and_subject_contracts_are_strict(tmp_path: Path) -> None: + root = tmp_path / "results" + root.mkdir() + (root / "results.json").write_text("{}") + + with pytest.raises(EvidenceManifestError, match="40-character Git SHA"): + build_evidence_manifest( + root, + snapshot_id="snapshot", + repository="SynthGL/ExcelBench", + source_sha="main", + observed_at=OBSERVED_AT, + ) + with pytest.raises(EvidenceManifestError, match="include a timezone"): + build_evidence_manifest( + root, + snapshot_id="snapshot", + repository="SynthGL/ExcelBench", + source_sha=SOURCE_SHA, + observed_at="2026-08-31T00:00:00", + ) + assert parse_subject(f"wolfxl@2.1.0={'b' * 64}") == EvidenceSubject( + "wolfxl", "b" * 64, "2.1.0" + ) + with pytest.raises(EvidenceManifestError, match="lowercase SHA-256"): + build_evidence_manifest( + root, + snapshot_id="snapshot", + repository="SynthGL/ExcelBench", + source_sha=SOURCE_SHA, + observed_at=OBSERVED_AT, + subjects=[EvidenceSubject("wolfxl", "invalid")], + ) + + +def test_tampered_aggregate_fields_are_rejected(tmp_path: Path) -> None: + root = tmp_path / "results" + root.mkdir() + (root / "results.json").write_text("{}") + manifest = _manifest(root) + tampered = json.loads(json.dumps(manifest)) + tampered["artifact_count"] = 99 + + with pytest.raises(EvidenceManifestError, match="artifact_count"): + verify_evidence_manifest(root, tampered) + + +def test_cli_builds_and_verifies_exact_snapshot(tmp_path: Path) -> None: + root = tmp_path / "results" + root.mkdir() + (root / "results.json").write_text("{}") + + built = RUNNER.invoke( + app, + [ + "evidence-manifest", + "--root", + str(root), + "--snapshot-id", + "release-linux", + "--source-sha", + SOURCE_SHA, + "--observed-at", + OBSERVED_AT, + "--subject", + f"wolfxl@2.1.0={'b' * 64}", + ], + ) + assert built.exit_code == 0, built.output + assert (root / "excelbench-evidence.json").exists() + + verified = RUNNER.invoke( + app, + [ + "verify-evidence", + "--root", + str(root), + "--expected-source-sha", + SOURCE_SHA, + ], + ) + assert verified.exit_code == 0, verified.output From a8aaac5bc018b67d3e307a41381330e5d3ab9d1b Mon Sep 17 00:00:00 2001 From: Wolfie Date: Sun, 30 Aug 2026 17:20:08 -0700 Subject: [PATCH 02/21] fix(evidence): use collections ABC imports --- src/excelbench/evidence_manifest.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/excelbench/evidence_manifest.py b/src/excelbench/evidence_manifest.py index 11ab840..cd806c9 100644 --- a/src/excelbench/evidence_manifest.py +++ b/src/excelbench/evidence_manifest.py @@ -8,10 +8,11 @@ import re import tempfile import unicodedata +from collections.abc import Iterable, Mapping, Sequence from dataclasses import dataclass from datetime import UTC, datetime from pathlib import Path, PurePosixPath -from typing import Any, Iterable, Mapping, Sequence +from typing import Any SCHEMA_VERSION = 1 DEFAULT_MANIFEST_NAME = "excelbench-evidence.json" From e819e4f3e6c244e14ea45c4f39b0f0a0f2c1a640 Mon Sep 17 00:00:00 2001 From: Wolfie Date: Sun, 30 Aug 2026 17:23:54 -0700 Subject: [PATCH 03/21] fix(evidence): report empty snapshots as exact mismatches --- src/excelbench/evidence_manifest.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/excelbench/evidence_manifest.py b/src/excelbench/evidence_manifest.py index cd806c9..2ff2d89 100644 --- a/src/excelbench/evidence_manifest.py +++ b/src/excelbench/evidence_manifest.py @@ -133,7 +133,9 @@ def verify_evidence_manifest( if subjects != _validate_subjects(subjects): raise EvidenceManifestError("subjects must be sorted by canonical identity") actual = _inventory( - root.resolve(strict=True), excluded_root_name=_safe_manifest_name(manifest_name) + root.resolve(strict=True), + excluded_root_name=_safe_manifest_name(manifest_name), + require_artifact=False, ) expected = [ _artifact_from_mapping(value, index) @@ -235,7 +237,9 @@ def parse_subject(value: str) -> EvidenceSubject: ) -def _inventory(root: Path, *, excluded_root_name: str) -> list[EvidenceArtifact]: +def _inventory( + root: Path, *, excluded_root_name: str, require_artifact: bool = True +) -> list[EvidenceArtifact]: artifacts: list[EvidenceArtifact] = [] seen_casefolded: set[str] = set() total_size = 0 @@ -266,7 +270,7 @@ def _inventory(root: Path, *, excluded_root_name: str) -> list[EvidenceArtifact] ) if len(artifacts) > MAX_FILES: raise EvidenceManifestError("evidence snapshot exceeds file-count limit") - if not artifacts: + if require_artifact and not artifacts: raise EvidenceManifestError("evidence snapshot must contain at least one artifact") return artifacts From b44b7d8e26518b45c16ba57bc616b4514573f0c1 Mon Sep 17 00:00:00 2001 From: Wolfie Date: Sun, 30 Aug 2026 17:23:55 -0700 Subject: [PATCH 04/21] fix(evidence): make manifest helper type explicit --- tests/test_evidence_manifest.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_evidence_manifest.py b/tests/test_evidence_manifest.py index fd87a25..ce0bca9 100644 --- a/tests/test_evidence_manifest.py +++ b/tests/test_evidence_manifest.py @@ -1,5 +1,6 @@ import json from pathlib import Path +from typing import Any import pytest from typer.testing import CliRunner @@ -21,7 +22,7 @@ RUNNER = CliRunner() -def _manifest(root: Path) -> dict[str, object]: +def _manifest(root: Path) -> dict[str, Any]: return build_evidence_manifest( root, snapshot_id="wolfxl-2.1-linux-x86_64", From 663668276d5661312e25cfdebad8e2ac9cf179cf Mon Sep 17 00:00:00 2001 From: Wolfie Date: Sun, 30 Aug 2026 17:29:34 -0700 Subject: [PATCH 05/21] fix(evidence): harden manifest replacement boundaries --- src/excelbench/evidence_manifest.py | 103 ++++++++++++++++++++-------- 1 file changed, 75 insertions(+), 28 deletions(-) diff --git a/src/excelbench/evidence_manifest.py b/src/excelbench/evidence_manifest.py index 2ff2d89..5556768 100644 --- a/src/excelbench/evidence_manifest.py +++ b/src/excelbench/evidence_manifest.py @@ -95,6 +95,7 @@ def build_evidence_manifest( raise EvidenceManifestError("source_sha must be a full lowercase 40-character Git SHA") observed_at = _canonical_utc_timestamp(observed_at) manifest_name = _safe_manifest_name(manifest_name) + _read_existing_manifest_destination(root / manifest_name) normalized_subjects = _validate_subjects(subjects) artifacts = _inventory(root, excluded_root_name=manifest_name) artifact_dicts = [artifact.to_dict() for artifact in artifacts] @@ -121,30 +122,24 @@ def verify_evidence_manifest( manifest_name: str = DEFAULT_MANIFEST_NAME, ) -> None: """Fail unless ``manifest`` exactly describes every regular file in ``root``.""" - _validate_manifest_shape(manifest) + expected = _validate_manifest_document(manifest) source = _mapping(manifest["source"], "source") source_sha = _string(source.get("commit"), "source.commit") if expected_source_sha is not None and source_sha != expected_source_sha: raise EvidenceManifestError("manifest source commit does not match expected source SHA") - subjects = [ - _subject_from_mapping(value, index) - for index, value in enumerate(manifest["subjects"]) - ] - if subjects != _validate_subjects(subjects): - raise EvidenceManifestError("subjects must be sorted by canonical identity") + root = root.resolve(strict=True) + manifest_name = _safe_manifest_name(manifest_name) + destination_manifest = _read_existing_manifest_destination(root / manifest_name) + if ( + destination_manifest is not None + and canonical_json_bytes(destination_manifest) != canonical_json_bytes(dict(manifest)) + ): + raise EvidenceManifestError("manifest file does not match the manifest being verified") actual = _inventory( - root.resolve(strict=True), - excluded_root_name=_safe_manifest_name(manifest_name), + root, + excluded_root_name=manifest_name, require_artifact=False, ) - expected = [ - _artifact_from_mapping(value, index) - for index, value in enumerate(manifest["artifacts"]) - ] - if expected != sorted(expected): - raise EvidenceManifestError("artifacts must be sorted by canonical path") - if len({artifact.path.casefold() for artifact in expected}) != len(expected): - raise EvidenceManifestError("manifest contains case-insensitive path collisions") if actual != expected: actual_by_path = {artifact.path: artifact for artifact in actual} expected_by_path = {artifact.path: artifact for artifact in expected} @@ -158,14 +153,6 @@ def verify_evidence_manifest( raise EvidenceManifestError( f"evidence mismatch: missing={missing!r}, extra={extra!r}, changed={changed!r}" ) - artifact_dicts = [artifact.to_dict() for artifact in expected] - digest = hashlib.sha256(canonical_json_bytes(artifact_dicts)).hexdigest() - if manifest["artifact_set_sha256"] != digest: - raise EvidenceManifestError("artifact_set_sha256 does not match the artifact inventory") - if manifest["artifact_count"] != len(expected): - raise EvidenceManifestError("artifact_count does not match the artifact inventory") - if manifest["total_size_bytes"] != sum(artifact.size_bytes for artifact in expected): - raise EvidenceManifestError("total_size_bytes does not match the artifact inventory") def read_evidence_manifest(path: Path) -> dict[str, Any]: @@ -196,8 +183,16 @@ def write_evidence_manifest( path: Path, manifest: Mapping[str, Any], *, replace: bool = False ) -> None: """Publish one canonical manifest without exposing a partially written file.""" - path = path.resolve() + if path.is_symlink(): + raise EvidenceManifestError("manifest destination must not be a symlink") + name = _safe_manifest_name(path.name) path.parent.mkdir(parents=True, exist_ok=True) + path = path.parent.resolve(strict=True) / name + if path.is_symlink(): + raise EvidenceManifestError("manifest destination must not be a symlink") + if replace: + _read_existing_manifest_destination(path) + _validate_manifest_document(manifest) contents = canonical_json_bytes(dict(manifest)) + b"\n" descriptor, temporary_name = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent) temporary = Path(temporary_name) @@ -207,6 +202,7 @@ def write_evidence_manifest( output.flush() os.fsync(output.fileno()) if replace: + _read_existing_manifest_destination(path) os.replace(temporary, path) else: try: @@ -345,8 +341,13 @@ def _validate_manifest_shape(manifest: Mapping[str, Any]) -> None: raise EvidenceManifestError( f"manifest fields must be exactly {sorted(required)!r}" ) - if manifest["schema_version"] != SCHEMA_VERSION: - raise EvidenceManifestError(f"unsupported schema_version: {manifest['schema_version']!r}") + schema_version = manifest["schema_version"] + if ( + not isinstance(schema_version, int) + or isinstance(schema_version, bool) + or schema_version != SCHEMA_VERSION + ): + raise EvidenceManifestError(f"unsupported schema_version: {schema_version!r}") if manifest["schema"] != "https://excelbench.dev/schemas/evidence-manifest/v1": raise EvidenceManifestError("unsupported manifest schema URI") snapshot_id = _string(manifest["snapshot_id"], "snapshot_id") @@ -387,6 +388,52 @@ def _validate_manifest_shape(manifest: Mapping[str, Any]) -> None: raise EvidenceManifestError("artifact_set_sha256 must be a lowercase SHA-256 digest") +def _validate_manifest_document(manifest: Mapping[str, Any]) -> list[EvidenceArtifact]: + _validate_manifest_shape(manifest) + subjects = [ + _subject_from_mapping(value, index) + for index, value in enumerate(manifest["subjects"]) + ] + if subjects != _validate_subjects(subjects): + raise EvidenceManifestError("subjects must be sorted by canonical identity") + artifacts = [ + _artifact_from_mapping(value, index) + for index, value in enumerate(manifest["artifacts"]) + ] + if artifacts != sorted(artifacts): + raise EvidenceManifestError("artifacts must be sorted by canonical path") + if len({artifact.path.casefold() for artifact in artifacts}) != len(artifacts): + raise EvidenceManifestError("manifest contains case-insensitive path collisions") + artifact_dicts = [artifact.to_dict() for artifact in artifacts] + digest = hashlib.sha256(canonical_json_bytes(artifact_dicts)).hexdigest() + if manifest["artifact_set_sha256"] != digest: + raise EvidenceManifestError("artifact_set_sha256 does not match the artifact inventory") + if manifest["artifact_count"] != len(artifacts): + raise EvidenceManifestError("artifact_count does not match the artifact inventory") + if manifest["total_size_bytes"] != sum( + artifact.size_bytes for artifact in artifacts + ): + raise EvidenceManifestError("total_size_bytes does not match the artifact inventory") + return artifacts + + +def _read_existing_manifest_destination(path: Path) -> dict[str, Any] | None: + if path.is_symlink(): + raise EvidenceManifestError("manifest destination must not be a symlink") + if not path.exists(): + return None + if not path.is_file(): + raise EvidenceManifestError("manifest destination must be a regular file") + try: + manifest = read_evidence_manifest(path) + _validate_manifest_document(manifest) + except EvidenceManifestError as exc: + raise EvidenceManifestError( + "manifest destination exists but is not a valid evidence manifest" + ) from exc + return manifest + + def _artifact_from_mapping(value: Any, index: int) -> EvidenceArtifact: item = _mapping(value, f"artifacts[{index}]") if set(item) != {"path", "sha256", "size_bytes"}: From 621f0032edfafd325f0852815f588db2f807b0c9 Mon Sep 17 00:00:00 2001 From: Wolfie Date: Sun, 30 Aug 2026 17:29:35 -0700 Subject: [PATCH 06/21] test(evidence): cover destination and schema attacks --- tests/test_evidence_manifest.py | 40 +++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/tests/test_evidence_manifest.py b/tests/test_evidence_manifest.py index ce0bca9..dcf44f0 100644 --- a/tests/test_evidence_manifest.py +++ b/tests/test_evidence_manifest.py @@ -90,6 +90,35 @@ def test_manifest_file_is_excluded_and_atomic_no_clobber_is_default(tmp_path: Pa write_evidence_manifest(path, manifest) +def test_manifest_destination_rejects_symlinks_and_unrelated_artifacts( + tmp_path: Path, +) -> None: + root = tmp_path / "results" + root.mkdir() + (root / "artifact.txt").write_text("evidence") + external = tmp_path / "external.json" + external.write_text("do not replace") + destination = root / "excelbench-evidence.json" + try: + destination.symlink_to(external) + except OSError: + pytest.skip("symlinks unavailable") + + with pytest.raises(EvidenceManifestError, match="must not be a symlink"): + _manifest(root) + with pytest.raises(EvidenceManifestError, match="must not be a symlink"): + write_evidence_manifest(destination, {"irrelevant": True}, replace=True) + assert external.read_text() == "do not replace" + + destination.unlink() + destination.write_text("benchmark output") + with pytest.raises(EvidenceManifestError, match="not a valid evidence manifest"): + _manifest(root) + with pytest.raises(EvidenceManifestError, match="not a valid evidence manifest"): + write_evidence_manifest(destination, {"irrelevant": True}, replace=True) + assert destination.read_text() == "benchmark output" + + def test_read_rejects_duplicate_json_keys(tmp_path: Path) -> None: path = tmp_path / "manifest.json" path.write_text('{"schema_version":1,"schema_version":1}') @@ -166,6 +195,17 @@ def test_tampered_aggregate_fields_are_rejected(tmp_path: Path) -> None: verify_evidence_manifest(root, tampered) +def test_boolean_schema_version_is_rejected(tmp_path: Path) -> None: + root = tmp_path / "results" + root.mkdir() + (root / "results.json").write_text("{}") + manifest = _manifest(root) + manifest["schema_version"] = True + + with pytest.raises(EvidenceManifestError, match="unsupported schema_version"): + verify_evidence_manifest(root, manifest) + + def test_cli_builds_and_verifies_exact_snapshot(tmp_path: Path) -> None: root = tmp_path / "results" root.mkdir() From 4842924f09b5a94a236a17599cbd50993f3b01be Mon Sep 17 00:00:00 2001 From: Wolfie Date: Sun, 30 Aug 2026 17:44:02 -0700 Subject: [PATCH 07/21] fix(evidence): enforce portable artifact paths --- src/excelbench/evidence_manifest.py | 51 +++++++++++++++++++++++++---- 1 file changed, 45 insertions(+), 6 deletions(-) diff --git a/src/excelbench/evidence_manifest.py b/src/excelbench/evidence_manifest.py index 5556768..d8432ee 100644 --- a/src/excelbench/evidence_manifest.py +++ b/src/excelbench/evidence_manifest.py @@ -21,6 +21,18 @@ MAX_TOTAL_BYTES = 2 * 1024 * 1024 * 1024 _SHA256_RE = re.compile(r"^[0-9a-f]{64}$") _GIT_SHA_RE = re.compile(r"^[0-9a-f]{40}$") +_WINDOWS_RESERVED_BASENAMES = { + "aux", + "clock$", + "con", + "conin$", + "conout$", + "nul", + "prn", + *(f"com{suffix}" for suffix in (*range(1, 10), "¹", "²", "³")), + *(f"lpt{suffix}" for suffix in (*range(1, 10), "¹", "²", "³")), +} +_WINDOWS_FORBIDDEN_CHARACTERS = frozenset('<>:"|?*') class EvidenceManifestError(ValueError): @@ -172,7 +184,9 @@ def no_duplicates(pairs: Iterable[tuple[str, Any]]) -> dict[str, Any]: try: decoded = json.loads(path.read_text(encoding="utf-8"), object_pairs_hook=no_duplicates) - except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc: + except EvidenceManifestError: + raise + except (OSError, UnicodeDecodeError, ValueError) as exc: raise EvidenceManifestError("manifest is not valid UTF-8 JSON") from exc if not isinstance(decoded, dict): raise EvidenceManifestError("manifest root must be an object") @@ -251,10 +265,10 @@ def _inventory( if not path.is_file(): raise EvidenceManifestError(f"non-regular evidence entry: {path.relative_to(root)}") relative = _canonical_relative_path(path.relative_to(root)) - folded = relative.casefold() - if folded in seen_casefolded: + portable_key = _portable_path_key(relative) + if portable_key in seen_casefolded: raise EvidenceManifestError(f"case-insensitive evidence path collision: {relative}") - seen_casefolded.add(folded) + seen_casefolded.add(portable_key) size = path.stat().st_size if size > MAX_FILE_BYTES: raise EvidenceManifestError(f"evidence file exceeds size limit: {relative}") @@ -281,7 +295,32 @@ def _canonical_relative_path(path: Path) -> str: pure = PurePosixPath(normalized) if pure.is_absolute() or ".." in pure.parts or not pure.parts: raise EvidenceManifestError(f"unsafe evidence path: {raw!r}") - return pure.as_posix() + canonical = pure.as_posix() + _portable_path_key(canonical) + return canonical + + +def _portable_path_key(path: str) -> str: + key_parts: list[str] = [] + for segment in PurePosixPath(path).parts: + if segment.endswith((".", " ")): + raise EvidenceManifestError( + f"evidence path is not portable to Windows: {path!r}" + ) + if any( + ord(character) < 32 or character in _WINDOWS_FORBIDDEN_CHARACTERS + for character in segment + ): + raise EvidenceManifestError( + f"evidence path contains a Windows-forbidden character: {path!r}" + ) + basename = segment.split(".", maxsplit=1)[0].casefold() + if basename in _WINDOWS_RESERVED_BASENAMES: + raise EvidenceManifestError( + f"evidence path uses a Windows-reserved basename: {path!r}" + ) + key_parts.append(segment.casefold()) + return "/".join(key_parts) def _sha256_file(path: Path) -> str: @@ -402,7 +441,7 @@ def _validate_manifest_document(manifest: Mapping[str, Any]) -> list[EvidenceArt ] if artifacts != sorted(artifacts): raise EvidenceManifestError("artifacts must be sorted by canonical path") - if len({artifact.path.casefold() for artifact in artifacts}) != len(artifacts): + if len({_portable_path_key(artifact.path) for artifact in artifacts}) != len(artifacts): raise EvidenceManifestError("manifest contains case-insensitive path collisions") artifact_dicts = [artifact.to_dict() for artifact in artifacts] digest = hashlib.sha256(canonical_json_bytes(artifact_dicts)).hexdigest() From d9b77c6dc963dd40bc0a95dd1a16684ad0fe02ee Mon Sep 17 00:00:00 2001 From: Wolfie Date: Sun, 30 Aug 2026 17:44:03 -0700 Subject: [PATCH 08/21] test(evidence): cover path aliases and decoder limits --- tests/test_evidence_manifest.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/tests/test_evidence_manifest.py b/tests/test_evidence_manifest.py index dcf44f0..1fd2564 100644 --- a/tests/test_evidence_manifest.py +++ b/tests/test_evidence_manifest.py @@ -127,6 +127,14 @@ def test_read_rejects_duplicate_json_keys(tmp_path: Path) -> None: read_evidence_manifest(path) +def test_read_wraps_oversized_json_integer_errors(tmp_path: Path) -> None: + path = tmp_path / "manifest.json" + path.write_text('{"artifact_count":' + "9" * 5_000 + "}") + + with pytest.raises(EvidenceManifestError, match="not valid UTF-8 JSON"): + read_evidence_manifest(path) + + def test_symlinks_and_case_collisions_fail_closed(tmp_path: Path) -> None: root = tmp_path / "results" root.mkdir() @@ -148,6 +156,29 @@ def test_symlinks_and_case_collisions_fail_closed(tmp_path: Path) -> None: _manifest(root) +def test_windows_aliases_and_reserved_names_fail_closed(tmp_path: Path) -> None: + root = tmp_path / "results" + root.mkdir() + (root / "artifact.json").write_text("{}") + trailing_dot = root / "report." + try: + trailing_dot.write_text("alias") + except OSError: + pytest.skip("host filesystem rejects Windows-aliased names") + + with pytest.raises(EvidenceManifestError, match="not portable to Windows"): + _manifest(root) + + trailing_dot.unlink() + reserved = root / "CON.txt" + try: + reserved.write_text("reserved") + except OSError: + pytest.skip("host filesystem rejects Windows-reserved names") + with pytest.raises(EvidenceManifestError, match="Windows-reserved basename"): + _manifest(root) + + def test_timestamp_source_and_subject_contracts_are_strict(tmp_path: Path) -> None: root = tmp_path / "results" root.mkdir() From 7ad5311479f2a0e4f528cc6caf5d76d69d9e8d88 Mon Sep 17 00:00:00 2001 From: Wolfie Date: Sun, 30 Aug 2026 17:44:04 -0700 Subject: [PATCH 09/21] docs(evidence): render the CLI example as a code block --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 359ad33..7961666 100644 --- a/README.md +++ b/README.md @@ -183,7 +183,7 @@ uv run excelbench cross-language-pivot-context --fixture fixtures/excel/tier2/15 A benchmark directory can be bound to its exact source and artifact identities with a deterministic, path-free manifest: -\`\`\`bash +```bash uv run excelbench evidence-manifest \ --root results-release-2026-08-31 \ --snapshot-id wolfxl-2.1-linux-x86_64 \ @@ -194,7 +194,7 @@ uv run excelbench evidence-manifest \ uv run excelbench verify-evidence \ --root results-release-2026-08-31 \ --expected-source-sha 0123456789abcdef0123456789abcdef01234567 -\`\`\` +``` The v1 contract inventories every regular file, hashes a canonical sorted file set, rejects symlinks and cross-platform path collisions, and refuses undeclared, missing, From e6f2c9dde60d19f633fcd64fa93602305a842031 Mon Sep 17 00:00:00 2001 From: Wolfie Date: Sun, 30 Aug 2026 17:49:46 -0700 Subject: [PATCH 10/21] fix(evidence): require portable manifest filenames --- src/excelbench/evidence_manifest.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/excelbench/evidence_manifest.py b/src/excelbench/evidence_manifest.py index d8432ee..22acee2 100644 --- a/src/excelbench/evidence_manifest.py +++ b/src/excelbench/evidence_manifest.py @@ -508,6 +508,8 @@ def _safe_manifest_name(value: str) -> str: name = _required_text(value, "manifest_name") if Path(name).name != name or name in {".", ".."}: raise EvidenceManifestError("manifest_name must be one filename") + if _canonical_relative_path(Path(name)) != name: + raise EvidenceManifestError("manifest_name must use its canonical portable form") return name From 9420e6c8f2263b8e7b92b58577c521f35e3b232f Mon Sep 17 00:00:00 2001 From: Wolfie Date: Sun, 30 Aug 2026 17:49:58 -0700 Subject: [PATCH 11/21] test(evidence): reject nonportable manifest names --- tests/test_evidence_manifest.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/test_evidence_manifest.py b/tests/test_evidence_manifest.py index 1fd2564..13209e1 100644 --- a/tests/test_evidence_manifest.py +++ b/tests/test_evidence_manifest.py @@ -179,6 +179,25 @@ def test_windows_aliases_and_reserved_names_fail_closed(tmp_path: Path) -> None: _manifest(root) +@pytest.mark.parametrize("manifest_name", ["CON.json", "report.", r"nested\\manifest.json"]) +def test_manifest_name_must_be_portable( + tmp_path: Path, manifest_name: str +) -> None: + root = tmp_path / "results" + root.mkdir() + (root / "artifact.json").write_text("{}") + + with pytest.raises(EvidenceManifestError): + build_evidence_manifest( + root, + snapshot_id="snapshot", + repository="SynthGL/ExcelBench", + source_sha=SOURCE_SHA, + observed_at=OBSERVED_AT, + manifest_name=manifest_name, + ) + + def test_timestamp_source_and_subject_contracts_are_strict(tmp_path: Path) -> None: root = tmp_path / "results" root.mkdir() From 1d54474cc71496f924510d5f9c8a9c713ca20ea6 Mon Sep 17 00:00:00 2001 From: Wolfie Date: Sun, 30 Aug 2026 18:04:11 -0700 Subject: [PATCH 12/21] fix(evidence): reject unstable and manifest-aliased artifacts --- src/excelbench/evidence_manifest.py | 57 +++++++++++++++++++++++++---- 1 file changed, 49 insertions(+), 8 deletions(-) diff --git a/src/excelbench/evidence_manifest.py b/src/excelbench/evidence_manifest.py index 22acee2..0c91e3d 100644 --- a/src/excelbench/evidence_manifest.py +++ b/src/excelbench/evidence_manifest.py @@ -6,6 +6,7 @@ import json import os import re +import stat import tempfile import unicodedata from collections.abc import Iterable, Mapping, Sequence @@ -251,7 +252,9 @@ def _inventory( root: Path, *, excluded_root_name: str, require_artifact: bool = True ) -> list[EvidenceArtifact]: artifacts: list[EvidenceArtifact] = [] - seen_casefolded: set[str] = set() + # The manifest is excluded from its own inventory, but its portable name is + # still reserved so a differently-cased artifact cannot alias it on Windows. + seen_casefolded: set[str] = {_portable_path_key(excluded_root_name)} total_size = 0 for path in sorted(root.rglob("*"), key=lambda item: item.as_posix()): if path.parent == root and path.name == excluded_root_name: @@ -269,14 +272,12 @@ def _inventory( if portable_key in seen_casefolded: raise EvidenceManifestError(f"case-insensitive evidence path collision: {relative}") seen_casefolded.add(portable_key) - size = path.stat().st_size - if size > MAX_FILE_BYTES: - raise EvidenceManifestError(f"evidence file exceeds size limit: {relative}") + digest, size = _sha256_stable_file(path) total_size += size if total_size > MAX_TOTAL_BYTES: raise EvidenceManifestError("evidence snapshot exceeds total size limit") artifacts.append( - EvidenceArtifact(path=relative, sha256=_sha256_file(path), size_bytes=size) + EvidenceArtifact(path=relative, sha256=digest, size_bytes=size) ) if len(artifacts) > MAX_FILES: raise EvidenceManifestError("evidence snapshot exceeds file-count limit") @@ -323,12 +324,52 @@ def _portable_path_key(path: str) -> str: return "/".join(key_parts) -def _sha256_file(path: Path) -> str: +def _file_signature(metadata: os.stat_result) -> tuple[int, int, int, int, int]: + return ( + metadata.st_dev, + metadata.st_ino, + metadata.st_size, + metadata.st_mtime_ns, + metadata.st_ctime_ns, + ) + + +def _sha256_stable_file(path: Path) -> tuple[str, int]: + flags = os.O_RDONLY | getattr(os, "O_BINARY", 0) + flags |= getattr(os, "O_NOFOLLOW", 0) + try: + descriptor = os.open(path, flags) + except OSError as exc: + raise EvidenceManifestError(f"cannot open evidence file safely: {path}") from exc + digest = hashlib.sha256() - with path.open("rb") as source: + with os.fdopen(descriptor, "rb") as source: + before = os.fstat(source.fileno()) + if not stat.S_ISREG(before.st_mode): + raise EvidenceManifestError(f"evidence entry is not a regular file: {path}") + if before.st_size > MAX_FILE_BYTES: + raise EvidenceManifestError(f"evidence file exceeds size limit: {path}") + + bytes_read = 0 for chunk in iter(lambda: source.read(1024 * 1024), b""): + bytes_read += len(chunk) + if bytes_read > MAX_FILE_BYTES: + raise EvidenceManifestError(f"evidence file exceeds size limit: {path}") digest.update(chunk) - return digest.hexdigest() + after = os.fstat(source.fileno()) + + try: + current = os.stat(path, follow_symlinks=False) + except OSError as exc: + raise EvidenceManifestError(f"evidence file changed while hashing: {path}") from exc + if ( + stat.S_ISLNK(current.st_mode) + or bytes_read != before.st_size + or _file_signature(before) != _file_signature(after) + or _file_signature(after) != _file_signature(current) + ): + raise EvidenceManifestError(f"evidence file changed while hashing: {path}") + return digest.hexdigest(), before.st_size def _canonical_utc_timestamp(value: str) -> str: From adfb5bfe20ccb04d1638fe9ec85053e8ba7f403b Mon Sep 17 00:00:00 2001 From: Wolfie Date: Sun, 30 Aug 2026 18:04:27 -0700 Subject: [PATCH 13/21] test(evidence): cover unstable files and name aliases --- tests/test_evidence_manifest.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/tests/test_evidence_manifest.py b/tests/test_evidence_manifest.py index 13209e1..a32d972 100644 --- a/tests/test_evidence_manifest.py +++ b/tests/test_evidence_manifest.py @@ -5,6 +5,7 @@ import pytest from typer.testing import CliRunner +import excelbench.evidence_manifest as evidence_manifest from excelbench.cli import app from excelbench.evidence_manifest import ( EvidenceManifestError, @@ -198,6 +199,38 @@ def test_manifest_name_must_be_portable( ) +def test_manifest_name_portable_key_is_reserved(tmp_path: Path) -> None: + root = tmp_path / "results" + root.mkdir() + (root / "artifact.json").write_text("{}") + (root / "ExcelBench-Evidence.json").write_text("collision") + + with pytest.raises(EvidenceManifestError, match="case-insensitive"): + _manifest(root) + + +def test_artifact_metadata_changes_during_hashing_fail_closed( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + root = tmp_path / "results" + root.mkdir() + (root / "artifact.json").write_text("{}") + real_signature = evidence_manifest._file_signature + calls = 0 + + def changing_signature(metadata: Any) -> tuple[int, int, int, int, int]: + nonlocal calls + calls += 1 + signature = real_signature(metadata) + if calls == 2: + return (*signature[:-1], signature[-1] + 1) + return signature + + monkeypatch.setattr(evidence_manifest, "_file_signature", changing_signature) + with pytest.raises(EvidenceManifestError, match="changed while hashing"): + _manifest(root) + + def test_timestamp_source_and_subject_contracts_are_strict(tmp_path: Path) -> None: root = tmp_path / "results" root.mkdir() From 95d5c41eedda9617ff4a09de89246ecaf6d1c8c9 Mon Sep 17 00:00:00 2001 From: Wolfie Date: Sun, 30 Aug 2026 18:21:31 -0700 Subject: [PATCH 14/21] fix(evidence): require a stable portable inventory --- src/excelbench/evidence_manifest.py | 72 ++++++++++++++++++++++------- tests/test_evidence_manifest.py | 38 +++++++++++++++ 2 files changed, 93 insertions(+), 17 deletions(-) diff --git a/src/excelbench/evidence_manifest.py b/src/excelbench/evidence_manifest.py index 0c91e3d..b596b3d 100644 --- a/src/excelbench/evidence_manifest.py +++ b/src/excelbench/evidence_manifest.py @@ -252,42 +252,78 @@ def _inventory( root: Path, *, excluded_root_name: str, require_artifact: bool = True ) -> list[EvidenceArtifact]: artifacts: list[EvidenceArtifact] = [] + entries = _inventory_entries(root, excluded_root_name=excluded_root_name) + hashed_signatures: dict[str, tuple[int, int, int, int, int]] = {} + total_size = 0 + for relative, path, _initial_signature in entries: + digest, size, signature = _sha256_stable_file(path) + hashed_signatures[relative] = signature + total_size += size + if total_size > MAX_TOTAL_BYTES: + raise EvidenceManifestError("evidence snapshot exceeds total size limit") + artifacts.append(EvidenceArtifact(path=relative, sha256=digest, size_bytes=size)) + if len(artifacts) > MAX_FILES: + raise EvidenceManifestError("evidence snapshot exceeds file-count limit") + + # Hashing can be long-running. Re-scan the complete namespace so a producer + # cannot add, remove, replace, or mutate an artifact after the first walk + # and still receive a successful exact-snapshot manifest. + final_entries = _inventory_entries(root, excluded_root_name=excluded_root_name) + final_signatures = { + relative: signature for relative, _path, signature in final_entries + } + if final_signatures != hashed_signatures: + raise EvidenceManifestError( + "evidence file set or metadata changed while inventorying" + ) + if require_artifact and not artifacts: + raise EvidenceManifestError("evidence snapshot must contain at least one artifact") + return artifacts + + +def _inventory_entries( + root: Path, *, excluded_root_name: str +) -> list[tuple[str, Path, tuple[int, int, int, int, int]]]: + entries: list[tuple[str, Path, tuple[int, int, int, int, int]]] = [] # The manifest is excluded from its own inventory, but its portable name is # still reserved so a differently-cased artifact cannot alias it on Windows. seen_casefolded: set[str] = {_portable_path_key(excluded_root_name)} - total_size = 0 for path in sorted(root.rglob("*"), key=lambda item: item.as_posix()): if path.parent == root and path.name == excluded_root_name: continue - if path.is_symlink(): + try: + metadata = os.stat(path, follow_symlinks=False) + except OSError as exc: + raise EvidenceManifestError( + f"evidence entry changed while inventorying: {path.relative_to(root)}" + ) from exc + if stat.S_ISLNK(metadata.st_mode): raise EvidenceManifestError( f"symlinks are not allowed in evidence: {path.relative_to(root)}" ) - if path.is_dir(): + if stat.S_ISDIR(metadata.st_mode): continue - if not path.is_file(): + if not stat.S_ISREG(metadata.st_mode): raise EvidenceManifestError(f"non-regular evidence entry: {path.relative_to(root)}") relative = _canonical_relative_path(path.relative_to(root)) portable_key = _portable_path_key(relative) if portable_key in seen_casefolded: raise EvidenceManifestError(f"case-insensitive evidence path collision: {relative}") seen_casefolded.add(portable_key) - digest, size = _sha256_stable_file(path) - total_size += size - if total_size > MAX_TOTAL_BYTES: - raise EvidenceManifestError("evidence snapshot exceeds total size limit") - artifacts.append( - EvidenceArtifact(path=relative, sha256=digest, size_bytes=size) - ) - if len(artifacts) > MAX_FILES: + entries.append((relative, path, _file_signature(metadata))) + if len(entries) > MAX_FILES: raise EvidenceManifestError("evidence snapshot exceeds file-count limit") - if require_artifact and not artifacts: - raise EvidenceManifestError("evidence snapshot must contain at least one artifact") - return artifacts + return entries def _canonical_relative_path(path: Path) -> str: raw = path.as_posix() + try: + raw.encode("utf-8", errors="strict") + except UnicodeEncodeError as exc: + raise EvidenceManifestError( + f"evidence path is not valid UTF-8: {raw!r}" + ) from exc if "\\" in raw: raise EvidenceManifestError(f"evidence path contains a backslash: {raw!r}") normalized = unicodedata.normalize("NFC", raw) @@ -334,7 +370,9 @@ def _file_signature(metadata: os.stat_result) -> tuple[int, int, int, int, int]: ) -def _sha256_stable_file(path: Path) -> tuple[str, int]: +def _sha256_stable_file( + path: Path, +) -> tuple[str, int, tuple[int, int, int, int, int]]: flags = os.O_RDONLY | getattr(os, "O_BINARY", 0) flags |= getattr(os, "O_NOFOLLOW", 0) try: @@ -369,7 +407,7 @@ def _sha256_stable_file(path: Path) -> tuple[str, int]: or _file_signature(after) != _file_signature(current) ): raise EvidenceManifestError(f"evidence file changed while hashing: {path}") - return digest.hexdigest(), before.st_size + return digest.hexdigest(), before.st_size, _file_signature(current) def _canonical_utc_timestamp(value: str) -> str: diff --git a/tests/test_evidence_manifest.py b/tests/test_evidence_manifest.py index a32d972..317c8e3 100644 --- a/tests/test_evidence_manifest.py +++ b/tests/test_evidence_manifest.py @@ -1,4 +1,5 @@ import json +import os from pathlib import Path from typing import Any @@ -231,6 +232,43 @@ def changing_signature(metadata: Any) -> tuple[int, int, int, int, int]: _manifest(root) +def test_artifact_created_during_hashing_fails_closed( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + root = tmp_path / "results" + root.mkdir() + (root / "artifact.json").write_text("{}") + real_hash = evidence_manifest._sha256_stable_file + created = False + + def create_late_artifact(path: Path) -> tuple[str, int, tuple[int, int, int, int, int]]: + nonlocal created + result = real_hash(path) + if not created: + (root / "late.json").write_text("late") + created = True + return result + + monkeypatch.setattr(evidence_manifest, "_sha256_stable_file", create_late_artifact) + with pytest.raises(EvidenceManifestError, match="changed while inventorying"): + _manifest(root) + + +@pytest.mark.skipif(os.name == "nt", reason="requires POSIX byte filenames") +def test_non_utf8_artifact_name_fails_closed(tmp_path: Path) -> None: + root = tmp_path / "results" + root.mkdir() + byte_path = os.path.join(os.fsencode(root), b"invalid-\xff.json") + descriptor = os.open(byte_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + try: + os.write(descriptor, b"{}") + finally: + os.close(descriptor) + + with pytest.raises(EvidenceManifestError, match="not valid UTF-8"): + _manifest(root) + + def test_timestamp_source_and_subject_contracts_are_strict(tmp_path: Path) -> None: root = tmp_path / "results" root.mkdir() From bbc5c3e0f4a228b7205833a53b56eca63a705786 Mon Sep 17 00:00:00 2001 From: Wolfie Date: Sun, 30 Aug 2026 18:30:08 -0700 Subject: [PATCH 15/21] fix(evidence): bind hashes to the initial walk --- src/excelbench/evidence_manifest.py | 6 +++++- tests/test_evidence_manifest.py | 23 +++++++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/src/excelbench/evidence_manifest.py b/src/excelbench/evidence_manifest.py index b596b3d..d60b814 100644 --- a/src/excelbench/evidence_manifest.py +++ b/src/excelbench/evidence_manifest.py @@ -255,8 +255,12 @@ def _inventory( entries = _inventory_entries(root, excluded_root_name=excluded_root_name) hashed_signatures: dict[str, tuple[int, int, int, int, int]] = {} total_size = 0 - for relative, path, _initial_signature in entries: + for relative, path, initial_signature in entries: digest, size, signature = _sha256_stable_file(path) + if signature != initial_signature: + raise EvidenceManifestError( + f"evidence file changed after the initial inventory: {path}" + ) hashed_signatures[relative] = signature total_size += size if total_size > MAX_TOTAL_BYTES: diff --git a/tests/test_evidence_manifest.py b/tests/test_evidence_manifest.py index 317c8e3..d6a962a 100644 --- a/tests/test_evidence_manifest.py +++ b/tests/test_evidence_manifest.py @@ -254,6 +254,29 @@ def create_late_artifact(path: Path) -> tuple[str, int, tuple[int, int, int, int _manifest(root) +def test_later_artifact_replaced_before_hashing_fails_closed( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + root = tmp_path / "results" + root.mkdir() + (root / "a.json").write_text("first") + later = root / "z.json" + later.write_text("original") + replacement = tmp_path / "replacement.json" + replacement.write_text("replacement") + real_hash = evidence_manifest._sha256_stable_file + + def replace_later(path: Path) -> tuple[str, int, tuple[int, int, int, int, int]]: + result = real_hash(path) + if path.name == "a.json": + os.replace(replacement, later) + return result + + monkeypatch.setattr(evidence_manifest, "_sha256_stable_file", replace_later) + with pytest.raises(EvidenceManifestError, match="after the initial inventory"): + _manifest(root) + + @pytest.mark.skipif(os.name == "nt", reason="requires POSIX byte filenames") def test_non_utf8_artifact_name_fails_closed(tmp_path: Path) -> None: root = tmp_path / "results" From a1c88594997f96eab08108e9c486d551d8dcab81 Mon Sep 17 00:00:00 2001 From: Wolfie Date: Sun, 30 Aug 2026 19:07:29 -0700 Subject: [PATCH 16/21] fix(evidence): align manifest generation and read limits --- src/excelbench/evidence_manifest.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/excelbench/evidence_manifest.py b/src/excelbench/evidence_manifest.py index d60b814..b452e3b 100644 --- a/src/excelbench/evidence_manifest.py +++ b/src/excelbench/evidence_manifest.py @@ -20,6 +20,7 @@ MAX_FILES = 10_000 MAX_FILE_BYTES = 512 * 1024 * 1024 MAX_TOTAL_BYTES = 2 * 1024 * 1024 * 1024 +MAX_MANIFEST_BYTES = 4 * 1024 * 1024 _SHA256_RE = re.compile(r"^[0-9a-f]{64}$") _GIT_SHA_RE = re.compile(r"^[0-9a-f]{40}$") _WINDOWS_RESERVED_BASENAMES = { @@ -82,6 +83,13 @@ def canonical_json_bytes(value: object) -> bytes: ).encode("utf-8") +def _bounded_manifest_bytes(value: object) -> bytes: + contents = canonical_json_bytes(value) + b"\n" + if len(contents) > MAX_MANIFEST_BYTES: + raise EvidenceManifestError("manifest exceeds the 4 MiB size limit") + return contents + + def build_evidence_manifest( root: Path, *, @@ -113,7 +121,7 @@ def build_evidence_manifest( artifacts = _inventory(root, excluded_root_name=manifest_name) artifact_dicts = [artifact.to_dict() for artifact in artifacts] artifact_set_sha256 = hashlib.sha256(canonical_json_bytes(artifact_dicts)).hexdigest() - return { + manifest = { "schema": "https://excelbench.dev/schemas/evidence-manifest/v1", "schema_version": SCHEMA_VERSION, "snapshot_id": snapshot_id, @@ -125,6 +133,8 @@ def build_evidence_manifest( "total_size_bytes": sum(artifact.size_bytes for artifact in artifacts), "artifact_set_sha256": artifact_set_sha256, } + _bounded_manifest_bytes(manifest) + return manifest def verify_evidence_manifest( @@ -172,7 +182,7 @@ def read_evidence_manifest(path: Path) -> dict[str, Any]: """Read a bounded UTF-8 manifest and reject duplicate JSON keys.""" if path.is_symlink() or not path.is_file(): raise EvidenceManifestError("manifest must be a regular non-symlink file") - if path.stat().st_size > 4 * 1024 * 1024: + if path.stat().st_size > MAX_MANIFEST_BYTES: raise EvidenceManifestError("manifest exceeds the 4 MiB size limit") def no_duplicates(pairs: Iterable[tuple[str, Any]]) -> dict[str, Any]: @@ -208,7 +218,7 @@ def write_evidence_manifest( if replace: _read_existing_manifest_destination(path) _validate_manifest_document(manifest) - contents = canonical_json_bytes(dict(manifest)) + b"\n" + contents = _bounded_manifest_bytes(dict(manifest)) descriptor, temporary_name = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent) temporary = Path(temporary_name) try: From 05a75a7980d12a060ca4be3f9f88e6f62daefd97 Mon Sep 17 00:00:00 2001 From: Wolfie Date: Sun, 30 Aug 2026 19:07:31 -0700 Subject: [PATCH 17/21] fix(evidence): verify snapshot after publication --- src/excelbench/cli.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/excelbench/cli.py b/src/excelbench/cli.py index ee40a02..160e287 100644 --- a/src/excelbench/cli.py +++ b/src/excelbench/cli.py @@ -2049,6 +2049,7 @@ def evidence_manifest( EvidenceManifestError, build_evidence_manifest, parse_subject, + verify_evidence_manifest, write_evidence_manifest, ) @@ -2065,6 +2066,7 @@ def evidence_manifest( ) output = root.resolve() / manifest_name write_evidence_manifest(output, manifest, replace=replace) + verify_evidence_manifest(root, manifest, manifest_name=manifest_name) except (EvidenceManifestError, OSError) as exc: console.print(f"[red]Evidence manifest refused: {exc}[/red]") raise typer.Exit(1) from exc From f5735a24597a68ace5f58120d7d008ea9428c593 Mon Sep 17 00:00:00 2001 From: Wolfie Date: Sun, 30 Aug 2026 19:07:33 -0700 Subject: [PATCH 18/21] test(evidence): cover publish races and size bounds --- tests/test_evidence_manifest.py | 52 +++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/tests/test_evidence_manifest.py b/tests/test_evidence_manifest.py index d6a962a..ec578e7 100644 --- a/tests/test_evidence_manifest.py +++ b/tests/test_evidence_manifest.py @@ -137,6 +137,22 @@ def test_read_wraps_oversized_json_integer_errors(tmp_path: Path) -> None: read_evidence_manifest(path) +def test_generated_and_written_manifests_share_the_reader_size_limit( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + root = tmp_path / "results" + root.mkdir() + (root / "results.json").write_text("{}") + manifest = _manifest(root) + limit = len(canonical_json_bytes(manifest)) + monkeypatch.setattr(evidence_manifest, "MAX_MANIFEST_BYTES", limit) + + with pytest.raises(EvidenceManifestError, match="4 MiB size limit"): + _manifest(root) + with pytest.raises(EvidenceManifestError, match="4 MiB size limit"): + write_evidence_manifest(root / "excelbench-evidence.json", manifest) + + def test_symlinks_and_case_collisions_fail_closed(tmp_path: Path) -> None: root = tmp_path / "results" root.mkdir() @@ -385,3 +401,39 @@ def test_cli_builds_and_verifies_exact_snapshot(tmp_path: Path) -> None: ], ) assert verified.exit_code == 0, verified.output + + +def test_cli_refuses_success_if_snapshot_changes_after_publication( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + root = tmp_path / "results" + root.mkdir() + artifact = root / "results.json" + artifact.write_text("before") + real_write = evidence_manifest.write_evidence_manifest + + def write_then_mutate( + path: Path, manifest: dict[str, Any], *, replace: bool = False + ) -> None: + real_write(path, manifest, replace=replace) + artifact.write_text("after") + + monkeypatch.setattr(evidence_manifest, "write_evidence_manifest", write_then_mutate) + built = RUNNER.invoke( + app, + [ + "evidence-manifest", + "--root", + str(root), + "--snapshot-id", + "release-linux", + "--source-sha", + SOURCE_SHA, + "--observed-at", + OBSERVED_AT, + ], + ) + + assert built.exit_code == 1 + assert "Evidence manifest refused" in built.output + assert "changed=['results.json']" in built.output From dbce5565f19d041e6a11ab0029946ae657c35790 Mon Sep 17 00:00:00 2001 From: Wolfie Date: Sun, 30 Aug 2026 19:26:47 -0700 Subject: [PATCH 19/21] fix(evidence): harden manifest validation and reads --- src/excelbench/evidence_manifest.py | 129 ++++++++++++++++++++++++---- tests/test_evidence_manifest.py | 108 +++++++++++++++++++++++ 2 files changed, 220 insertions(+), 17 deletions(-) diff --git a/src/excelbench/evidence_manifest.py b/src/excelbench/evidence_manifest.py index b452e3b..ef2e8ef 100644 --- a/src/excelbench/evidence_manifest.py +++ b/src/excelbench/evidence_manifest.py @@ -74,13 +74,18 @@ def to_dict(self) -> dict[str, str | int]: def canonical_json_bytes(value: object) -> bytes: """Return the one wire representation used for hashing and publication.""" - return json.dumps( - value, - ensure_ascii=False, - allow_nan=False, - sort_keys=True, - separators=(",", ":"), - ).encode("utf-8") + try: + return json.dumps( + value, + ensure_ascii=False, + allow_nan=False, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8", errors="strict") + except UnicodeEncodeError as exc: + raise EvidenceManifestError( + "manifest contains text that is not valid UTF-8" + ) from exc def _bounded_manifest_bytes(value: object) -> bytes: @@ -180,10 +185,7 @@ def verify_evidence_manifest( def read_evidence_manifest(path: Path) -> dict[str, Any]: """Read a bounded UTF-8 manifest and reject duplicate JSON keys.""" - if path.is_symlink() or not path.is_file(): - raise EvidenceManifestError("manifest must be a regular non-symlink file") - if path.stat().st_size > MAX_MANIFEST_BYTES: - raise EvidenceManifestError("manifest exceeds the 4 MiB size limit") + contents = _read_manifest_bytes(path) def no_duplicates(pairs: Iterable[tuple[str, Any]]) -> dict[str, Any]: value: dict[str, Any] = {} @@ -194,16 +196,73 @@ def no_duplicates(pairs: Iterable[tuple[str, Any]]) -> dict[str, Any]: return value try: - decoded = json.loads(path.read_text(encoding="utf-8"), object_pairs_hook=no_duplicates) + decoded = json.loads( + contents.decode("utf-8", errors="strict"), + object_pairs_hook=no_duplicates, + ) except EvidenceManifestError: raise - except (OSError, UnicodeDecodeError, ValueError) as exc: + except (UnicodeDecodeError, ValueError) as exc: raise EvidenceManifestError("manifest is not valid UTF-8 JSON") from exc if not isinstance(decoded, dict): raise EvidenceManifestError("manifest root must be an object") + _validate_serialized_text(decoded) return decoded +def _read_manifest_bytes(path: Path) -> bytes: + """Read one stable regular file through a bounded no-follow descriptor.""" + flags = os.O_RDONLY | getattr(os, "O_BINARY", 0) + flags |= getattr(os, "O_NOFOLLOW", 0) + try: + descriptor = os.open(path, flags) + except OSError as exc: + raise EvidenceManifestError( + "manifest must be a regular non-symlink file" + ) from exc + + try: + before = os.fstat(descriptor) + if not stat.S_ISREG(before.st_mode): + raise EvidenceManifestError( + "manifest must be a regular non-symlink file" + ) + if before.st_size > MAX_MANIFEST_BYTES: + raise EvidenceManifestError("manifest exceeds the 4 MiB size limit") + chunks: list[bytes] = [] + remaining = MAX_MANIFEST_BYTES + 1 + while remaining: + chunk = os.read(descriptor, min(1024 * 1024, remaining)) + if not chunk: + break + chunks.append(chunk) + remaining -= len(chunk) + contents = b"".join(chunks) + after = os.fstat(descriptor) + except EvidenceManifestError: + raise + except OSError as exc: + raise EvidenceManifestError("manifest changed while reading") from exc + finally: + os.close(descriptor) + + if len(contents) > MAX_MANIFEST_BYTES: + raise EvidenceManifestError("manifest exceeds the 4 MiB size limit") + try: + current = os.stat(path, follow_symlinks=False) + except OSError as exc: + raise EvidenceManifestError("manifest changed while reading") from exc + if ( + stat.S_ISLNK(current.st_mode) + or not stat.S_ISREG(current.st_mode) + or len(contents) != before.st_size + or _file_signature(before) != _file_signature(after) + or _file_signature(after) != _file_signature(current) + ): + raise EvidenceManifestError("manifest changed while reading") + return contents + + def write_evidence_manifest( path: Path, manifest: Mapping[str, Any], *, replace: bool = False ) -> None: @@ -506,6 +565,8 @@ def _validate_manifest_shape(manifest: Mapping[str, Any]) -> None: or artifact_count < 1 ): raise EvidenceManifestError("artifact_count must be a positive integer") + if artifact_count > MAX_FILES: + raise EvidenceManifestError("artifact_count exceeds the file-count limit") total_size_bytes = manifest["total_size_bytes"] if ( not isinstance(total_size_bytes, int) @@ -513,6 +574,8 @@ def _validate_manifest_shape(manifest: Mapping[str, Any]) -> None: or total_size_bytes < 0 ): raise EvidenceManifestError("total_size_bytes must be a non-negative integer") + if total_size_bytes > MAX_TOTAL_BYTES: + raise EvidenceManifestError("total_size_bytes exceeds the total size limit") artifact_set_sha256 = _string( manifest["artifact_set_sha256"], "artifact_set_sha256" ) @@ -521,6 +584,7 @@ def _validate_manifest_shape(manifest: Mapping[str, Any]) -> None: def _validate_manifest_document(manifest: Mapping[str, Any]) -> list[EvidenceArtifact]: + _validate_serialized_text(manifest) _validate_manifest_shape(manifest) subjects = [ _subject_from_mapping(value, index) @@ -532,6 +596,13 @@ def _validate_manifest_document(manifest: Mapping[str, Any]) -> list[EvidenceArt _artifact_from_mapping(value, index) for index, value in enumerate(manifest["artifacts"]) ] + if len(artifacts) > MAX_FILES: + raise EvidenceManifestError("manifest exceeds the file-count limit") + if any(artifact.size_bytes > MAX_FILE_BYTES for artifact in artifacts): + raise EvidenceManifestError("manifest artifact exceeds the per-file size limit") + total_size = sum(artifact.size_bytes for artifact in artifacts) + if total_size > MAX_TOTAL_BYTES: + raise EvidenceManifestError("manifest exceeds the total size limit") if artifacts != sorted(artifacts): raise EvidenceManifestError("artifacts must be sorted by canonical path") if len({_portable_path_key(artifact.path) for artifact in artifacts}) != len(artifacts): @@ -542,9 +613,7 @@ def _validate_manifest_document(manifest: Mapping[str, Any]) -> list[EvidenceArt raise EvidenceManifestError("artifact_set_sha256 does not match the artifact inventory") if manifest["artifact_count"] != len(artifacts): raise EvidenceManifestError("artifact_count does not match the artifact inventory") - if manifest["total_size_bytes"] != sum( - artifact.size_bytes for artifact in artifacts - ): + if manifest["total_size_bytes"] != total_size: raise EvidenceManifestError("total_size_bytes does not match the artifact inventory") return artifacts @@ -577,7 +646,7 @@ def _artifact_from_mapping(value: Any, index: int) -> EvidenceArtifact: if _SHA256_RE.fullmatch(digest) is None: raise EvidenceManifestError(f"artifacts[{index}].sha256 is invalid") size = item["size_bytes"] - if not isinstance(size, int) or isinstance(size, bool) or size < 0 or size > MAX_FILE_BYTES: + if not isinstance(size, int) or isinstance(size, bool) or size < 0: raise EvidenceManifestError(f"artifacts[{index}].size_bytes is invalid") return EvidenceArtifact(path=path, sha256=digest, size_bytes=size) @@ -607,6 +676,9 @@ def _safe_manifest_name(value: str) -> str: def _required_text(value: str, label: str) -> str: + if not isinstance(value, str): + raise EvidenceManifestError(f"{label} must be a string") + _validate_utf8_text(value, label) stripped = value.strip() if not stripped or any(character in stripped for character in "\r\n\x00"): raise EvidenceManifestError(f"{label} must be non-empty single-line text") @@ -616,10 +688,33 @@ def _required_text(value: str, label: str) -> str: def _mapping(value: Any, label: str) -> Mapping[str, Any]: if not isinstance(value, Mapping) or not all(isinstance(key, str) for key in value): raise EvidenceManifestError(f"{label} must be an object with string keys") + for key in value: + _validate_utf8_text(key, f"{label} key") return value def _string(value: Any, label: str) -> str: if not isinstance(value, str): raise EvidenceManifestError(f"{label} must be a string") + _validate_utf8_text(value, label) return value + + +def _validate_utf8_text(value: str, label: str) -> None: + try: + value.encode("utf-8", errors="strict") + except UnicodeEncodeError as exc: + raise EvidenceManifestError(f"{label} must be valid UTF-8 text") from exc + + +def _validate_serialized_text(value: object) -> None: + if isinstance(value, str): + _validate_utf8_text(value, "manifest text") + elif isinstance(value, Mapping): + for key, item in value.items(): + if isinstance(key, str): + _validate_utf8_text(key, "manifest object key") + _validate_serialized_text(item) + elif isinstance(value, (list, tuple)): + for item in value: + _validate_serialized_text(item) diff --git a/tests/test_evidence_manifest.py b/tests/test_evidence_manifest.py index ec578e7..9eeb6d2 100644 --- a/tests/test_evidence_manifest.py +++ b/tests/test_evidence_manifest.py @@ -137,6 +137,88 @@ def test_read_wraps_oversized_json_integer_errors(tmp_path: Path) -> None: read_evidence_manifest(path) +@pytest.mark.parametrize( + "field", + ["snapshot_id", "source.repository", "subjects[0].name"], +) +def test_lone_surrogate_metadata_is_a_controlled_utf8_refusal( + tmp_path: Path, + field: str, +) -> None: + root = tmp_path / "results" + root.mkdir() + (root / "results.json").write_text("{}") + manifest = _manifest(root) + if field == "snapshot_id": + manifest["snapshot_id"] = "invalid-\ud800" + elif field == "source.repository": + manifest["source"]["repository"] = "invalid-\ud800" + else: + manifest["subjects"][0]["name"] = "invalid-\ud800" + + with pytest.raises(EvidenceManifestError, match="valid UTF-8"): + canonical_json_bytes(manifest) + with pytest.raises(EvidenceManifestError, match="valid UTF-8"): + write_evidence_manifest(tmp_path / "refused.json", manifest) + + path = root / "excelbench-evidence.json" + path.write_text(json.dumps(manifest), encoding="utf-8") + with pytest.raises(EvidenceManifestError, match="valid UTF-8"): + read_evidence_manifest(path) + + verified = RUNNER.invoke(app, ["verify-evidence", "--root", str(root)]) + assert verified.exit_code == 1 + assert "Evidence verification failed" in verified.output + assert "valid UTF-8" in verified.output + + +def test_read_rejects_oversized_descriptor_content( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + path = tmp_path / "manifest.json" + path.write_bytes(b"{" + b" " * 32) + monkeypatch.setattr(evidence_manifest, "MAX_MANIFEST_BYTES", 16) + + with pytest.raises(EvidenceManifestError, match="4 MiB size limit"): + read_evidence_manifest(path) + + +def test_read_detects_path_replacement_after_open( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + path = tmp_path / "manifest.json" + path.write_text("{}") + replacement = tmp_path / "replacement.json" + replacement.write_text('{"replacement":true}') + real_fstat = os.fstat + calls = 0 + + def replace_after_read(descriptor: int) -> os.stat_result: + nonlocal calls + metadata = real_fstat(descriptor) + calls += 1 + if calls == 2: + os.replace(replacement, path) + return metadata + + monkeypatch.setattr(os, "fstat", replace_after_read) + with pytest.raises(EvidenceManifestError, match="changed while reading"): + read_evidence_manifest(path) + + +def test_read_rejects_manifest_symlink(tmp_path: Path) -> None: + target = tmp_path / "target.json" + target.write_text("{}") + path = tmp_path / "manifest.json" + try: + path.symlink_to(target) + except OSError: + pytest.skip("symlinks unavailable") + + with pytest.raises(EvidenceManifestError, match="regular non-symlink"): + read_evidence_manifest(path) + + def test_generated_and_written_manifests_share_the_reader_size_limit( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: @@ -153,6 +235,32 @@ def test_generated_and_written_manifests_share_the_reader_size_limit( write_evidence_manifest(root / "excelbench-evidence.json", manifest) +@pytest.mark.parametrize( + ("limit_name", "limit", "message"), + [ + ("MAX_FILES", 1, "file-count limit"), + ("MAX_FILE_BYTES", 4, "per-file size limit"), + ("MAX_TOTAL_BYTES", 8, "total size limit"), + ], +) +def test_writer_rejects_manifest_inventory_beyond_verifier_limits( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + limit_name: str, + limit: int, + message: str, +) -> None: + root = tmp_path / "results" + root.mkdir() + (root / "a.json").write_text("four") + (root / "b.json").write_text("five!") + manifest = _manifest(root) + monkeypatch.setattr(evidence_manifest, limit_name, limit) + + with pytest.raises(EvidenceManifestError, match=message): + write_evidence_manifest(root / "excelbench-evidence.json", manifest) + + def test_symlinks_and_case_collisions_fail_closed(tmp_path: Path) -> None: root = tmp_path / "results" root.mkdir() From 4ee48cb99b5532e65a61741aacbd691a93330b4a Mon Sep 17 00:00:00 2001 From: Wolfie Date: Sun, 30 Aug 2026 19:36:07 -0700 Subject: [PATCH 20/21] fix(evidence): close verification race windows --- src/excelbench/evidence_manifest.py | 57 +++++++++++++++++------- tests/test_evidence_manifest.py | 69 +++++++++++++++++++++++++++++ 2 files changed, 110 insertions(+), 16 deletions(-) diff --git a/src/excelbench/evidence_manifest.py b/src/excelbench/evidence_manifest.py index ef2e8ef..8fd3dd3 100644 --- a/src/excelbench/evidence_manifest.py +++ b/src/excelbench/evidence_manifest.py @@ -150,13 +150,16 @@ def verify_evidence_manifest( manifest_name: str = DEFAULT_MANIFEST_NAME, ) -> None: """Fail unless ``manifest`` exactly describes every regular file in ``root``.""" - expected = _validate_manifest_document(manifest) + manifest_name = _safe_manifest_name(manifest_name) + expected = _validate_manifest_document( + manifest, + reserved_manifest_name=manifest_name, + ) source = _mapping(manifest["source"], "source") source_sha = _string(source.get("commit"), "source.commit") if expected_source_sha is not None and source_sha != expected_source_sha: raise EvidenceManifestError("manifest source commit does not match expected source SHA") root = root.resolve(strict=True) - manifest_name = _safe_manifest_name(manifest_name) destination_manifest = _read_existing_manifest_destination(root / manifest_name) if ( destination_manifest is not None @@ -168,6 +171,13 @@ def verify_evidence_manifest( excluded_root_name=manifest_name, require_artifact=False, ) + final_destination_manifest = _read_existing_manifest_destination(root / manifest_name) + if (destination_manifest is None) != (final_destination_manifest is None) or ( + final_destination_manifest is not None + and canonical_json_bytes(final_destination_manifest) + != canonical_json_bytes(dict(manifest)) + ): + raise EvidenceManifestError("manifest file changed while verifying evidence") if actual != expected: actual_by_path = {artifact.path: artifact for artifact in actual} expected_by_path = {artifact.path: artifact for artifact in expected} @@ -202,7 +212,7 @@ def no_duplicates(pairs: Iterable[tuple[str, Any]]) -> dict[str, Any]: ) except EvidenceManifestError: raise - except (UnicodeDecodeError, ValueError) as exc: + except (RecursionError, UnicodeDecodeError, ValueError) as exc: raise EvidenceManifestError("manifest is not valid UTF-8 JSON") from exc if not isinstance(decoded, dict): raise EvidenceManifestError("manifest root must be an object") @@ -276,7 +286,7 @@ def write_evidence_manifest( raise EvidenceManifestError("manifest destination must not be a symlink") if replace: _read_existing_manifest_destination(path) - _validate_manifest_document(manifest) + _validate_manifest_document(manifest, reserved_manifest_name=name) contents = _bounded_manifest_bytes(dict(manifest)) descriptor, temporary_name = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent) temporary = Path(temporary_name) @@ -583,7 +593,11 @@ def _validate_manifest_shape(manifest: Mapping[str, Any]) -> None: raise EvidenceManifestError("artifact_set_sha256 must be a lowercase SHA-256 digest") -def _validate_manifest_document(manifest: Mapping[str, Any]) -> list[EvidenceArtifact]: +def _validate_manifest_document( + manifest: Mapping[str, Any], + *, + reserved_manifest_name: str | None = None, +) -> list[EvidenceArtifact]: _validate_serialized_text(manifest) _validate_manifest_shape(manifest) subjects = [ @@ -607,6 +621,12 @@ def _validate_manifest_document(manifest: Mapping[str, Any]) -> list[EvidenceArt raise EvidenceManifestError("artifacts must be sorted by canonical path") if len({_portable_path_key(artifact.path) for artifact in artifacts}) != len(artifacts): raise EvidenceManifestError("manifest contains case-insensitive path collisions") + if reserved_manifest_name is not None: + reserved_key = _portable_path_key(_safe_manifest_name(reserved_manifest_name)) + if any(_portable_path_key(artifact.path) == reserved_key for artifact in artifacts): + raise EvidenceManifestError( + "manifest artifact collides with the manifest destination" + ) artifact_dicts = [artifact.to_dict() for artifact in artifacts] digest = hashlib.sha256(canonical_json_bytes(artifact_dicts)).hexdigest() if manifest["artifact_set_sha256"] != digest: @@ -627,7 +647,10 @@ def _read_existing_manifest_destination(path: Path) -> dict[str, Any] | None: raise EvidenceManifestError("manifest destination must be a regular file") try: manifest = read_evidence_manifest(path) - _validate_manifest_document(manifest) + _validate_manifest_document( + manifest, + reserved_manifest_name=path.name, + ) except EvidenceManifestError as exc: raise EvidenceManifestError( "manifest destination exists but is not a valid evidence manifest" @@ -708,13 +731,15 @@ def _validate_utf8_text(value: str, label: str) -> None: def _validate_serialized_text(value: object) -> None: - if isinstance(value, str): - _validate_utf8_text(value, "manifest text") - elif isinstance(value, Mapping): - for key, item in value.items(): - if isinstance(key, str): - _validate_utf8_text(key, "manifest object key") - _validate_serialized_text(item) - elif isinstance(value, (list, tuple)): - for item in value: - _validate_serialized_text(item) + pending = [value] + while pending: + item = pending.pop() + if isinstance(item, str): + _validate_utf8_text(item, "manifest text") + elif isinstance(item, Mapping): + for key, nested in item.items(): + if isinstance(key, str): + _validate_utf8_text(key, "manifest object key") + pending.append(nested) + elif isinstance(item, (list, tuple)): + pending.extend(item) diff --git a/tests/test_evidence_manifest.py b/tests/test_evidence_manifest.py index 9eeb6d2..e9548b5 100644 --- a/tests/test_evidence_manifest.py +++ b/tests/test_evidence_manifest.py @@ -1,3 +1,4 @@ +import hashlib import json import os from pathlib import Path @@ -77,6 +78,38 @@ def test_verification_rejects_missing_extra_and_changed_files(tmp_path: Path) -> verify_evidence_manifest(root, manifest) +def test_verification_rechecks_manifest_after_inventory( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + root = tmp_path / "results" + root.mkdir() + (root / "results.json").write_text("{}") + manifest = _manifest(root) + path = root / "excelbench-evidence.json" + write_evidence_manifest(path, manifest) + replacement = json.loads(json.dumps(manifest)) + replacement["snapshot_id"] = "replacement-snapshot" + real_inventory = evidence_manifest._inventory + + def replace_after_inventory( + inventory_root: Path, + *, + excluded_root_name: str, + require_artifact: bool = True, + ) -> list[evidence_manifest.EvidenceArtifact]: + artifacts = real_inventory( + inventory_root, + excluded_root_name=excluded_root_name, + require_artifact=require_artifact, + ) + path.write_bytes(canonical_json_bytes(replacement) + b"\n") + return artifacts + + monkeypatch.setattr(evidence_manifest, "_inventory", replace_after_inventory) + with pytest.raises(EvidenceManifestError, match="changed while verifying evidence"): + verify_evidence_manifest(root, manifest) + + def test_manifest_file_is_excluded_and_atomic_no_clobber_is_default(tmp_path: Path) -> None: root = tmp_path / "results" root.mkdir() @@ -219,6 +252,19 @@ def test_read_rejects_manifest_symlink(tmp_path: Path) -> None: read_evidence_manifest(path) +def test_deeply_nested_json_is_a_controlled_refusal(tmp_path: Path) -> None: + root = tmp_path / "results" + root.mkdir() + path = root / "excelbench-evidence.json" + path.write_text('{"unexpected":' + "[" * 2_000 + "null" + "]" * 2_000 + "}") + + assert isinstance(read_evidence_manifest(path), dict) + verified = RUNNER.invoke(app, ["verify-evidence", "--root", str(root)]) + assert verified.exit_code == 1 + assert "Evidence verification failed" in verified.output + assert "manifest fields must be exactly" in verified.output + + def test_generated_and_written_manifests_share_the_reader_size_limit( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: @@ -261,6 +307,29 @@ def test_writer_rejects_manifest_inventory_beyond_verifier_limits( write_evidence_manifest(root / "excelbench-evidence.json", manifest) +@pytest.mark.parametrize( + "artifact_path", + ["excelbench-evidence.json", "ExcelBench-Evidence.json"], +) +def test_writer_rejects_manifest_destination_artifact_alias( + tmp_path: Path, + artifact_path: str, +) -> None: + root = tmp_path / "results" + root.mkdir() + (root / "artifact.json").write_text("{}") + manifest = _manifest(root) + manifest["artifacts"][0]["path"] = artifact_path + manifest["artifact_set_sha256"] = hashlib.sha256( + canonical_json_bytes(manifest["artifacts"]) + ).hexdigest() + + destination = root / "excelbench-evidence.json" + with pytest.raises(EvidenceManifestError, match="collides with the manifest destination"): + write_evidence_manifest(destination, manifest) + assert not destination.exists() + + def test_symlinks_and_case_collisions_fail_closed(tmp_path: Path) -> None: root = tmp_path / "results" root.mkdir() From a16160ab7d7be70724b33123d7e1682ce9b22da7 Mon Sep 17 00:00:00 2001 From: Wolfie Date: Sun, 30 Aug 2026 19:40:26 -0700 Subject: [PATCH 21/21] test(evidence): keep deep refusal cross-version --- tests/test_evidence_manifest.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/test_evidence_manifest.py b/tests/test_evidence_manifest.py index e9548b5..fcb4cd8 100644 --- a/tests/test_evidence_manifest.py +++ b/tests/test_evidence_manifest.py @@ -258,11 +258,14 @@ def test_deeply_nested_json_is_a_controlled_refusal(tmp_path: Path) -> None: path = root / "excelbench-evidence.json" path.write_text('{"unexpected":' + "[" * 2_000 + "null" + "]" * 2_000 + "}") - assert isinstance(read_evidence_manifest(path), dict) verified = RUNNER.invoke(app, ["verify-evidence", "--root", str(root)]) assert verified.exit_code == 1 assert "Evidence verification failed" in verified.output - assert "manifest fields must be exactly" in verified.output + assert "Traceback" not in verified.output + assert ( + "manifest is not valid UTF-8 JSON" in verified.output + or "manifest fields must be exactly" in verified.output + ) def test_generated_and_written_manifests_share_the_reader_size_limit(