From e39a01006b3db7b574ef2ee43baba130a8218e1d Mon Sep 17 00:00:00 2001 From: Tazeem Mahashin Date: Mon, 3 Aug 2026 11:37:05 -0400 Subject: [PATCH] Add captured qualification evidence workflow --- CHANGELOG.md | 6 + README.md | 9 + docs/api-stability.md | 5 + docs/commands.md | 21 + docs/conformance.md | 78 ++- src/tracebook/conformance/cli.py | 84 +++- src/tracebook/conformance/evidence.py | 676 ++++++++++++++++++++++++++ src/tracebook/conformance/external.py | 10 + src/tracebook/conformance/model.py | 62 ++- tests/test_conformance_evidence.py | 502 +++++++++++++++++++ 10 files changed, 1448 insertions(+), 5 deletions(-) create mode 100644 src/tracebook/conformance/evidence.py create mode 100644 tests/test_conformance_evidence.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 1ec4d48..b363249 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,12 @@ The project follows [Keep a Changelog](https://keepachangelog.com/) conventions. ## Unreleased +- Added a captured two-run qualification workflow. Candidate adapters can now + report optional revision and snapshot metadata, candidate-facing commands can + enforce a task-pinned identity before the first event, and `evidence-init` / + `evidence-verify` prepare clean roots and emit a grading-ready manifest only + for unchanged, complete, canonical, byte-identical qualification bundles. + ## 0.6.0 - 2026-07-27 - Fixed conformance CLI exit classification across single runs, suites, diff --git a/README.md b/README.md index aa05b63..4b0513d 100644 --- a/README.md +++ b/README.md @@ -189,6 +189,15 @@ these are separate version boundaries. `fifo-limit-v1` qualification does not run STP or pro-rata cases, so unsupported features stay explicit without becoming false failures. +For release evidence, do not rely on one successful workspace. Start a captured +pair with `tracebook-conformance evidence-init`, run the canonical qualification +once in each generated clean root with the plan's pinned candidate name, +revision, and snapshot, then run `tracebook-conformance evidence-verify`. The +verifier emits a compact `evidence-manifest.json` only when both source trees +remain unchanged and both qualification bundles agree on terminal result, +candidate metadata, deterministic IDs, counts, coverage, and every artifact +byte. See the [captured evidence workflow](https://github.com/Taz33m/tracebook/blob/main/docs/conformance.md#captured-two-run-evidence). + [Read the research-grounded roadmap and adoption experiment](https://github.com/Taz33m/tracebook/blob/main/docs/research-roadmap.md). ## Architecture diff --git a/docs/api-stability.md b/docs/api-stability.md index 390c444..e4cc29d 100644 --- a/docs/api-stability.md +++ b/docs/api-stability.md @@ -30,6 +30,11 @@ normalization, or hashing requires an explicit protocol/schema version decision. `suite_hash` binds case configuration and fixture identity; an intentional suite edit must update it. +- Protocol-v1 engine metadata may add the optional `revision` and `snapshot_id` + identity fields. Ordinary adapters may omit them; a command using the three + task-pinned `--candidate-*` flags requires an exact match before any event is + sent. Evidence plan schema v1 and evidence manifest schema v1 are public + artifacts for the canonical captured two-run qualification workflow. - Campaign artifact schema version 1 is public. Campaign generator version 2, the built-in versioned profile definitions, seed derivation, and trace hashes are reproducibility contracts. An intentional generation change requires a diff --git a/docs/commands.md b/docs/commands.md index 945587a..252658c 100644 --- a/docs/commands.md +++ b/docs/commands.md @@ -132,6 +132,27 @@ automatically minimized failure. Use this command for a first integration or a profile-level CI claim; use `suite` when intentionally comparing every broader fixed semantic surface. +Prepare and verify a release-grade two-run evidence pair: + +```bash +tracebook-conformance evidence-init /path/to/candidate \ + --workspace .tracebook/release-evidence \ + --candidate-name owner/repository \ + --candidate-revision REVISION + +# Run the canonical qualification from evidence-plan.json in both generated +# roots, with all three --candidate-* identity flags, then: +tracebook-conformance evidence-verify \ + .tracebook/release-evidence/evidence-plan.json +``` + +`evidence-init` refuses an existing workspace and creates independent candidate, +adapter, build, cache, and qualification paths for `run-1` and `run-2`. +`evidence-verify` writes `evidence-manifest.json` only after both canonical +bundles are qualified, byte-identical, and bound to the unchanged pinned +candidate. See [Captured Two-Run Evidence](conformance.md#captured-two-run-evidence) +for the exact qualification command and trust boundary. + Copy and run the standard suite: ```bash diff --git a/docs/conformance.md b/docs/conformance.md index c4e94ce..2246cd0 100644 --- a/docs/conformance.md +++ b/docs/conformance.md @@ -135,6 +135,64 @@ not exchange certification and does not imply support for unselected suite cases. Selection version 1 is part of the artifact identity and will not change silently. +## Captured Two-Run Evidence + +Use the evidence workflow when a qualification result will support a release, +benchmark, or external claim. It pins the candidate before adapter readiness and +requires the same canonical qualification from two prepared roots. + +```bash +REVISION=$(git -C /path/to/candidate rev-parse HEAD) +WORKSPACE=$PWD/.tracebook/release-evidence + +tracebook-conformance evidence-init /path/to/candidate \ + --workspace "$WORKSPACE" \ + --candidate-name owner/repository \ + --candidate-revision "$REVISION" +``` + +`evidence-init` strips `.git` from the captured source identity, copies that +exact tree into `runs/run-1/candidate` and `runs/run-2/candidate`, and creates a +separate empty `adapter`, `build`, and `cache` directory for each run. It writes +`evidence-plan.json` last and refuses an existing or source-nested workspace. +Use each run's paths for that run's adapter and build; do not share generated +files between the two runs. + +The command prints the candidate snapshot ID. The adapter's `ready` frame must +report that ID and the task-pinned name and revision. Run the plan's canonical +contract in each root, changing only the run number and adapter command: + +```bash +SNAPSHOT=$(python -c \ + 'import json,sys; print(json.load(open(sys.argv[1]))["candidate"]["snapshot_id"])' \ + "$WORKSPACE/evidence-plan.json") + +tracebook-conformance qualify \ + --profile fifo-limit-v1 --suite-version v2 \ + --seed 42 --traces 25 --events-per-trace 200 --max-minimize-runs 100 \ + --candidate-name owner/repository \ + --candidate-revision "$REVISION" \ + --candidate-snapshot "$SNAPSHOT" \ + --candidate-cmd "$WORKSPACE/runs/run-1/adapter/launch" \ + --output-dir "$WORKSPACE/runs/run-1/qualification" +``` + +Repeat for `run-2`, using only its candidate, adapter, build, cache, and +qualification paths. Then verify the pair: + +```bash +tracebook-conformance evidence-verify "$WORKSPACE/evidence-plan.json" +``` + +Verification fails closed if either captured tree changed; either bundle is +missing, noncanonical, incomplete, or unqualified; candidate identity differs; +or terminal result, deterministic IDs, counts, semantic coverage, or artifact +bytes differ between runs. Success writes one exclusive +`evidence-manifest.json` at the workspace root. The workflow prepares and checks +the filesystem contract, but it does not sandbox a compiler or prove that an +adapter avoided undeclared external caches; CI or a disposable host remains the +strongest execution boundary. + ## Differential Campaigns Campaigns generate stateful traces, compare them one at a time, and stop at the @@ -346,9 +404,15 @@ Host to candidate, once: Candidate to host: ```json -{"type":"ready","protocol":"tracebook.conformance","protocol_version":1,"engine":{"name":"my-engine","version":"1.4.2","language":"Rust"}} +{"type":"ready","protocol":"tracebook.conformance","protocol_version":1,"engine":{"name":"owner/repository","version":"1.4.2","language":"Rust","revision":"8f31c2a","snapshot_id":"sha256:..."}} ``` +`revision` and `snapshot_id` are optional for ordinary comparisons. When +`--candidate-name`, `--candidate-revision`, and `--candidate-snapshot` are +supplied, all three flags are required together and the `ready` metadata must +match before Tracebook sends the first event. Captured two-run evidence always +uses the pinned form. + Host to candidate for each event: ```json @@ -429,7 +493,13 @@ from tracebook.conformance import EngineMetadata, serve_stdio class MyAdapter: def __init__(self, config): - self.metadata = EngineMetadata("my-engine", "1.0", "Python") + self.metadata = EngineMetadata( + "owner/repository", + "1.0", + "Python", + revision="8f31c2a", + snapshot_id="sha256:...", + ) self.config = config def apply(self, event, index): @@ -447,6 +517,10 @@ class MyAdapter: raise SystemExit(serve_stdio(MyAdapter)) ``` +The last two fields are needed only when the invoking command pins candidate +identity. In a captured evidence run, populate them from the immutable task or +`evidence-plan.json`, not by inspecting the host after the adapter starts. + [`examples/conformance_adapter.py`](../examples/conformance_adapter.py) is a runnable reference. Non-Python adapters implement the same frames directly. diff --git a/src/tracebook/conformance/cli.py b/src/tracebook/conformance/cli.py index bc3b51c..0a0a085 100644 --- a/src/tracebook/conformance/cli.py +++ b/src/tracebook/conformance/cli.py @@ -20,11 +20,12 @@ write_campaign_corpus, ) from .compare import run_conformance +from .evidence import prepare_evidence_workspace, write_evidence_manifest from .external import AdapterProtocolError, ExternalProcessAdapterFactory from .exit_codes import exit_code_for_artifact from .junit import write_junit from .minimize import minimize_failing_trace -from .model import ConformanceConfig, ConformanceError +from .model import ConformanceConfig, ConformanceError, PinnedCandidateIdentity from .qualification import _QualificationOutputReservation, run_qualification from .reproduce import ( discover_failure_metadata, @@ -54,6 +55,18 @@ def _add_config_arguments(parser: argparse.ArgumentParser) -> None: def _add_candidate_arguments(parser: argparse.ArgumentParser) -> None: parser.add_argument("--timeout", type=float, default=5.0) + parser.add_argument( + "--candidate-name", + help="Task-pinned candidate name expected in the ready frame.", + ) + parser.add_argument( + "--candidate-revision", + help="Task-pinned revision expected in the ready frame.", + ) + parser.add_argument( + "--candidate-snapshot", + help="Task-pinned snapshot ID expected in the ready frame.", + ) candidate = parser.add_mutually_exclusive_group(required=True) candidate.add_argument( "--candidate-cmd", @@ -163,6 +176,23 @@ def _build_parser() -> argparse.ArgumentParser: reproduce.add_argument("--junit-output") _add_config_arguments(reproduce) _add_candidate_arguments(reproduce) + + evidence_init = commands.add_parser( + "evidence-init", + help="Create two clean roots for a captured qualification pair.", + ) + evidence_init.add_argument("candidate_source") + evidence_init.add_argument("--workspace", required=True) + evidence_init.add_argument("--candidate-name", required=True) + evidence_init.add_argument("--candidate-revision", required=True) + evidence_init.add_argument("--expected-snapshot") + + evidence_verify = commands.add_parser( + "evidence-verify", + help="Verify two captured qualification bundles and write one manifest.", + ) + evidence_verify.add_argument("plan") + evidence_verify.add_argument("--output") return parser @@ -174,7 +204,32 @@ def _candidate_factory(args) -> ExternalProcessAdapterFactory: command = command[1:] if not command: raise ConformanceError("--candidate requires a command") - return ExternalProcessAdapterFactory(command, timeout_seconds=args.timeout) + identity_values = ( + args.candidate_name, + args.candidate_revision, + args.candidate_snapshot, + ) + if any(value is not None for value in identity_values) and not all( + value is not None for value in identity_values + ): + raise ConformanceError( + "--candidate-name, --candidate-revision, and --candidate-snapshot " + "must be provided together" + ) + expected_identity = ( + PinnedCandidateIdentity( + name=args.candidate_name, + revision=args.candidate_revision, + snapshot_id=args.candidate_snapshot, + ) + if all(value is not None for value in identity_values) + else None + ) + return ExternalProcessAdapterFactory( + command, + timeout_seconds=args.timeout, + expected_identity=expected_identity, + ) def _config(args) -> ConformanceConfig: @@ -253,6 +308,31 @@ def main(argv: Optional[List[str]] = None) -> int: print(f"Suite id: {suite.suite_id}") print(f"Cases: {len(suite.cases)}") return 0 + if args.command == "evidence-init": + plan_path = prepare_evidence_workspace( + args.candidate_source, + args.workspace, + candidate_name=args.candidate_name, + candidate_revision=args.candidate_revision, + expected_snapshot=args.expected_snapshot, + ) + plan = json.loads(plan_path.read_text(encoding="utf-8")) + print(f"Evidence plan written: {plan_path}") + print(f"Candidate snapshot: {plan['candidate']['snapshot_id']}") + for run in plan["runs"]: + print( + f"{run['run_id']}: candidate={run['candidate_root']} " + f"qualification={run['qualification_dir']}" + ) + return 0 + if args.command == "evidence-verify": + manifest_path = write_evidence_manifest(args.plan, args.output) + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + print(f"Evidence manifest written: {manifest_path}") + print(f"Manifest: {manifest['manifest_id']}") + print(f"Qualification: {manifest['runs'][0]['qualification_id']}") + print("Evidence pair: PASS") + return 0 if args.command == "run": _require_distinct_paths(args.events, args.output, args.junit_output) single_report = run_conformance( diff --git a/src/tracebook/conformance/evidence.py b/src/tracebook/conformance/evidence.py new file mode 100644 index 0000000..e245fb4 --- /dev/null +++ b/src/tracebook/conformance/evidence.py @@ -0,0 +1,676 @@ +"""Machine-enforced preparation and verification for qualification evidence.""" + +from __future__ import annotations + +import hashlib +import json +import os +import shutil +import stat +from pathlib import Path +from typing import Any, Dict, Iterable, Mapping, Optional, Tuple + +from .campaign import ( + CAMPAIGN_GENERATOR_VERSION, + _validated_campaign_parameters, + get_campaign_profile, +) +from .junit import render_junit +from .model import ARTIFACT_SCHEMA_VERSION, ConformanceError, PinnedCandidateIdentity +from .qualification import QUALIFICATION_VERSION, qualification_case_names +from .suite import BUNDLED_SUITE_VERSIONS + +EVIDENCE_PLAN_VERSION = 1 +EVIDENCE_MANIFEST_VERSION = 1 +PLAN_FILENAME = "evidence-plan.json" +MANIFEST_FILENAME = "evidence-manifest.json" +_RUN_IDS = ("run-1", "run-2") +_BUNDLE_FILES = ( + "campaign.json", + "qualification.json", + "qualification.xml", + "suite.json", +) +_REQUIRED_PROFILE = "fifo-limit-v1" +_REQUIRED_SUITE_VERSION = "v2" +_REQUIRED_SEED = 42 +_REQUIRED_TRACES = 25 +_REQUIRED_EVENTS_PER_TRACE = 200 +_REQUIRED_MAX_MINIMIZE_RUNS = 100 + + +def _json_bytes(payload: Mapping[str, Any]) -> bytes: + return ( + json.dumps(payload, indent=2, sort_keys=True, ensure_ascii=False, allow_nan=False) + "\n" + ).encode("utf-8") + + +def _content_id(payload: Mapping[str, Any]) -> str: + encoded = json.dumps( + payload, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + allow_nan=False, + ).encode("utf-8") + return "sha256:" + hashlib.sha256(encoded).hexdigest() + + +def _required_text(value: Any, field_name: str) -> str: + if not isinstance(value, str) or not value.strip(): + raise ConformanceError(f"{field_name} must be a non-empty string") + return value.strip() + + +def _sha256_id(value: Any, field_name: str) -> str: + text = _required_text(value, field_name) + if len(text) != 71 or not text.startswith("sha256:"): + raise ConformanceError(f"{field_name} must be a SHA-256 identifier") + try: + int(text.removeprefix("sha256:"), 16) + except ValueError as exc: + raise ConformanceError(f"{field_name} must be a SHA-256 identifier") from exc + return text + + +def _safe_relative_path(value: Any, field_name: str) -> Path: + text = _required_text(value, field_name) + path = Path(text) + if path.is_absolute() or path == Path(".") or ".." in path.parts: + raise ConformanceError(f"{field_name} must be a safe relative path") + return path + + +def _inside(root: Path, path: Path, field_name: str) -> Path: + lexical = Path(os.path.abspath(path.expanduser())) + resolved = lexical.resolve() + if resolved != root and root not in resolved.parents: + raise ConformanceError(f"{field_name} must stay inside evidence workspace") + if resolved != lexical: + raise ConformanceError(f"{field_name} must not traverse symbolic links") + return resolved + + +def _iter_snapshot_entries(root: Path) -> Iterable[Tuple[str, Path, os.stat_result]]: + for directory, directory_names, file_names in os.walk(root, followlinks=False): + directory_path = Path(directory) + directory_names[:] = sorted(name for name in directory_names if name != ".git") + file_names = sorted(name for name in file_names if name != ".git") + for name in directory_names + file_names: + path = directory_path / name + relative = path.relative_to(root).as_posix() + yield relative, path, path.lstat() + + +def candidate_snapshot_id(candidate_root: str | Path) -> str: + """Return the origin-stripped, content-addressed ID for one candidate tree.""" + source = Path(candidate_root).expanduser() + if source.is_symlink(): + raise ConformanceError("candidate source must be a real directory") + root = source.resolve() + if not root.is_dir(): + raise ConformanceError("candidate source must be a real directory") + digest = hashlib.sha256(b"tracebook-candidate-snapshot-v1\0") + for relative, path, metadata in _iter_snapshot_entries(root): + encoded_path = relative.encode("utf-8") + if stat.S_ISDIR(metadata.st_mode): + kind = b"directory" + content = b"" + elif stat.S_ISREG(metadata.st_mode): + kind = b"file-executable" if metadata.st_mode & 0o111 else b"file" + content = path.read_bytes() + elif stat.S_ISLNK(metadata.st_mode): + target_text = os.readlink(path) + try: + target = path.resolve(strict=True) + except OSError as exc: + raise ConformanceError(f"candidate symlink is broken: {relative}") from exc + if target != root and root not in target.parents: + raise ConformanceError(f"candidate symlink escapes source root: {relative}") + kind = b"symlink" + content = target_text.encode("utf-8") + else: + raise ConformanceError(f"candidate source contains a special file: {relative}") + digest.update(kind) + digest.update(b"\0") + digest.update(encoded_path) + digest.update(b"\0") + digest.update(str(len(content)).encode("ascii")) + digest.update(b"\0") + digest.update(content) + digest.update(b"\0") + return "sha256:" + digest.hexdigest() + + +def _copy_candidate(source: Path, destination: Path) -> None: + def ignore(_directory: str, names: list[str]) -> set[str]: + return {".git"} if ".git" in names else set() + + shutil.copytree(source, destination, symlinks=True, ignore=ignore) + + +def prepare_evidence_workspace( + candidate_source: str | Path, + workspace: str | Path, + *, + candidate_name: str, + candidate_revision: str, + expected_snapshot: Optional[str] = None, +) -> Path: + """Create two clean, captured roots and their immutable evidence plan.""" + unresolved_source = Path(candidate_source).expanduser() + if unresolved_source.is_symlink(): + raise ConformanceError("candidate source must be a real directory") + source = unresolved_source.resolve() + destination = Path(workspace).expanduser().resolve() + if not source.is_dir(): + raise ConformanceError("candidate source must be a real directory") + if destination == source or source in destination.parents: + raise ConformanceError("evidence workspace must be outside candidate source") + if destination.exists() or destination.is_symlink(): + raise ConformanceError(f"evidence workspace already exists: {destination}") + + identity = PinnedCandidateIdentity( + name=candidate_name, + revision=candidate_revision, + snapshot_id=candidate_snapshot_id(source), + ) + if expected_snapshot is not None and identity.snapshot_id != expected_snapshot.strip(): + raise ConformanceError( + "candidate snapshot does not match --expected-snapshot: " + f"expected {expected_snapshot.strip()!r}, observed {identity.snapshot_id!r}" + ) + suite_version = _REQUIRED_SUITE_VERSION + if suite_version not in BUNDLED_SUITE_VERSIONS: + choices = ", ".join(BUNDLED_SUITE_VERSIONS) + raise ConformanceError( + f"unknown bundled suite version {suite_version!r}; expected one of: {choices}" + ) + seed, traces, events_per_trace, max_minimize_runs = _validated_campaign_parameters( + _REQUIRED_SEED, + _REQUIRED_TRACES, + _REQUIRED_EVENTS_PER_TRACE, + _REQUIRED_MAX_MINIMIZE_RUNS, + ) + selected_profile = get_campaign_profile(_REQUIRED_PROFILE) + selected_cases = qualification_case_names(selected_profile) + parameters = { + "profile": selected_profile.name, + "suite_version": suite_version, + "suite_id": f"tracebook-conformance-{suite_version}", + "seed": seed, + "traces": traces, + "events_per_trace": events_per_trace, + "max_minimize_runs": max_minimize_runs, + "fixed_cases": list(selected_cases), + "capabilities": list(selected_profile.capabilities), + } + runs = [] + created = False + try: + destination.parent.mkdir(parents=True, exist_ok=True) + destination.mkdir(mode=0o700) + created = True + for run_id in _RUN_IDS: + run_root = destination / "runs" / run_id + run_root.mkdir(parents=True) + candidate_root = run_root / "candidate" + _copy_candidate(source, candidate_root) + copied_snapshot = candidate_snapshot_id(candidate_root) + if copied_snapshot != identity.snapshot_id: + raise ConformanceError(f"{run_id} candidate copy changed snapshot identity") + for name in ("adapter", "build", "cache"): + (run_root / name).mkdir() + runs.append( + { + "run_id": run_id, + "candidate_root": f"runs/{run_id}/candidate", + "adapter_root": f"runs/{run_id}/adapter", + "build_root": f"runs/{run_id}/build", + "cache_root": f"runs/{run_id}/cache", + "qualification_dir": f"runs/{run_id}/qualification", + } + ) + plan_without_id: Dict[str, Any] = { + "artifact_type": "tracebook.conformance.evidence-plan", + "schema_version": EVIDENCE_PLAN_VERSION, + "candidate": identity.to_dict(), + "qualification": parameters, + "runs": runs, + } + plan = {**plan_without_id, "plan_id": _content_id(plan_without_id)} + plan_path = destination / PLAN_FILENAME + with plan_path.open("xb") as handle: + handle.write(_json_bytes(plan)) + return plan_path + except Exception: + if created: + shutil.rmtree(destination, ignore_errors=True) + raise + + +def _load_json_object(path: Path, description: str) -> dict: + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeError, json.JSONDecodeError) as exc: + raise ConformanceError(f"could not read {description} {path}: {exc}") from exc + if not isinstance(payload, dict): + raise ConformanceError(f"{description} must be a JSON object: {path}") + return payload + + +def load_evidence_plan(path: str | Path) -> Tuple[Path, dict]: + """Load and structurally validate one evidence plan.""" + unresolved_plan = Path(path).expanduser() + if unresolved_plan.is_symlink(): + raise ConformanceError(f"evidence plan must be a real {PLAN_FILENAME} file") + plan_path = unresolved_plan.resolve() + if plan_path.name != PLAN_FILENAME or not plan_path.is_file(): + raise ConformanceError(f"evidence plan must be a real {PLAN_FILENAME} file") + plan = _load_json_object(plan_path, "evidence plan") + if plan.get("artifact_type") != "tracebook.conformance.evidence-plan": + raise ConformanceError("evidence plan has the wrong artifact_type") + if plan.get("schema_version") != EVIDENCE_PLAN_VERSION: + raise ConformanceError("evidence plan has an unsupported schema_version") + plan_id = _required_text(plan.get("plan_id"), "plan_id") + unsigned = dict(plan) + del unsigned["plan_id"] + if plan_id != _content_id(unsigned): + raise ConformanceError("evidence plan content does not match plan_id") + candidate = plan.get("candidate") + if not isinstance(candidate, dict): + raise ConformanceError("evidence plan candidate must be an object") + pinned_candidate = PinnedCandidateIdentity.from_dict(candidate) + _sha256_id(pinned_candidate.snapshot_id, "candidate.snapshot_id") + qualification = plan.get("qualification") + if not isinstance(qualification, dict): + raise ConformanceError("evidence plan qualification must be an object") + selected_profile = get_campaign_profile(_REQUIRED_PROFILE) + expected_qualification = { + "profile": selected_profile.name, + "suite_version": _REQUIRED_SUITE_VERSION, + "suite_id": f"tracebook-conformance-{_REQUIRED_SUITE_VERSION}", + "seed": _REQUIRED_SEED, + "traces": _REQUIRED_TRACES, + "events_per_trace": _REQUIRED_EVENTS_PER_TRACE, + "max_minimize_runs": _REQUIRED_MAX_MINIMIZE_RUNS, + "fixed_cases": list(qualification_case_names(selected_profile)), + "capabilities": list(selected_profile.capabilities), + } + if qualification != expected_qualification: + raise ConformanceError("evidence plan qualification contract is not canonical") + runs = plan.get("runs") + expected_runs = [ + { + "run_id": run_id, + "candidate_root": f"runs/{run_id}/candidate", + "adapter_root": f"runs/{run_id}/adapter", + "build_root": f"runs/{run_id}/build", + "cache_root": f"runs/{run_id}/cache", + "qualification_dir": f"runs/{run_id}/qualification", + } + for run_id in _RUN_IDS + ] + if runs != expected_runs: + raise ConformanceError("evidence plan does not use the canonical two-run layout") + return plan_path, plan + + +def _positive_integer(value: Any, field_name: str) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value <= 0: + raise ConformanceError(f"{field_name} must be a positive integer") + return value + + +def _validate_candidate_metadata( + payload: Any, expected: PinnedCandidateIdentity, path: str +) -> dict: + if not isinstance(payload, dict): + raise ConformanceError(f"{path} candidate metadata must be an object") + for field_name, expected_value in expected.to_dict().items(): + if payload.get(field_name) != expected_value: + raise ConformanceError( + f"{path} candidate {field_name} does not match evidence plan: " + f"expected {expected_value!r}, observed {payload.get(field_name)!r}" + ) + _required_text(payload.get("version"), f"{path}.candidate.version") + _required_text(payload.get("language"), f"{path}.candidate.language") + return payload + + +def _validate_qualification( + payload: dict, + expected: PinnedCandidateIdentity, + parameters: Mapping[str, Any], + run_id: str, +) -> dict: + prefix = f"{run_id} qualification" + if payload.get("artifact_type") != "tracebook.conformance.qualification": + raise ConformanceError(f"{prefix} has the wrong artifact_type") + if ( + payload.get("schema_version") != ARTIFACT_SCHEMA_VERSION + or payload.get("qualification_version") != QUALIFICATION_VERSION + ): + raise ConformanceError(f"{prefix} has an unsupported schema or contract version") + if payload.get("qualified") is not True: + raise ConformanceError(f"{prefix} is not qualified") + metadata = _validate_candidate_metadata(payload.get("candidate_engine"), expected, prefix) + expected_profile = get_campaign_profile(str(parameters.get("profile"))).to_dict() + profile = payload.get("profile") + if profile != expected_profile: + raise ConformanceError(f"{prefix} profile does not match evidence plan") + suite = payload.get("suite") + if not isinstance(suite, dict): + raise ConformanceError(f"{prefix} suite must be an object") + if suite.get("selection_version") != 1: + raise ConformanceError(f"{prefix} selection version must be 1") + if suite.get("suite_id") != parameters.get("suite_id"): + raise ConformanceError(f"{prefix} suite ID does not match evidence plan") + if suite.get("selected_cases") != parameters.get("fixed_cases"): + raise ConformanceError(f"{prefix} fixed-case selection does not match evidence plan") + suite_hash = _sha256_id(suite.get("suite_hash"), f"{prefix}.suite.suite_hash") + campaign = payload.get("campaign") + if not isinstance(campaign, dict): + raise ConformanceError(f"{prefix} campaign must be an object") + if ( + campaign.get("artifact_type") != "tracebook.conformance.campaign" + or campaign.get("schema_version") != ARTIFACT_SCHEMA_VERSION + or campaign.get("generator_version") != CAMPAIGN_GENERATOR_VERSION + ): + raise ConformanceError(f"{prefix} campaign has an unsupported artifact version") + expected_campaign = { + "seed": parameters.get("seed"), + "requested_traces": parameters.get("traces"), + "events_per_trace": parameters.get("events_per_trace"), + "max_minimize_runs": parameters.get("max_minimize_runs"), + } + for field_name, expected_value in expected_campaign.items(): + if campaign.get(field_name) != expected_value: + raise ConformanceError(f"{prefix} campaign {field_name} does not match evidence plan") + if campaign.get("profile") != expected_profile: + raise ConformanceError(f"{prefix} campaign profile does not match evidence plan") + if ( + campaign.get("completed_traces") != parameters.get("traces") + or campaign.get("candidate_runs") != parameters.get("traces") + or campaign.get("conformant") is not True + or campaign.get("failure") is not None + or campaign.get("stopped_at_first_divergence") is not True + ): + raise ConformanceError(f"{prefix} campaign terminal result is incomplete") + expected_events = _positive_integer(parameters.get("traces"), "qualification.traces") * ( + _positive_integer(parameters.get("events_per_trace"), "qualification.events_per_trace") + ) + if campaign.get("generated_events") != expected_events: + raise ConformanceError(f"{prefix} generated-event count is incomplete") + traces = campaign.get("traces") + if not isinstance(traces, list) or len(traces) != parameters.get("traces"): + raise ConformanceError(f"{prefix} trace inventory is incomplete") + for expected_index, trace in enumerate(traces, 1): + if ( + not isinstance(trace, dict) + or trace.get("index") != expected_index + or trace.get("conformant") is not True + or trace.get("divergence") is not None + or trace.get("event_count") != parameters.get("events_per_trace") + or trace.get("compared_events") != parameters.get("events_per_trace") + ): + raise ConformanceError(f"{prefix} contains an incomplete generated trace") + _sha256_id(trace.get("trace_sha256"), f"{prefix}.trace[{expected_index}].trace_sha256") + checks = payload.get("checks") + if not isinstance(checks, dict): + raise ConformanceError(f"{prefix} checks must be an object") + fixed = checks.get("fixed_cases") + generated = checks.get("generated_campaign") + coverage = checks.get("semantic_coverage") + expected_fixed = len(parameters.get("fixed_cases", [])) + if fixed != {"passed": expected_fixed, "total": expected_fixed, "complete": True}: + raise ConformanceError(f"{prefix} fixed-case counts are incomplete") + expected_traces = parameters.get("traces") + if generated != { + "completed_traces": expected_traces, + "requested_traces": expected_traces, + "conformant": True, + }: + raise ConformanceError(f"{prefix} generated campaign is incomplete") + if not isinstance(coverage, dict) or coverage.get("complete") is not True: + raise ConformanceError(f"{prefix} semantic coverage is incomplete") + expected_capabilities = len(parameters.get("capabilities", [])) + if ( + coverage.get("covered") != expected_capabilities + or coverage.get("expected") != expected_capabilities + or coverage.get("uncovered") != [] + ): + raise ConformanceError(f"{prefix} semantic coverage counts are incomplete") + campaign_coverage = campaign.get("semantic_coverage") + if not isinstance(campaign_coverage, dict): + raise ConformanceError(f"{prefix} campaign semantic coverage is missing") + if ( + campaign_coverage.get("covered_count") != expected_capabilities + or campaign_coverage.get("expected_count") != expected_capabilities + or campaign_coverage.get("expected_capabilities") != parameters.get("capabilities") + or campaign_coverage.get("covered_capabilities") != parameters.get("capabilities") + or campaign_coverage.get("uncovered_capabilities") != [] + or campaign_coverage.get("compared_events") != expected_events + ): + raise ConformanceError(f"{prefix} campaign semantic coverage is incomplete") + suite_report = suite.get("report") + if ( + not isinstance(suite_report, dict) + or suite_report.get("artifact_type") != "tracebook.conformance.suite_report" + or suite_report.get("schema_version") != ARTIFACT_SCHEMA_VERSION + or suite_report.get("suite_id") != parameters.get("suite_id") + or suite_report.get("suite_hash") != suite_hash + or suite_report.get("conformant") is not True + or suite_report.get("conformant_cases") != expected_fixed + or suite_report.get("case_count") != expected_fixed + or not isinstance(suite_report.get("cases"), list) + or len(suite_report["cases"]) != expected_fixed + ): + raise ConformanceError(f"{prefix} suite report is incomplete") + if [case.get("name") for case in suite_report["cases"] if isinstance(case, dict)] != list( + parameters.get("fixed_cases", []) + ): + raise ConformanceError(f"{prefix} suite case inventory does not match evidence plan") + for case in suite_report["cases"]: + if ( + not isinstance(case, dict) + or not isinstance(case.get("report"), dict) + or case["report"].get("artifact_type") != "tracebook.conformance.report" + or case["report"].get("schema_version") != ARTIFACT_SCHEMA_VERSION + or case["report"].get("conformant") is not True + or case["report"].get("divergence") is not None + or case["report"].get("candidate_engine") != metadata + ): + raise ConformanceError(f"{prefix} contains a nonconformant fixed case") + _sha256_id(case.get("events_sha256"), f"{prefix}.case.events_sha256") + expected_candidate_runs = expected_fixed + _positive_integer( + parameters.get("traces"), "qualification.traces" + ) + if payload.get("candidate_runs") != expected_candidate_runs: + raise ConformanceError(f"{prefix} candidate-run count is incomplete") + if payload.get("paths") != { + "suite": "suite.json", + "campaign": "campaign.json", + "junit": "qualification.xml", + "reduced": None, + }: + raise ConformanceError(f"{prefix} artifact paths are not canonical") + campaign_identity = { + "generator_version": CAMPAIGN_GENERATOR_VERSION, + "profile": expected_profile, + "seed": parameters.get("seed"), + "requested_traces": parameters.get("traces"), + "events_per_trace": parameters.get("events_per_trace"), + } + campaign_id = _sha256_id(campaign.get("campaign_id"), f"{prefix}.campaign_id") + if campaign_id != _content_id(campaign_identity): + raise ConformanceError(f"{prefix} campaign content does not match campaign_id") + qualification_identity = { + "qualification_version": QUALIFICATION_VERSION, + "profile": expected_profile, + "suite_id": parameters.get("suite_id"), + "suite_hash": suite_hash, + "selected_cases": parameters.get("fixed_cases"), + "suite_report": suite_report, + "campaign": campaign, + } + qualification_id = _sha256_id(payload.get("qualification_id"), f"{prefix}.qualification_id") + if qualification_id != _content_id(qualification_identity): + raise ConformanceError(f"{prefix} content does not match qualification_id") + for nested_path, nested in ( + ("campaign", campaign.get("candidate_engine")), + ("suite.report", suite_report.get("candidate_engine")), + ): + if nested != metadata: + raise ConformanceError(f"{prefix} {nested_path} candidate metadata changed") + return { + "qualification_id": qualification_id, + "campaign_id": campaign_id, + "candidate_engine": metadata, + "fixed_cases": fixed, + "generated_campaign": generated, + "semantic_coverage": coverage, + } + + +def _bundle_record(bundle: Path) -> Tuple[dict, Dict[str, bytes]]: + if not bundle.is_dir() or bundle.is_symlink(): + raise ConformanceError(f"qualification bundle is not a real directory: {bundle}") + observed = sorted(path.name for path in bundle.iterdir()) + if observed != sorted(_BUNDLE_FILES): + raise ConformanceError( + f"qualification bundle must contain exactly {', '.join(_BUNDLE_FILES)}: {bundle}" + ) + contents: Dict[str, bytes] = {} + hashes: Dict[str, str] = {} + for name in _BUNDLE_FILES: + path = bundle / name + if not path.is_file() or path.is_symlink(): + raise ConformanceError(f"qualification artifact must be a real file: {path}") + contents[name] = path.read_bytes() + hashes[name] = "sha256:" + hashlib.sha256(contents[name]).hexdigest() + return {"files": hashes}, contents + + +def verify_evidence_plan(path: str | Path) -> dict: + """Verify two captured qualification bundles and return a grading manifest.""" + plan_path, plan = load_evidence_plan(path) + root = plan_path.parent.resolve() + expected = PinnedCandidateIdentity.from_dict(plan["candidate"]) + parameters = plan["qualification"] + run_records = [] + bundle_contents = [] + qualification_summaries = [] + for run in plan["runs"]: + run_id = run["run_id"] + prepared_roots = {} + for field_name in ("candidate_root", "adapter_root", "build_root", "cache_root"): + prepared = _inside( + root, + root / _safe_relative_path(run[field_name], field_name), + field_name, + ) + if not prepared.is_dir(): + raise ConformanceError(f"{run_id} {field_name} is not a real directory") + prepared_roots[field_name] = prepared + candidate_root = prepared_roots["candidate_root"] + if candidate_snapshot_id(candidate_root) != expected.snapshot_id: + raise ConformanceError(f"{run_id} candidate tree changed after evidence preparation") + qualification_dir = _inside( + root, + root / _safe_relative_path(run["qualification_dir"], "qualification_dir"), + "qualification_dir", + ) + bundle_record, contents = _bundle_record(qualification_dir) + qualification = _load_json_object( + qualification_dir / "qualification.json", f"{run_id} qualification" + ) + suite = qualification.get("suite") + campaign_payload = qualification.get("campaign") + if not isinstance(campaign_payload, dict): + raise ConformanceError(f"{run_id} qualification campaign must be an object") + if not isinstance(suite, dict) or not isinstance(suite.get("report"), dict): + raise ConformanceError(f"{run_id} qualification suite report must be an object") + suite_report_payload = suite["report"] + canonical_artifacts = { + "qualification.json": _json_bytes(qualification), + "campaign.json": _json_bytes(campaign_payload), + "suite.json": _json_bytes(suite_report_payload), + "qualification.xml": render_junit(qualification).encode("utf-8"), + } + for name, canonical in canonical_artifacts.items(): + if contents[name] != canonical: + raise ConformanceError( + f"{run_id} {name} is not the canonical artifact for qualification.json" + ) + summary = _validate_qualification(qualification, expected, parameters, run_id) + run_records.append( + { + "run_id": run_id, + "candidate_root": run["candidate_root"], + "adapter_root": run["adapter_root"], + "build_root": run["build_root"], + "cache_root": run["cache_root"], + "qualification_dir": run["qualification_dir"], + "candidate_snapshot_id": expected.snapshot_id, + **bundle_record, + **summary, + } + ) + bundle_contents.append(contents) + qualification_summaries.append(summary) + if qualification_summaries[0] != qualification_summaries[1]: + raise ConformanceError("qualification terminal result or deterministic identifiers differ") + for name in _BUNDLE_FILES: + if bundle_contents[0][name] != bundle_contents[1][name]: + raise ConformanceError(f"qualification artifact bytes differ between runs: {name}") + unsigned: Dict[str, Any] = { + "artifact_type": "tracebook.conformance.evidence-manifest", + "schema_version": EVIDENCE_MANIFEST_VERSION, + "status": "qualified", + "plan_id": plan["plan_id"], + "candidate": plan["candidate"], + "qualification": parameters, + "equality": { + "terminal_result": True, + "candidate_metadata": True, + "campaign_id": True, + "qualification_id": True, + "counts_and_coverage": True, + "artifact_bytes": True, + }, + "runs": run_records, + } + return {**unsigned, "manifest_id": _content_id(unsigned)} + + +def write_evidence_manifest( + plan_path: str | Path, + output: Optional[str | Path] = None, +) -> Path: + """Verify one plan and exclusively write its grading-ready manifest.""" + resolved_plan, _ = load_evidence_plan(plan_path) + root = resolved_plan.parent.resolve() + destination = ( + root / MANIFEST_FILENAME + if output is None + else _inside( + root, + Path(output) if Path(output).is_absolute() else root / Path(output), + "manifest output", + ) + ) + if destination.parent != root: + raise ConformanceError("evidence manifest must be written at workspace root") + if destination == resolved_plan: + raise ConformanceError("evidence manifest must not overwrite the evidence plan") + manifest = verify_evidence_plan(resolved_plan) + try: + destination.parent.mkdir(parents=True, exist_ok=True) + with destination.open("xb") as handle: + handle.write(_json_bytes(manifest)) + except FileExistsError as exc: + raise ConformanceError(f"evidence manifest already exists: {destination}") from exc + except OSError as exc: + raise ConformanceError(f"could not write evidence manifest {destination}: {exc}") from exc + return destination diff --git a/src/tracebook/conformance/external.py b/src/tracebook/conformance/external.py index c5676be..e4f1d48 100644 --- a/src/tracebook/conformance/external.py +++ b/src/tracebook/conformance/external.py @@ -20,6 +20,7 @@ ConformanceError, EngineMetadata, Observation, + PinnedCandidateIdentity, ) _END_OF_STREAM = object() @@ -38,6 +39,7 @@ def __init__( config: ConformanceConfig, timeout_seconds: float = 5.0, cwd: Optional[str | Path] = None, + expected_identity: Optional[PinnedCandidateIdentity] = None, ) -> None: if isinstance(command, (str, bytes)) or not command: raise ConformanceError("candidate command must be a non-empty argument list") @@ -113,6 +115,11 @@ def __init__( if ready.get("protocol_version") != PROTOCOL_VERSION: raise AdapterProtocolError("ready message reported the wrong protocol version") self.metadata = EngineMetadata.from_dict(ready.get("engine", {})) + if expected_identity is not None: + try: + expected_identity.validate(self.metadata) + except ConformanceError as exc: + raise AdapterProtocolError(str(exc)) from exc except Exception: self._broken = True self._shutdown() @@ -288,10 +295,12 @@ def __init__( command: Sequence[str], timeout_seconds: float = 5.0, cwd: Optional[str | Path] = None, + expected_identity: Optional[PinnedCandidateIdentity] = None, ) -> None: self.command = tuple(command) self.timeout_seconds = timeout_seconds self.cwd = cwd + self.expected_identity = expected_identity def __call__(self, config: ConformanceConfig) -> ExternalProcessAdapter: return ExternalProcessAdapter( @@ -299,4 +308,5 @@ def __call__(self, config: ConformanceConfig) -> ExternalProcessAdapter: config, timeout_seconds=self.timeout_seconds, cwd=self.cwd, + expected_identity=self.expected_identity, ) diff --git a/src/tracebook/conformance/model.py b/src/tracebook/conformance/model.py index a6a35c3..b7f6027 100644 --- a/src/tracebook/conformance/model.py +++ b/src/tracebook/conformance/model.py @@ -169,6 +169,8 @@ class EngineMetadata: name: str version: str language: str + revision: Optional[str] = None + snapshot_id: Optional[str] = None def __post_init__(self) -> None: for field_name in ("name", "version", "language"): @@ -176,9 +178,20 @@ def __post_init__(self) -> None: if not isinstance(value, str) or not value.strip(): raise ConformanceError(f"engine {field_name} must be a non-empty string") object.__setattr__(self, field_name, value.strip()) + for field_name in ("revision", "snapshot_id"): + value = getattr(self, field_name) + if value is not None: + if not isinstance(value, str) or not value.strip(): + raise ConformanceError(f"engine {field_name} must be null or non-empty") + object.__setattr__(self, field_name, value.strip()) def to_dict(self) -> dict: - return {"name": self.name, "version": self.version, "language": self.language} + payload = {"name": self.name, "version": self.version, "language": self.language} + if self.revision is not None: + payload["revision"] = self.revision + if self.snapshot_id is not None: + payload["snapshot_id"] = self.snapshot_id + return payload @classmethod def from_dict(cls, data: Mapping[str, Any]) -> "EngineMetadata": @@ -188,9 +201,56 @@ def from_dict(cls, data: Mapping[str, Any]) -> "EngineMetadata": name=data.get("name", ""), version=data.get("version", ""), language=data.get("language", ""), + revision=data.get("revision"), + snapshot_id=data.get("snapshot_id"), ) +@dataclass(frozen=True) +class PinnedCandidateIdentity: + """Task-pinned candidate identity required by release-evidence workflows.""" + + name: str + revision: str + snapshot_id: str + + def __post_init__(self) -> None: + for field_name in ("name", "revision", "snapshot_id"): + value = getattr(self, field_name) + if not isinstance(value, str) or not value.strip(): + raise ConformanceError(f"candidate {field_name} must be a non-empty string") + object.__setattr__(self, field_name, value.strip()) + + def to_dict(self) -> dict: + return { + "name": self.name, + "revision": self.revision, + "snapshot_id": self.snapshot_id, + } + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "PinnedCandidateIdentity": + if not isinstance(data, Mapping): + raise ConformanceError("candidate identity must be an object") + return cls( + name=data.get("name", ""), + revision=data.get("revision", ""), + snapshot_id=data.get("snapshot_id", ""), + ) + + def validate(self, metadata: EngineMetadata) -> None: + if not isinstance(metadata, EngineMetadata): + raise ConformanceError("candidate metadata must be EngineMetadata") + for field_name in ("name", "revision", "snapshot_id"): + expected = getattr(self, field_name) + observed = getattr(metadata, field_name) + if observed != expected: + raise ConformanceError( + f"candidate {field_name} does not match task-pinned identity: " + f"expected {expected!r}, observed {observed!r}" + ) + + @dataclass(frozen=True) class Outcome: """Canonical applied/rejected result for one source event.""" diff --git a/tests/test_conformance_evidence.py b/tests/test_conformance_evidence.py new file mode 100644 index 0000000..c6d7c64 --- /dev/null +++ b/tests/test_conformance_evidence.py @@ -0,0 +1,502 @@ +import hashlib +import json +import sys +from pathlib import Path + +import pytest + +from tracebook.conformance.campaign import get_campaign_profile +from tracebook.conformance.cli import main +from tracebook.conformance.evidence import ( + MANIFEST_FILENAME, + candidate_snapshot_id, + prepare_evidence_workspace, + verify_evidence_plan, + write_evidence_manifest, +) +from tracebook.conformance.external import AdapterProtocolError, ExternalProcessAdapter +from tracebook.conformance.junit import render_junit +from tracebook.conformance.model import ( + ConformanceConfig, + ConformanceError, + EngineMetadata, + PinnedCandidateIdentity, +) + + +def _artifact_id(payload: dict) -> str: + encoded = json.dumps( + payload, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + allow_nan=False, + ).encode("utf-8") + return "sha256:" + hashlib.sha256(encoded).hexdigest() + + +def _candidate(tmp_path: Path) -> Path: + candidate = tmp_path / "candidate" + candidate.mkdir() + (candidate / "README.md").write_text("candidate\n", encoding="utf-8") + (candidate / "src").mkdir() + (candidate / "src" / "engine.txt").write_text("price-time\n", encoding="utf-8") + (candidate / ".git").mkdir() + (candidate / ".git" / "HEAD").write_text("ignored\n", encoding="utf-8") + return candidate + + +def _plan(tmp_path: Path) -> tuple[Path, dict]: + candidate = _candidate(tmp_path) + plan_path = prepare_evidence_workspace( + candidate, + tmp_path / "evidence", + candidate_name="example/matcher", + candidate_revision="abc123", + ) + return plan_path, json.loads(plan_path.read_text(encoding="utf-8")) + + +def _qualification(plan: dict) -> dict: + identity = plan["candidate"] + parameters = plan["qualification"] + metadata = { + **identity, + "version": "1.2.3", + "language": "Rust", + } + capabilities = parameters["capabilities"] + profile = get_campaign_profile(parameters["profile"]).to_dict() + coverage = { + "schema_version": 1, + "basis": "reference observations for candidate-compared events", + "expected_capabilities": capabilities, + "covered_capabilities": capabilities, + "uncovered_capabilities": [], + "covered_count": len(capabilities), + "expected_count": len(capabilities), + "coverage_ratio": 1.0, + "evidence": {name: 1 for name in capabilities}, + "operations": {}, + "order_types": {}, + "applied_order_types": {}, + "outcomes": {}, + "rejection_reasons": {}, + "symbols": ["ALPHA", "BETA"], + "compared_events": 5000, + "trade_events": 1, + "trades": 1, + "partial_fill_events": 1, + "queue_priority_probes": 1, + } + traces = [ + { + "index": index + 1, + "seed": index, + "trace_sha256": f"sha256:{index:064x}", + "event_count": parameters["events_per_trace"], + "compared_events": parameters["events_per_trace"], + "conformant": True, + "divergence": None, + } + for index in range(parameters["traces"]) + ] + cases = [ + { + "name": name, + "events_sha256": "sha256:" + f"{index + 10:064x}", + "report": { + "artifact_type": "tracebook.conformance.report", + "schema_version": 1, + "candidate_engine": metadata, + "conformant": True, + "divergence": None, + }, + } + for index, name in enumerate(parameters["fixed_cases"]) + ] + suite_report = { + "artifact_type": "tracebook.conformance.suite_report", + "schema_version": 1, + "suite_id": parameters["suite_id"], + "suite_hash": "sha256:" + "1" * 64, + "candidate_engine": metadata, + "case_count": len(cases), + "conformant_cases": len(cases), + "conformant": True, + "cases": cases, + } + campaign = { + "artifact_type": "tracebook.conformance.campaign", + "schema_version": 1, + "candidate_engine": metadata, + "profile": profile, + "generator_version": 2, + "seed": parameters["seed"], + "requested_traces": parameters["traces"], + "completed_traces": parameters["traces"], + "events_per_trace": parameters["events_per_trace"], + "generated_events": parameters["traces"] * parameters["events_per_trace"], + "max_minimize_runs": parameters["max_minimize_runs"], + "candidate_runs": parameters["traces"], + "conformant": True, + "failure": None, + "stopped_at_first_divergence": True, + "semantic_coverage": coverage, + "traces": traces, + } + campaign["campaign_id"] = _artifact_id( + { + "generator_version": 2, + "profile": profile, + "seed": parameters["seed"], + "requested_traces": parameters["traces"], + "events_per_trace": parameters["events_per_trace"], + } + ) + checks = { + "fixed_cases": { + "passed": len(cases), + "total": len(cases), + "complete": True, + }, + "generated_campaign": { + "completed_traces": parameters["traces"], + "requested_traces": parameters["traces"], + "conformant": True, + }, + "semantic_coverage": { + "covered": len(capabilities), + "expected": len(capabilities), + "uncovered": [], + "complete": True, + }, + } + qualification = { + "artifact_type": "tracebook.conformance.qualification", + "schema_version": 1, + "qualification_version": 1, + "qualified": True, + "candidate_engine": metadata, + "profile": profile, + "suite": { + "suite_id": parameters["suite_id"], + "suite_hash": suite_report["suite_hash"], + "selection_version": 1, + "selected_cases": parameters["fixed_cases"], + "report": suite_report, + }, + "campaign": campaign, + "checks": checks, + "candidate_runs": len(cases) + parameters["traces"], + "paths": { + "suite": "suite.json", + "campaign": "campaign.json", + "junit": "qualification.xml", + "reduced": None, + }, + } + qualification["qualification_id"] = _artifact_id( + { + "qualification_version": 1, + "profile": profile, + "suite_id": parameters["suite_id"], + "suite_hash": suite_report["suite_hash"], + "selected_cases": parameters["fixed_cases"], + "suite_report": suite_report, + "campaign": campaign, + } + ) + return qualification + + +def _write_bundle(root: Path, payload: dict) -> None: + root.mkdir() + (root / "qualification.json").write_text( + json.dumps(payload, indent=2, sort_keys=True, allow_nan=False) + "\n", + encoding="utf-8", + ) + (root / "campaign.json").write_text( + json.dumps(payload["campaign"], indent=2, sort_keys=True, allow_nan=False) + "\n", + encoding="utf-8", + ) + (root / "suite.json").write_text( + json.dumps(payload["suite"]["report"], indent=2, sort_keys=True, allow_nan=False) + "\n", + encoding="utf-8", + ) + (root / "qualification.xml").write_text(render_junit(payload), encoding="utf-8") + + +def _write_pair(plan_path: Path, plan: dict, payload: dict) -> None: + for run in plan["runs"]: + _write_bundle(plan_path.parent / run["qualification_dir"], payload) + + +def test_prepare_creates_two_clean_roots_and_pins_snapshot(tmp_path): + candidate = _candidate(tmp_path) + snapshot = candidate_snapshot_id(candidate) + + plan_path = prepare_evidence_workspace( + candidate, + tmp_path / "evidence", + candidate_name="example/matcher", + candidate_revision="abc123", + expected_snapshot=snapshot, + ) + plan = json.loads(plan_path.read_text(encoding="utf-8")) + + assert plan["candidate"] == { + "name": "example/matcher", + "revision": "abc123", + "snapshot_id": snapshot, + } + assert plan["qualification"]["profile"] == "fifo-limit-v1" + assert plan["qualification"]["seed"] == 42 + assert plan["qualification"]["traces"] == 25 + assert plan["qualification"]["events_per_trace"] == 200 + assert [run["run_id"] for run in plan["runs"]] == ["run-1", "run-2"] + for run in plan["runs"]: + root = plan_path.parent / "runs" / run["run_id"] + assert candidate_snapshot_id(root / "candidate") == snapshot + assert not (root / "candidate" / ".git").exists() + assert (root / "adapter").is_dir() + assert (root / "build").is_dir() + assert (root / "cache").is_dir() + assert not (root / "qualification").exists() + + +def test_prepare_refuses_snapshot_mismatch_and_nested_workspace(tmp_path): + candidate = _candidate(tmp_path) + + with pytest.raises(ConformanceError, match="does not match"): + prepare_evidence_workspace( + candidate, + tmp_path / "wrong", + candidate_name="example/matcher", + candidate_revision="abc123", + expected_snapshot="sha256:wrong", + ) + with pytest.raises(ConformanceError, match="outside candidate source"): + prepare_evidence_workspace( + candidate, + candidate / "evidence", + candidate_name="example/matcher", + candidate_revision="abc123", + ) + + +def test_prepare_refuses_a_symlinked_candidate_root(tmp_path): + candidate = _candidate(tmp_path) + alias = tmp_path / "candidate-link" + alias.symlink_to(candidate, target_is_directory=True) + + with pytest.raises(ConformanceError, match="real directory"): + candidate_snapshot_id(alias) + with pytest.raises(ConformanceError, match="real directory"): + prepare_evidence_workspace( + alias, + tmp_path / "evidence", + candidate_name="example/matcher", + candidate_revision="abc123", + ) + + +def test_verify_writes_grading_manifest_for_identical_canonical_pair(tmp_path): + plan_path, plan = _plan(tmp_path) + payload = _qualification(plan) + _write_pair(plan_path, plan, payload) + + manifest_path = write_evidence_manifest(plan_path) + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + + assert manifest_path.name == MANIFEST_FILENAME + assert manifest["status"] == "qualified" + assert manifest["candidate"] == plan["candidate"] + assert manifest["equality"] == { + "terminal_result": True, + "candidate_metadata": True, + "campaign_id": True, + "qualification_id": True, + "counts_and_coverage": True, + "artifact_bytes": True, + } + assert len(manifest["runs"]) == 2 + assert manifest["runs"][0]["files"] == manifest["runs"][1]["files"] + + +def test_verify_refuses_unpinned_metadata_and_changed_candidate(tmp_path): + plan_path, plan = _plan(tmp_path) + payload = _qualification(plan) + payload["candidate_engine"]["revision"] = "wrong" + _write_pair(plan_path, plan, payload) + + with pytest.raises(ConformanceError, match="candidate revision"): + verify_evidence_plan(plan_path) + + for run in plan["runs"]: + qualification = plan_path.parent / run["qualification_dir"] + for child in qualification.iterdir(): + child.unlink() + qualification.rmdir() + payload = _qualification(plan) + _write_pair(plan_path, plan, payload) + candidate_file = plan_path.parent / plan["runs"][0]["candidate_root"] / "README.md" + candidate_file.write_text("mutated\n", encoding="utf-8") + + with pytest.raises(ConformanceError, match="candidate tree changed"): + verify_evidence_plan(plan_path) + + +def test_verify_refuses_noncanonical_or_different_bundle_bytes(tmp_path): + plan_path, plan = _plan(tmp_path) + payload = _qualification(plan) + _write_pair(plan_path, plan, payload) + first = plan_path.parent / plan["runs"][0]["qualification_dir"] / "campaign.json" + first.write_text("{}\n", encoding="utf-8") + + with pytest.raises(ConformanceError, match="not the canonical artifact"): + verify_evidence_plan(plan_path) + + +@pytest.mark.parametrize( + ("mutation", "message"), + ( + ("campaign-result", "terminal result"), + ("capability-list", "semantic coverage"), + ("case-inventory", "case inventory"), + ("candidate-runs", "candidate-run count"), + ), +) +def test_verify_refuses_incomplete_or_substituted_claims(tmp_path, mutation, message): + plan_path, plan = _plan(tmp_path) + payload = _qualification(plan) + if mutation == "campaign-result": + payload["campaign"]["conformant"] = False + elif mutation == "capability-list": + payload["campaign"]["semantic_coverage"]["covered_capabilities"] = [] + elif mutation == "case-inventory": + payload["suite"]["report"]["cases"][0]["name"] = "substituted" + else: + payload["candidate_runs"] -= 1 + _write_pair(plan_path, plan, payload) + + with pytest.raises(ConformanceError, match=message): + verify_evidence_plan(plan_path) + + +def test_manifest_output_stays_at_workspace_root_and_plan_must_be_real(tmp_path): + plan_path, plan = _plan(tmp_path) + _write_pair(plan_path, plan, _qualification(plan)) + + with pytest.raises(ConformanceError, match="workspace root"): + write_evidence_manifest(plan_path, "nested/manifest.json") + + custom = write_evidence_manifest(plan_path, "review.json") + assert custom == plan_path.parent / "review.json" + + plan_link = tmp_path / "plan-link.json" + plan_link.symlink_to(plan_path) + with pytest.raises(ConformanceError, match="real evidence-plan.json"): + verify_evidence_plan(plan_link) + + +def test_verify_requires_all_prepared_roots_to_remain_real_directories(tmp_path): + plan_path, plan = _plan(tmp_path) + _write_pair(plan_path, plan, _qualification(plan)) + cache = plan_path.parent / plan["runs"][0]["cache_root"] + cache.rmdir() + + with pytest.raises(ConformanceError, match="cache_root is not a real directory"): + verify_evidence_plan(plan_path) + + +def test_engine_metadata_can_bind_and_validate_task_identity(): + identity = PinnedCandidateIdentity("example/matcher", "abc123", "sha256:tree") + metadata = EngineMetadata( + "example/matcher", + "1.2.3", + "Rust", + revision="abc123", + snapshot_id="sha256:tree", + ) + + identity.validate(metadata) + assert EngineMetadata.from_dict(metadata.to_dict()) == metadata + with pytest.raises(ConformanceError, match="candidate revision"): + identity.validate( + EngineMetadata( + "example/matcher", + "1.2.3", + "Rust", + revision="wrong", + snapshot_id="sha256:tree", + ) + ) + + +def test_external_adapter_rejects_unpinned_ready_metadata_before_events(): + script = """ +import json +import sys + +json.loads(sys.stdin.readline()) +print(json.dumps({ + "type": "ready", + "protocol": "tracebook.conformance", + "protocol_version": 1, + "engine": {"name": "example/matcher", "version": "1", "language": "Python"}, +}), flush=True) +""" + identity = PinnedCandidateIdentity("example/matcher", "abc123", "sha256:tree") + + with pytest.raises(AdapterProtocolError, match="candidate revision"): + ExternalProcessAdapter( + [sys.executable, "-c", script], + ConformanceConfig(), + timeout_seconds=1, + expected_identity=identity, + ) + + +def test_cli_requires_all_identity_flags_and_drives_evidence_commands(tmp_path, capsys): + candidate = _candidate(tmp_path) + workspace = tmp_path / "evidence" + assert ( + main( + [ + "evidence-init", + str(candidate), + "--workspace", + str(workspace), + "--candidate-name", + "example/matcher", + "--candidate-revision", + "abc123", + ] + ) + == 0 + ) + assert "Candidate snapshot: sha256:" in capsys.readouterr().out + + plan_path = workspace / "evidence-plan.json" + plan = json.loads(plan_path.read_text(encoding="utf-8")) + _write_pair(plan_path, plan, _qualification(plan)) + assert main(["evidence-verify", str(plan_path)]) == 0 + assert "Evidence pair: PASS" in capsys.readouterr().out + + events = tmp_path / "empty.jsonl" + events.write_text("", encoding="utf-8") + assert ( + main( + [ + "run", + str(events), + "--candidate-name", + "example/matcher", + "--candidate-cmd", + f"{sys.executable} -c 'raise SystemExit(99)'", + ] + ) + == 2 + ) + assert "must be provided together" in capsys.readouterr().err