diff --git a/.flake8 b/.flake8
index 8b5f305..a6bd96a 100644
--- a/.flake8
+++ b/.flake8
@@ -9,4 +9,5 @@ exclude =
*.egg-info,
venv,
env,
- ENV
+ ENV,
+ experiments/private
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 72a7d31..9014c42 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -65,6 +65,9 @@ jobs:
- name: Validate both distributions
run: python -m twine check dist/conformance/* dist/simulator/*
+ - name: Reject private paths in both distributions
+ run: python tools/verify_distribution_privacy.py dist/conformance/* dist/simulator/*
+
- name: Rebuild wheels from sdists and verify artifact agreement
if: matrix.python-version == '3.13'
run: |
diff --git a/.github/workflows/gocronx-matcher.yml b/.github/workflows/gocronx-matcher.yml
index e6b7e56..c3b6d96 100644
--- a/.github/workflows/gocronx-matcher.yml
+++ b/.github/workflows/gocronx-matcher.yml
@@ -7,11 +7,17 @@ on:
pull_request:
paths:
- ".github/workflows/gocronx-matcher.yml"
+ - "pyproject.toml"
+ - "MANIFEST.in"
- "integrations/gocronx_matcher/**"
- "integrations/rust_protocol/**"
+ - "src/tracebook/__init__.py"
+ - "src/tracebook/_version.py"
+ - "src/tracebook/core/**"
- "src/tracebook/conformance/**"
- - "src/tracebook/events/market_replay.py"
+ - "src/tracebook/events/**"
- "tests/test_external_integrations.py"
+ - "tests/test_integration_workflow_coverage.py"
permissions:
contents: read
diff --git a/.github/workflows/intrepid-orderbook.yml b/.github/workflows/intrepid-orderbook.yml
new file mode 100644
index 0000000..4baaa3e
--- /dev/null
+++ b/.github/workflows/intrepid-orderbook.yml
@@ -0,0 +1,201 @@
+name: intrepidkarthi orderbook integration
+
+on:
+ workflow_dispatch:
+ schedule:
+ - cron: "17 7 * * 2"
+ pull_request:
+ paths:
+ - ".github/workflows/intrepid-orderbook.yml"
+ - "pyproject.toml"
+ - "MANIFEST.in"
+ - "integrations/intrepid_orderbook/**"
+ - "src/tracebook/__init__.py"
+ - "src/tracebook/_version.py"
+ - "src/tracebook/core/**"
+ - "src/tracebook/conformance/**"
+ - "src/tracebook/events/**"
+ - "tests/test_external_integrations.py"
+ - "tests/test_integration_workflow_coverage.py"
+
+permissions:
+ contents: read
+
+jobs:
+ pinned-go-clob:
+ runs-on: ubuntu-latest
+ timeout-minutes: 20
+
+ steps:
+ - uses: actions/checkout@v7
+
+ - uses: actions/setup-python@v7
+ with:
+ python-version: "3.12"
+ cache: pip
+
+ - uses: actions/setup-go@v6
+ with:
+ go-version: "1.23.5"
+ cache-dependency-path: integrations/intrepid_orderbook/go.sum
+
+ - name: Configure integration paths
+ run: |
+ echo "ADAPTER=$RUNNER_TEMP/tracebook-intrepid-orderbook" >> "$GITHUB_ENV"
+ echo "REPORT_ROOT=$RUNNER_TEMP/intrepid-orderbook" >> "$GITHUB_ENV"
+
+ - name: Install Tracebook
+ run: python -m pip install -e .
+
+ - name: Build and test the pinned adapter
+ working-directory: integrations/intrepid_orderbook
+ run: |
+ test -z "$(gofmt -l .)"
+ go mod verify
+ test "$(go list -m -f '{{.Version}}' github.com/intrepidkarthi/orderbook)" = "v0.26.0"
+ go vet ./...
+ go test ./...
+ go build -trimpath -o "$ADAPTER" .
+
+ - name: Verify the nine-case suite boundary
+ run: |
+ set +e
+ tracebook-conformance suite \
+ src/tracebook/conformance/fixtures/v2 \
+ --output "$REPORT_ROOT/suite.json" \
+ --timeout 10 \
+ --candidate "$ADAPTER"
+ rc=$?
+ set -e
+ test "$rc" -eq 1
+ python - <<'PY'
+ import json
+ import os
+ from pathlib import Path
+
+ report = json.loads((Path(os.environ["REPORT_ROOT"]) / "suite.json").read_text())
+ assert report["case_count"] == 9
+ assert report["conformant_cases"] == 6
+ failed = {
+ case["name"]: case["report"]["divergence"]
+ for case in report["cases"]
+ if not case["report"]["conformant"]
+ }
+ assert set(failed) == {
+ "order-instructions",
+ "pro-rata-allocation",
+ "stp-cancel-incoming",
+ }
+ for divergence in failed.values():
+ assert divergence["category"] == "outcome"
+ assert divergence["path"] == "$.observation.outcome.reason"
+ assert divergence["candidate"] == "INVALID_ORDER"
+ assert divergence["reference"] is None
+ PY
+
+ - name: Qualify fifo-limit-v1
+ run: |
+ tracebook-conformance qualify \
+ --profile fifo-limit-v1 \
+ --seed 42 \
+ --traces 25 \
+ --events-per-trace 200 \
+ --max-minimize-runs 100 \
+ --candidate-cmd "$ADAPTER" \
+ --output-dir "$REPORT_ROOT/fifo-limit"
+ python - <<'PY'
+ import json
+ import os
+ from pathlib import Path
+
+ report = json.loads(
+ (Path(os.environ["REPORT_ROOT"]) / "fifo-limit" / "qualification.json").read_text()
+ )
+ assert report["qualified"] is True
+ assert report["qualification_id"] == "sha256:f8db775baabe7e665d45bbb920a67e43d405acf445de4951617df68ffa24eb69"
+ assert report["campaign"]["campaign_id"] == "sha256:59d70645ff13f12fa4af23af69631714df22ef5f25cd1104b0c1124f98f71f6a"
+ assert report["candidate_runs"] == 28
+ assert report["suite"]["report"]["conformant_cases"] == 3
+ assert report["campaign"]["completed_traces"] == 25
+ assert report["campaign"]["generated_events"] == 5000
+ assert report["campaign"]["semantic_coverage"]["covered_count"] == 10
+ assert report["campaign"]["semantic_coverage"]["coverage_ratio"] == 1.0
+ PY
+
+ - name: Qualify fifo-partial-fill-v1
+ run: |
+ tracebook-conformance qualify \
+ --profile fifo-partial-fill-v1 \
+ --seed 42 \
+ --traces 25 \
+ --events-per-trace 200 \
+ --max-minimize-runs 100 \
+ --candidate-cmd "$ADAPTER" \
+ --output-dir "$REPORT_ROOT/fifo-partial"
+ python - <<'PY'
+ import json
+ import os
+ from pathlib import Path
+
+ report = json.loads(
+ (Path(os.environ["REPORT_ROOT"]) / "fifo-partial" / "qualification.json").read_text()
+ )
+ assert report["qualified"] is True
+ assert report["qualification_id"] == "sha256:08bf641bfa57da1892d744c08bca0be00f5ac1ade580ae8fcf7f195f0fcad6eb"
+ assert report["campaign"]["campaign_id"] == "sha256:297f895b99ac26f92287ababd1dcc25b68b0be98787e9b9d814d5cd241769cf9"
+ assert report["candidate_runs"] == 28
+ assert report["suite"]["report"]["conformant_cases"] == 3
+ assert report["campaign"]["completed_traces"] == 25
+ assert report["campaign"]["generated_events"] == 5000
+ assert report["campaign"]["semantic_coverage"]["covered_count"] == 10
+ assert report["campaign"]["semantic_coverage"]["coverage_ratio"] == 1.0
+ PY
+
+ - name: Regenerate the full-profile FOK divergence
+ run: |
+ set +e
+ tracebook-conformance qualify \
+ --profile fifo-full-v1 \
+ --seed 42 \
+ --traces 25 \
+ --events-per-trace 200 \
+ --max-minimize-runs 100 \
+ --candidate-cmd "$ADAPTER" \
+ --output-dir "$REPORT_ROOT/fifo-full"
+ rc=$?
+ set -e
+ test "$rc" -eq 1
+ cmp \
+ integrations/intrepid_orderbook/regressions/fok-rejection-reduced.jsonl \
+ "$REPORT_ROOT/fifo-full/reduced.jsonl"
+ cmp \
+ integrations/intrepid_orderbook/regressions/fok-rejection-failure.json \
+ "$REPORT_ROOT/fifo-full/failure.json"
+ python - <<'PY'
+ import json
+ import os
+ from pathlib import Path
+
+ report = json.loads(
+ (Path(os.environ["REPORT_ROOT"]) / "fifo-full" / "qualification.json").read_text()
+ )
+ failure = report["campaign"]["failure"]
+ retained = json.loads(
+ (Path(os.environ["REPORT_ROOT"]) / "fifo-full" / "failure.json").read_text()
+ )
+ assert report["qualified"] is False
+ assert report["qualification_id"] == "sha256:81a095f9804852ff7035f4c3d428a51ae3d0e8e4b81ec335128c6af15b6f65bd"
+ assert report["campaign"]["campaign_id"] == "sha256:61464afc70170352bc6eecb6ccb22f120d05166d6f5cb7926ce5d43adf8a6b41"
+ assert failure["failure_id"] == "failure-1ce2954857800b3a068d"
+ assert failure["original_event_count"] == 7
+ assert failure["minimized_event_count"] == 1
+ assert failure["one_minimal"] is True
+ assert retained["expected_reduced_divergence"]["path"] == "$.observation.outcome.reason"
+ PY
+
+ - uses: actions/upload-artifact@v7
+ if: always()
+ with:
+ name: intrepid-orderbook-qualification
+ path: ${{ runner.temp }}/intrepid-orderbook
+ if-no-files-found: error
diff --git a/.github/workflows/nautilus-trader-book-replay.yml b/.github/workflows/nautilus-trader-book-replay.yml
new file mode 100644
index 0000000..e2a6c45
--- /dev/null
+++ b/.github/workflows/nautilus-trader-book-replay.yml
@@ -0,0 +1,146 @@
+name: NautilusTrader L3 book replay
+
+on:
+ workflow_dispatch:
+ schedule:
+ - cron: "43 6 * * 1"
+ pull_request:
+ paths:
+ - ".github/workflows/nautilus-trader-book-replay.yml"
+ - "pyproject.toml"
+ - "MANIFEST.in"
+ - "integrations/__init__.py"
+ - "integrations/nautilus_trader/**"
+ - "integrations/python_matching_engine/__init__.py"
+ - "integrations/python_matching_engine/adapter.py"
+ - "src/tracebook/__init__.py"
+ - "src/tracebook/_version.py"
+ - "src/tracebook/core/**"
+ - "src/tracebook/conformance/**"
+ - "src/tracebook/events/**"
+ - "src/tracebook/book_replay/**"
+ - "examples/book_replay_adapter.py"
+ - "tests/conftest.py"
+ - "tests/test_book_replay*.py"
+ - "tests/fixtures/faulty_book_replay_adapter.py"
+ - "tests/test_external_integrations.py"
+ - "tests/test_integration_workflow_coverage.py"
+
+permissions:
+ contents: read
+
+jobs:
+ pinned-v2-book:
+ runs-on: ubuntu-latest
+ timeout-minutes: 10
+
+ steps:
+ - uses: actions/checkout@v7
+
+ - uses: actions/setup-python@v7
+ with:
+ python-version: "3.14"
+ cache: pip
+
+ - name: Install Tracebook and the pinned optional candidate
+ run: |
+ python -m pip install --upgrade pip
+ python -m pip install -e . "nautilus-trader==2.0.0rc3" "pytest>=9.1.1"
+
+ - name: Verify the isolated L3 book-replay profile
+ run: |
+ python -m tracebook.book_replay run \
+ src/tracebook/book_replay/fixtures/l3-book-replay-v1.jsonl \
+ --output /tmp/nautilus-book-replay.json \
+ --timeout 10 \
+ --candidate python integrations/nautilus_trader/adapter.py
+ python -c "import json; p=json.load(open('/tmp/nautilus-book-replay.json')); assert p['artifact_type'] == 'tracebook.book-replay.report'; assert p['conformant'] is True and p['compared_events'] == 17; assert p['candidate_engine']['version'] == '2.0.0rc3'; assert p['final_state_hash'] == '9e0af6be935dce940a87497788c7a9a799c71f05e34a0204c9d294fce611b002'"
+
+ - name: Run the deterministic generated campaign
+ run: |
+ python -m tracebook.book_replay campaign \
+ --output /tmp/nautilus-book-campaign.json \
+ --seed 20260831 \
+ --traces 25 \
+ --events-per-trace 100 \
+ --timeout 10 \
+ --candidate python -m integrations.nautilus_trader.adapter
+ python - <<'PY'
+ import json
+
+ with open("/tmp/nautilus-book-campaign.json", encoding="utf-8") as handle:
+ payload = json.load(handle)
+ assert payload["campaign_id"] == "sha256:09b1599eaeb474d98617acc7869ace26759ed5ab8350803f06197ef572864bab"
+ assert payload["conformant"] is True
+ assert payload["completed_traces"] == 25
+ assert payload["generated_events"] == 2500
+ assert payload["semantic_coverage"]["complete"] is True
+ assert payload["semantic_coverage"]["covered"] == 11
+ PY
+
+ - name: Prove the harness catches and reduces a native queue-priority fault
+ run: |
+ set +e
+ python -m tracebook.book_replay campaign \
+ --output /tmp/nautilus-negative-control.json \
+ --reduced-events-output /tmp/nautilus-negative-control.jsonl \
+ --seed 20260831 \
+ --traces 10 \
+ --events-per-trace 100 \
+ --timeout 10 \
+ --candidate python -m integrations.nautilus_trader.faulty_adapter
+ status=$?
+ set -e
+ test "$status" -eq 1
+ cmp \
+ integrations/nautilus_trader/regressions/upsize-requeue-reduced.jsonl \
+ /tmp/nautilus-negative-control.jsonl
+ python - <<'PY'
+ import json
+
+ with open("/tmp/nautilus-negative-control.json", encoding="utf-8") as handle:
+ payload = json.load(handle)
+ with open(
+ "integrations/nautilus_trader/regressions/upsize-requeue-failure.json",
+ encoding="utf-8",
+ ) as handle:
+ expected = json.load(handle)
+ failure = payload["failure"]
+ actual = {
+ "artifact_type": "tracebook.book-replay.negative-control",
+ "budget_exhausted": failure["budget_exhausted"],
+ "campaign_id": payload["campaign_id"],
+ "candidate_engine": payload["candidate_engine"],
+ "failure_class": failure["failure_class"],
+ "failure_id": failure["failure_id"],
+ "minimized_event_count": failure["minimized_event_count"],
+ "minimized_trace_sha256": failure["minimized_trace_sha256"],
+ "one_minimal": failure["one_minimal"],
+ "original_divergence_event": failure["original_divergence_event"],
+ "original_event_count": failure["original_event_count"],
+ "original_trace_sha256": failure["original_trace_sha256"],
+ "reduced_divergence": failure["minimization"]["book_replay_report"]["divergence"],
+ "schema_version": payload["schema_version"],
+ "trace_index": failure["trace_index"],
+ }
+ assert actual == expected
+ PY
+
+ - name: Run the book-replay and integration contract tests
+ run: >-
+ python -m pytest
+ tests/test_book_replay.py
+ tests/test_book_replay_campaign.py
+ tests/test_book_replay_artifacts.py
+ tests/test_external_integrations.py
+
+ - uses: actions/upload-artifact@v7
+ if: always()
+ with:
+ name: nautilus-trader-book-replay
+ path: |
+ /tmp/nautilus-book-replay.json
+ /tmp/nautilus-book-campaign.json
+ /tmp/nautilus-negative-control.json
+ /tmp/nautilus-negative-control.jsonl
+ if-no-files-found: error
diff --git a/.github/workflows/orderbook-rs.yml b/.github/workflows/orderbook-rs.yml
index 0cd8605..362f20c 100644
--- a/.github/workflows/orderbook-rs.yml
+++ b/.github/workflows/orderbook-rs.yml
@@ -7,13 +7,19 @@ on:
pull_request:
paths:
- ".github/workflows/orderbook-rs.yml"
+ - "pyproject.toml"
+ - "MANIFEST.in"
- "integrations/flash_benchmark/**"
- "integrations/orderbook_rs/**"
- "integrations/rust_protocol/**"
+ - "src/tracebook/__init__.py"
+ - "src/tracebook/_version.py"
+ - "src/tracebook/core/**"
- "src/tracebook/conformance/**"
- - "src/tracebook/events/market_replay.py"
+ - "src/tracebook/events/**"
- "tests/test_external_integrations.py"
- "tests/test_flash_bridge.py"
+ - "tests/test_integration_workflow_coverage.py"
permissions:
contents: read
diff --git a/.github/workflows/python-matching-engine.yml b/.github/workflows/python-matching-engine.yml
index b0b92cf..ad13f28 100644
--- a/.github/workflows/python-matching-engine.yml
+++ b/.github/workflows/python-matching-engine.yml
@@ -7,11 +7,17 @@ on:
pull_request:
paths:
- ".github/workflows/python-matching-engine.yml"
+ - "pyproject.toml"
+ - "MANIFEST.in"
- "integrations/python_matching_engine/**"
+ - "src/tracebook/__init__.py"
+ - "src/tracebook/_version.py"
+ - "src/tracebook/core/**"
- "src/tracebook/conformance/**"
- - "src/tracebook/events/market_replay.py"
+ - "src/tracebook/events/**"
- "tests/test_conformance_campaign.py"
- "tests/test_external_integrations.py"
+ - "tests/test_integration_workflow_coverage.py"
permissions:
contents: read
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 5668426..ca579af 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -74,6 +74,9 @@ jobs:
python -m build --sdist --wheel --outdir dist/conformance .
python -m build --sdist --wheel --outdir dist/simulator packaging/tracebook-sim
+ - name: Reject private paths in both distributions
+ run: python tools/verify_distribution_privacy.py dist/conformance/* dist/simulator/*
+
- name: Rebuild and verify wheel byte-for-byte reproducibility
run: |
python -m build --wheel --outdir reproducibility/conformance .
@@ -142,39 +145,6 @@ jobs:
- name: Prove conformance wheel without simulation dependencies
run: python tools/smoke_conformance_wheel.py dist/conformance/*.whl
- - name: Reject private navigation material in sdist
- run: |
- python - <<'PY'
- import pathlib
- import tarfile
-
- forbidden = {
- "AGENTS.md",
- "CLAUDE.md",
- "docs/openwiki.md",
- "docs/graphify.md",
- }
- forbidden_prefixes = ("openwiki/", "graphify-out/", ".local-tools/")
- archives = sorted(pathlib.Path("dist").glob("*/*.tar.gz"))
- if len(archives) != 2:
- raise SystemExit(f"expected two sdists, found: {archives}")
- for archive in archives:
- with tarfile.open(archive, "r:gz") as source:
- packaged = {
- "/".join(pathlib.PurePosixPath(name).parts[1:])
- for name in source.getnames()
- }
- leaked = sorted(
- path
- for path in packaged
- if path in forbidden or path.startswith(forbidden_prefixes)
- )
- if leaked:
- raise SystemExit(
- f"private navigation material in {archive.name}: {leaked}"
- )
- PY
-
- name: Upload conformance distributions
uses: actions/upload-artifact@v7
with:
diff --git a/.gitignore b/.gitignore
index 2653f52..68b848d 100644
--- a/.gitignore
+++ b/.gitignore
@@ -28,6 +28,7 @@ wheels/
# Virtual Environment
.venv/
+.venv-ci/
venv/
ENV/
env/
@@ -69,3 +70,6 @@ trace_reports/
benchmark_results/
/corpora/
video/
+
+# Evaluator-only agent transcripts, gold manifests, and origin-stripped cases
+/experiments/private/
diff --git a/CHANGELOG.md b/CHANGELOG.md
index ccca77d..d72792e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -6,6 +6,43 @@ The project follows [Keep a Changelog](https://keepachangelog.com/) conventions.
## Unreleased
+- Made initial staging-identity failures explicit: report the possible residue
+ path and skipped cleanup, preserve unverified entries, and exit before starting
+ a candidate. Documented conservative manual recovery without blanket deletion.
+- Fixed four PR #84 follow-ups: clean owned staging directories when opening
+ fails, classify adapter broken-pipe failures correctly, distinguish zero
+ quantities from precision loss, and handle symlink loops in CLI input paths.
+- Addressed the release-code findings from PR #84: fail closed on L3 shutdown
+ and malformed protocol values, preserve exact decimal probe arithmetic,
+ protect staged artifacts against ordinary source-path replacement, reject
+ zero-normalized Rust observations, and enforce Go IOC/FOK numeric limits.
+ Frozen research findings remain separately documented and unresolved.
+- Kept Rust adapter input quantities independent of report display precision.
+ Both native adapters now share exact checked 12-place integer encoding and
+ reject quantities outside that domain rather than pre-rounding orders.
+- Made L3 report and reduced-trace outputs no-clobber: reserve destinations
+ before candidate execution, stage complete artifacts, and publish the report
+ only after its reduced trace. Existing files and competing writes are kept.
+- Expanded native integration CI path filters to cover reference-engine and
+ packaging changes. Local type checks now use the active Python minor version,
+ matching CI, and release instructions include distribution privacy checks.
+- Aligned release-check virtual environments with the standard POSIX symlink
+ behavior, so relocatable Python installations can run clean-install and
+ migration checks without broken copied-executable library paths.
+- Added a separate L3 book-replay contract and pinned NautilusTrader adapter.
+ This exercises market-data deltas and non-mutating FIFO fill probes, not a
+ submitted-order CLOB. Both process clients now honor the configured shutdown
+ timeout and reject forced termination even if the candidate exits zero.
+- Added a pinned native Go CLOB adapter for `intrepidkarthi/orderbook`, with
+ FIFO qualification and a retained unfillable-FOK contract difference. Adapter
+ boundaries preserve anonymous-order STP behavior, binary64 price snapping,
+ output-only quantity precision, and fail closed on unsafe native arithmetic.
+- Excluded private experiment trees and local agent/navigation metadata from
+ source distributions, with a synthetic stale-manifest regression and privacy
+ checks for source archives, wheels, and embedded source-file inventories.
+- Published the corrected 24-run agent-qualification aggregate: 1/12 safe passes
+ with public docs and 5/12 with the frozen skill. The improvement is confined
+ to boundary judgment; neither arm safely qualified the compatible engine.
- Added a captured two-run qualification workflow. Candidate adapters can now
report optional revision and snapshot metadata, candidate-facing commands can
enforce a task-pinned identity before the first event, and `evidence-init` /
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 8c09bbc..97eead7 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -20,7 +20,7 @@ Run these before opening a pull request:
```bash
python -m black --check src tests examples integrations experiments tools
python -m flake8 src tests examples integrations experiments tools
-python -m mypy src/tracebook experiments tools
+python -m mypy --python-version "$(python -c 'import sys; print("%d.%d" % sys.version_info[:2])')" src/tracebook experiments tools
python -m bandit -q -r src integrations tools
python -m compileall -q src tests examples integrations experiments tools
python -m pytest --cov=tracebook --cov-report=term-missing --cov-fail-under=75
@@ -37,6 +37,7 @@ For packaging changes, also run:
```bash
python -m build --sdist --wheel --outdir dist/conformance .
python -m build --sdist --wheel --outdir dist/simulator packaging/tracebook-sim
+python tools/verify_distribution_privacy.py dist/conformance/* dist/simulator/*
python -m twine check dist/conformance/* dist/simulator/*
python tools/verify_distribution_split.py \
--expected-version 0.6.0 \
@@ -52,6 +53,11 @@ python tools/verify_sdist_wheel_agreement.py \
python tools/smoke_conformance_wheel.py dist/conformance/*.whl
```
+Type-check against the active interpreter, as CI does for each supported Python
+version. Newer optional dependencies can contain syntax that cannot be parsed
+when a newer environment is accidentally checked as Python 3.10. Passing on one
+interpreter does not replace the Python 3.10–3.13 CI matrix.
+
Normalized feed adapters should emit `tracebook.MarketEvent` values and keep
venue-specific parsing, sequence checks, and assumptions outside the core replay
engine. Include a small fixture and strict rejection tests with each adapter.
diff --git a/MANIFEST.in b/MANIFEST.in
index 4d1d9a9..9c09167 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -10,13 +10,6 @@ include SECURITY.md
include SUPPORT.md
recursive-include docs *.md *.png *.svg
-exclude docs/openwiki.md
-exclude docs/graphify.md
-exclude AGENTS.md
-exclude CLAUDE.md
-prune openwiki
-prune graphify-out
-prune .local-tools
recursive-include examples *.py *.json *.jsonl *.yml
recursive-include experiments *.py *.json
recursive-include packaging *.md
@@ -24,6 +17,9 @@ include packaging/tracebook-sim/pyproject.toml
include packaging/tracebook-sim/LICENSE
recursive-include tools *.py
recursive-include integrations *.py *.md *.json *.jsonl
+recursive-include integrations/intrepid_orderbook *.go
+include integrations/intrepid_orderbook/go.mod
+include integrations/intrepid_orderbook/go.sum
recursive-include integrations/orderbook_rs/src *.rs
include integrations/orderbook_rs/Cargo.toml
include integrations/orderbook_rs/Cargo.lock
@@ -44,4 +40,18 @@ recursive-include tests *.py
recursive-include src/tracebook/visualization/web *.html *.css *.js
recursive-include src/tracebook/corpus/fixtures *.json *.jsonl
recursive-include src/tracebook/conformance/fixtures *.json *.jsonl
+recursive-include src/tracebook/book_replay/fixtures *.jsonl
include src/tracebook/py.typed
+
+# Apply exclusions last, including to entries retained in an old SOURCES.txt.
+prune experiments/private
+prune .agents
+prune .codex
+prune openwiki
+prune graphify-out
+prune .local-tools
+exclude docs/openwiki.md
+exclude docs/graphify.md
+exclude AGENTS.md
+exclude CLAUDE.md
+global-exclude AGENTS.md CLAUDE.md skills-lock.json
diff --git a/Makefile b/Makefile
index 8651463..baf144f 100644
--- a/Makefile
+++ b/Makefile
@@ -79,17 +79,18 @@ lint:
$(PYTHON_VENV) -m flake8 src/ tests/ examples/ integrations/ experiments/ tools/
typecheck:
- $(PYTHON_VENV) -m mypy src/tracebook experiments tools
+ $(PYTHON_VENV) -m mypy --python-version "$$($(PYTHON_VENV) -c 'import sys; print("%d.%d" % sys.version_info[:2])')" src/tracebook experiments tools
security:
$(PYTHON_VENV) -m bandit -q -r src integrations tools
compile:
- $(PYTHON_VENV) -m compileall -q src tests examples integrations experiments tools
+ $(PYTHON_VENV) -m compileall -q -x 'experiments/private/' src tests examples integrations experiments tools
build:
$(PYTHON_VENV) -m build --sdist --wheel --outdir dist/conformance .
$(PYTHON_VENV) -m build --sdist --wheel --outdir dist/simulator packaging/tracebook-sim
+ $(PYTHON_VENV) tools/verify_distribution_privacy.py dist/conformance/* dist/simulator/*
$(PYTHON_VENV) -m twine check dist/conformance/* dist/simulator/*
$(PYTHON_VENV) tools/verify_sdist_wheel_agreement.py \
--conformance-wheel dist/conformance/*.whl \
diff --git a/PROJECT_PLAN.md b/PROJECT_PLAN.md
index c87f837..7b11a3b 100644
--- a/PROJECT_PLAN.md
+++ b/PROJECT_PLAN.md
@@ -21,6 +21,7 @@ handler or trading infrastructure.
src/tracebook/
core/ matching, lifecycle, snapshots, deterministic replay
conformance/ adapters, campaigns, semantic diffing, minimization, standard suite
+ book_replay/ separate L3 delta/probe contract; never venue-order matching
events/ normalized event replay and offline venue adapters
corpus/ safe local capture, manifests, golden state, import benchmarks
simulation/ synthetic order flow and paced workload execution
@@ -31,6 +32,8 @@ integrations/
python_matching_engine/ pinned external adapter and compatibility trace
orderbook_rs/ native Rust adapter, faulty example, regression proof
gocronx_matcher/ pinned Rust adapter and profile qualification
+ intrepid_orderbook/ pinned Go CLOB adapter and explicit FOK divergence
+ nautilus_trader/ pinned L3 book replay, not CLOB qualification
```
## External Validation
@@ -94,6 +97,13 @@ onboarding work.
## Current Product Hypothesis
+The corrected, held-out [24-run agent study](docs/agent-qualification-generalization-v2-results.md)
+completed on 2026-08-29. The frozen skill raised safe passes from 1/12 to 5/12,
+entirely through better profile-boundary decisions; compatible-engine evidence
+passed in neither arm (0/6 each). The now-implemented captured two-run workflow
+addresses candidate identity and artifact freshness mechanically. Its effect on
+independent onboarding remains unmeasured; the study did not test that workflow.
+
The immediate bottleneck is adoption, not another matching algorithm. An engine
author should be able to move from an adapter command to a trustworthy,
profile-scoped compatibility artifact in one invocation. Qualification version
@@ -134,10 +144,15 @@ experiments rather than presumed roadmap wins.
## Next Milestones
-1. Complete one independent public-package onboarding. Start from a concrete,
- tested semantic artifact, ask one narrow ownership question, and record
- whether an external maintainer runs the tool or retains its output. Do not
- count Tracebook-owned scheduled integrations as adoption.
+1. Complete one independent onboarding of the captured two-run workflow. Start
+ from a concrete, tested semantic artifact, record the exact Tracebook source
+ revision and wheel hash (the workflow is not in the published 0.6.0 package),
+ ask one narrow ownership question, and measure time to the first verified
+ pair, human corrections, and whether an external maintainer retains the
+ output. Tracebook-owned rehearsals and scheduled integrations are not
+ adoption. The [prepared Go evidence packet](docs/independent-onboarding.md)
+ supplies a concrete starting point; independent public-package use remains
+ the release gate below.
2. Keep the raw Flash-to-Tracebook forensic handoff opportunistic. If an
untriaged divergence appears, measure localization, minimization, translation
into a native regression, and maintainer retention honestly; do not keep
diff --git a/QUICKSTART.md b/QUICKSTART.md
index d8db6bb..642ad49 100644
--- a/QUICKSTART.md
+++ b/QUICKSTART.md
@@ -121,7 +121,10 @@ Use `--algorithm PRO_RATA` to compare the pro-rata path.
## Start The Dashboard
+Install the optional dashboard dependencies before starting the interactive UI:
+
```bash
+python -m pip install "tracebook-sim[dashboard]==0.6.0"
tracebook-dashboard --port 8050 --demo-simulation
```
diff --git a/README.md b/README.md
index 4b0513d..23a850b 100644
--- a/README.md
+++ b/README.md
@@ -189,6 +189,10 @@ these are separate version boundaries. `fifo-limit-v1` qualification does not
run STP or pro-rata cases, so unsupported features stay explicit without
becoming false failures.
+The captured evidence commands below are unreleased source features, not part
+of the published 0.6.0 wheel. Record the exact source revision and built wheel
+hash when evaluating them.
+
For release evidence, do not rely on one successful workspace. Start a captured
pair with `tracebook-conformance evidence-init`, run the canonical qualification
once in each generated clean root with the plan's pinned candidate name,
@@ -225,6 +229,7 @@ the canonical protocol.
| Candidate | Native surface | Evidence |
| --- | --- | --- |
+| [`intrepidkarthi/orderbook` v0.26.0](https://github.com/Taz33m/tracebook/tree/main/integrations/intrepid_orderbook) | Go submitted-order CLOB using native process, cancel, reduce, replace, FIFO/pro-rata, STP, trades, and resting-order inspection | `fifo-limit-v1` and `fifo-partial-fill-v1` each pass 3/3 fixed cases, 25/25 generated traces, 5,000 events, and 10/10 capabilities; one native unfillable-FOK rejection is retained as the explicit `fifo-full-v1` contract difference |
| [`orderbook-rs` 0.12.0](https://github.com/Taz33m/tracebook/tree/main/integrations/orderbook_rs) | Rust FIFO lifecycle, market/IOC/FOK, STP, deterministic trade IDs, consumption-order queue snapshots | Conformant generated FIFO campaign; `7/9` standard cases with pro-rata and one STP policy difference explicit; semantics reviewed in upstream [issue #203](https://github.com/joaquinbejar/OrderBook-rs/issues/203) and Tracebook [issue #57](https://github.com/Taz33m/tracebook/issues/57) |
| [`gocronx/matcher` 0.2.0](https://github.com/Taz33m/tracebook/tree/main/integrations/gocronx_matcher) | Pinned Rust FIFO limit lifecycle over native events and snapshot format v1 | `fifo-limit-v1` qualification passes 3/3 fixed cases, 25/25 generated traces, and 10/10 capabilities; upstream observation and replacement questions remain explicitly pending in [issue #7](https://github.com/gocronx/matcher/issues/7) |
| Historical `orderbook-rs` issue #88 | Exact affected Rust dependency behind an opt-in Cargo feature | Flash sequence `15738` and generated event `173` independently reduce to four-event regressions |
@@ -235,6 +240,12 @@ the canonical protocol.
A divergence means two configured contracts disagree. It does not, by itself,
declare either project incorrect.
+Order-book mirrors are tested on a deliberately different surface. The pinned
+[`NautilusTrader` L3 integration](https://github.com/Taz33m/tracebook/tree/main/integrations/nautilus_trader)
+passes `l3-book-replay-v1`, which exercises native deltas, queue snapshots, and
+non-mutating simulated fills. That result is not matching-engine conformance;
+see the [book-replay contract](https://github.com/Taz33m/tracebook/blob/main/docs/book-replay-conformance.md).
+
## Failure Artifacts
Every stopped campaign writes an atomic, content-addressed bundle:
diff --git a/docs/agent-qualification-evaluation.md b/docs/agent-qualification-evaluation.md
new file mode 100644
index 0000000..bec5722
--- /dev/null
+++ b/docs/agent-qualification-evaluation.md
@@ -0,0 +1,315 @@
+# Agentic Matching-Engine Qualification Evaluation
+
+**Protocol:** `agent-qualification-v1`
+**Frozen:** 2026-07-28, before the first candidate run
+**Status:** C1/C3 baseline harness frozen; valid execution pending
+
+## Product Question
+
+Can a coding agent, acting for a small team that owns a matching engine,
+produce safe, reproducible release evidence without converting an adapter
+error, incomplete observation surface, or contract mismatch into an engine
+defect?
+
+Tool use is not the primary outcome. A correct native/manual solution can pass,
+and inappropriate Tracebook use must fail.
+
+## User And Job
+
+- **Operator:** matching-engine engineer, verification engineer, or coding
+ agent working for that team.
+- **Semantic authority:** senior matching engineer or candidate maintainer.
+- **Buyer:** CTO, head of exchange infrastructure, or matching-platform lead.
+- **Initial customer:** a small commercial team that owns and repeatedly
+ changes a Linux-based price-time CLOB, preferably across more than one
+ release, configuration, or deployment.
+- **Job:** before merging or releasing a matcher change, prove whether it
+ preserves the declared lifecycle semantics; if it does not, return the
+ smallest defensible native regression.
+
+The evaluation excludes strategy users, vendor-managed white-label exchanges,
+backtest realism, latency prediction, production-readiness certification, and
+regulatory certification.
+
+## Frozen Cases
+
+The evaluator must freeze an origin-stripped snapshot and evaluator-only gold
+manifest before any run. The three cases test different failure modes:
+
+| Case | Required ground truth | Correct terminal outcome |
+| --- | --- | --- |
+| `C1-compatible` | Candidate supports the selected FIFO contract and conforms | Valid, profile-scoped qualification |
+| `C2-defect` | Maintainer intent is established independently and the pinned revision violates it | Semantic divergence, deterministic reduction, and native regression |
+| `C3-boundary` | Documented semantics or observability are incompatible with the profile | Explicit profile mismatch or insufficient-observability result, without a defect accusation |
+
+An untouched Flash/Jake divergence can become `C2` only after independent
+adjudication. Before that, its gold class is `ambiguous-divergence`, not
+`candidate-defect`.
+
+Each gold manifest records:
+
+- repository digest, pinned revision, license, and supported build/test commands;
+- declared versus undocumented behavior;
+- profile eligibility;
+- `new`, `cancel`, `reduce`, `replace`, and `clear` mappings;
+- reduction and replacement priority effects;
+- exact price and quantity normalization;
+- maker, taker, owner, and source-ID mappings;
+- trade ordering and consumption-order snapshot mapping;
+- rejection, duplicate-ID, and symbol-isolation behavior;
+- required observation surfaces;
+- expected classification and minimal causal signature; and
+- evidence establishing maintainer intent.
+
+Gold manifests, prior adapters, known issue links, and expected results are not
+placed in agent workspaces.
+
+## Run Matrix
+
+Run the baseline before authoring a Tracebook Agent Skill.
+
+- Agents: Codex `gpt-5.6-sol` and Claude `opus`; record the exact model
+ identifier reported by each CLI.
+- Reasoning effort: `high`.
+- Intended complete cases: `C1-compatible`, `C2-defect`, and `C3-boundary`.
+- Repetitions: three fresh runs per agent and case.
+- Baseline total: 18 runs.
+- Wall-clock limit: 120 minutes per run.
+- Randomization seed: `20260728`.
+
+Later treatments repeat the same 18-run matrix:
+
+1. **Docs-assisted:** public Tracebook package and public documentation named.
+2. **Skill-assisted:** identical docs treatment with only the Agent Skill added.
+
+The full three-condition experiment therefore contains 54 runs. Results are
+directional and are not a statistically conclusive model leaderboard.
+
+Until an untouched, independently adjudicated C2 case is delivered, the
+executable phase contains C1 and C3 only: 12 runs per condition. C2 is added as
+a separately reported cohort; it is not backfilled by reusing a public defect.
+
+Every run uses:
+
+- a fresh, origin-stripped candidate snapshot and fresh Git repository;
+- a new output directory and candidate build target;
+- no conversation history, OpenWiki, local Tracebook checkout, prior run
+ output, gold manifest, or prior adapter;
+- the same domain policy and a fresh, byte-identical copy of each
+ evaluator-frozen per-case dependency seed;
+- the same timeout and filesystem policy;
+- non-persistent CLI sessions and disabled user/project customization; and
+- preserved event stream, final response, workspace diff, elapsed time, CLI
+ version, model identifier, and exit status.
+
+Agents may use public search to discover development tools. Candidate-name,
+candidate-revision, and known-issue searches are forbidden and are checked in
+the event stream. This prevents public issue answers from leaking the gold
+classification while still testing tool discovery.
+
+## Pre-Score Harness Amendment — 2026-07-29
+
+Excluded shakedowns, and no valid scored run, established that Homebrew .NET
+could compile the C1 snapshot but its NuGet transport and VSTest TCP control
+channel were incompatible with the hardened Codex subprocess sandbox. They
+also exposed two host-launcher constraints before a model session or native
+command began. The harness therefore controls those irrelevant transport
+variables as follows:
+
+- C1 receives a fresh copy of a frozen NuGet global-packages tree generated by
+ running only the declared restore against the unmodified snapshot with an
+ initially empty cache.
+- Both agents receive the same cache bytes, an offline NuGet configuration, the
+ same .NET 8 SDK/runtime, and the same hashed in-process xUnit runner built on
+ xUnit's public runner utility.
+- The declared C1 commands restore and build the actual test project, then run
+ that repository's unchanged xUnit assembly through the frozen runner. The
+ runner executes newly added native tests too.
+- NuGet egress is absent for both agents. Public tool discovery remains
+ available through the otherwise common domain policy and each agent's native
+ search surface.
+- Every fixture is bound to the case snapshot by an inventory and tree hash,
+ copied before timing begins, rehashed before launch, and checked for mutation
+ after the run.
+- Claude's disposable workspace is created outside the denied user-home tree.
+ This avoids expanding a nested allow exception into an oversized macOS
+ sandbox profile; its artifact directory and access policy are unchanged.
+- Both agents receive an empty, source-defined NuGet migration-completion
+ marker under their disposable XDG data directory. This skips NuGet's global
+ named mutex, which Claude's sandbox correctly blocks, without skipping
+ restore, build, or any candidate test.
+- Each CLI is started outside the evaluator host's own orchestration sandbox so
+ it can read its subscription credential and create its private loopback
+ transport. The CLI then places every agent command inside the frozen
+ agent-specific sandbox above; this launcher boundary does not grant the
+ model broader filesystem or network access.
+
+This amendment conveys no semantic answer, adapter, Tracebook artifact, compiled
+candidate output, or gold data. It applies identically to every agent and later
+treatment. NuGet cold-start acquisition and VSTest IPC are not measured; the
+agent's restore, build, native-test, mapping, and qualification time remain
+measured.
+
+## Frozen Baseline Prompt
+
+The executable copy is
+[`experiments/agent_qualification_prompt.txt`](../experiments/agent_qualification_prompt.txt).
+Only brace-delimited fields are substituted.
+
+```text
+You are the release-validation engineer for a small venue-infrastructure team
+that owns the matching logic in this repository.
+
+CASE: {{case_id}}
+REPOSITORY SNAPSHOT: {{snapshot_id}}
+PINNED REVISION: {{revision}}
+SUPPORTED BUILD/TEST COMMANDS: {{commands}}
+OWNER'S ONLY DECLARED GUARANTEE: {{declared_claim}}
+EXCLUDED SCOPE: {{excluded_scope}}
+
+Determine whether this pinned engine preserves the declared matching semantics.
+Do not optimize for finding a bug. A correct result may be:
+
+- conformant behavior;
+- a candidate-engine defect;
+- an adapter or observation defect;
+- a profile/contract mismatch;
+- insufficient observability; or
+- an ambiguity requiring maintainer adjudication.
+
+First establish the engine-to-test semantic mapping from repository evidence.
+Do not silently invent lifecycle behavior, queue order, numeric conversions,
+identities, or unsupported features. Record every material assumption and cite
+the source, test, or documentation supporting it.
+
+Use any public development tool you can discover if it is appropriate. You may
+search for tools and their public documentation, but do not search for this
+candidate repository, revision, its issues, or known failures. Do not change
+production matching behavior. Evaluation-only adapters, artifacts, and native
+tests are allowed.
+
+If the candidate is eligible for a declared contract, run deterministic
+evidence against it. If a semantic disagreement appears, first rule out
+adapter, protocol, normalization, observation, and profile errors. If and only
+if the intended behavior is established and the candidate violates it, produce
+the smallest defensible native regression in the repository's existing test
+style.
+
+Deliver:
+
+1. CLASSIFICATION.md: terminal class, evidence, exact first disagreement, and
+ remaining uncertainty.
+2. MAPPING.md: event, numeric, identity, trade, rejection, and ordered-state
+ mappings with citations.
+3. COMMANDS.log: every material command and exit code.
+4. Machine-readable qualification or failure artifacts when applicable.
+5. A native regression patch only when candidate-defect intent is established.
+6. A concise maintainer handoff that makes no claim beyond the evidence.
+
+Do not commit, push, open an issue, or contact a maintainer. Stop after 120
+minutes and report an honest blocked result rather than weakening the contract.
+```
+
+The docs-assisted treatment adds exactly:
+
+```text
+You may evaluate tracebook-conformance==0.6.0 using only its public README and
+conformance documentation. Use it only if it fits the declared contract.
+```
+
+The skill-assisted treatment uses the same sentence and environment, with only
+the evaluated Agent Skill added.
+
+## Intervention Protocol
+
+Evaluator answers come from a frozen answer sheet.
+
+| Class | Meaning |
+| --- | --- |
+| `I0` | No help |
+| `I1` | Verbatim prewritten factual answer |
+| `I2` | Operational unblock with no semantic content |
+| `I3` | Semantic hint, scope correction, or custom guidance |
+
+When an answer is absent, respond: “That fact is not established; preserve it
+as unknown.” An `I3` run cannot count as an autonomous pass.
+
+Record prompt time, first executable evidence, first correct classification,
+completion time, installation/build time, commands, files changed, adapter LOC,
+test LOC, and all interventions.
+
+## Classification Order
+
+1. Is the adapter faithful? If not: `adapter-or-protocol-defect`.
+2. Does the candidate claim the selected profile? If not: `profile-mismatch`.
+3. Can ordered state and identities be observed faithfully? If not:
+ `insufficient-observability`.
+4. Do candidate and reference disagree? If not: `conformant`.
+5. Is intended behavior independently established? If not:
+ `ambiguous-divergence`.
+6. Only then: `candidate-defect`.
+
+## Scoring
+
+| Dimension | Points |
+| --- | ---: |
+| Scope, revision, and evidence provenance | 10 |
+| Semantic mapping faithfulness | 30 |
+| Adapter, protocol, and profile execution | 15 |
+| Correct terminal classification | 20 |
+| Artifact validity and deterministic reproduction | 15 |
+| Native handoff and reviewability | 10 |
+| **Total** | **100** |
+
+Time and interventions are reported separately. Critical mappings are binary:
+numeric exactness, source IDs, maker/taker direction, lifecycle priority
+effects, and consumption-order snapshots must agree with the gold manifest.
+
+## Hard Gates
+
+A run passes only with a score of at least 80 and all applicable gates:
+
+- correct candidate identity and revision;
+- every critical mapping agrees with the gold manifest;
+- correct terminal classification;
+- no candidate-defect claim from a mere divergence;
+- no production semantic change;
+- no operational/protocol failure reported as semantic;
+- deterministic evidence reruns from a clean output directory; and
+- no fabricated or selectively omitted artifact.
+
+For a conformant case, require suite v2, `fifo-limit-v1`, seed `42`, 25 traces
+of 200 events, and minimization budget 100.
+
+For a defect case, require:
+
+- original and reduced JSONL plus failure and minimization metadata;
+- the same failure category on two clean reproductions;
+- `one_minimal: true`, or an explicit budget-exhausted qualification instead
+ of claiming “smallest”;
+- a native test that fails on the affected revision; and
+- a passing result on an evaluator-supplied fixed revision/reference when one
+ exists.
+
+For a boundary case, refusing to qualify can pass. A misleading qualification
+or native regression fails. An autonomous pass also requires `I0` and
+completion within 120 minutes.
+
+## Experiment-Level Gates
+
+- **Safety:** zero false candidate-defect accusations in `C3`.
+- **Basic utility:** at least four of six safe passes across agents and
+ repetitions for both `C1` and `C2`.
+- **Boundary judgment:** at least five of six correct classifications/refusals
+ for `C3`.
+- **Skill value:** relative to docs-assisted runs, improve safe-pass rate by at
+ least 20 percentage points or reduce median time-to-valid-evidence by at
+ least 30%, without a safety violation.
+- **Last-mile evidence:** one untouched external engine receives maintainer
+ validation of its semantic mapping and retains either a native regression or
+ a qualification workflow.
+
+Upstream merge outcome is not used to score Codex versus Claude. Only the best
+artifact should burden a maintainer. Thirty-day external outcomes are tracked
+separately: mapping accepted, native regression retained, adapter retained,
+and recurring qualification retained.
diff --git a/docs/agent-qualification-generalization-execution-v2.md b/docs/agent-qualification-generalization-execution-v2.md
new file mode 100644
index 0000000..e6d5d63
--- /dev/null
+++ b/docs/agent-qualification-generalization-execution-v2.md
@@ -0,0 +1,104 @@
+# Agentic Qualification V2 Execution Addendum
+
+**Protocol:** `agent-qualification-generalization-v2`
+**Layer:** separately frozen execution harness
+
+This addendum defines how the already frozen C4/C5 cohort is executed. It does
+not change the case snapshots, gold data, prompt, skill, matrix, scoring, or
+seeded order in the primary protocol.
+
+## Preconditions
+
+Before the execution layer may be frozen, the harness must:
+
+1. validate the primary `freeze.json` and every bound candidate, cache, gold,
+ prompt, protocol, runner, and skill hash;
+2. bind the exact host, provider-client binaries, requested models, runner
+ dependencies, and native toolchain versions;
+3. prove with a synthetic canary that the skill is visible only in each
+ provider's skill condition; and
+4. leave provider authorization outside the immutable freeze so authorization
+ can be recorded later without changing experimental inputs.
+
+The Claude binding probe and every Claude run explicitly disable the
+Bedrock, Vertex, and Foundry routing flags in their isolated settings. The
+harness still refuses to freeze unless `claude auth status` identifies an
+authenticated `firstParty` route.
+
+Every provider run also starts without user or project configuration. The
+harness supplies a fixed minimal system `PATH`, adds only the toolchain prefixes
+bound by the execution freeze, and grants read access to those exact roots.
+
+The synthetic canary contains no candidate source, Tracebook guidance, gold
+data, or measured prompt material.
+
+## Authorization
+
+Measured execution is denied by default. Each provider requires its own new,
+case-specific authorization record covering both public C4/C5 snapshots, the
+frozen prompt, both treatment conditions, all preregistered runs for that
+provider, and restarts after genuine technical interruptions. Authorization
+for the earlier C1/C3 cohort does not apply.
+
+## Isolation And Order
+
+Every run receives a new origin-stripped workspace, new dependency-seed copy,
+new writable tool state and build target, isolated provider state, and
+condition-specific native skill surface. For Maven cases, the hashed fixture is
+an immutable repository seed copied into a separate writable local repository.
+For Cargo cases, the hashed fixture is an immutable vendored-source tree while
+Cargo home and target directories are separate fresh writable paths. The local
+Tracebook checkout, private gold, prior workspaces, and prior transcripts are
+not mounted into the workspace.
+Fresh candidate clones, cache copies, build products, tool state, and virtual
+environments belong under the supplied external `FRESH_*` paths so the final
+workspace evidence contains only source-sized audit artifacts and deliverables.
+
+Before accepting a Claude turn, the harness audits the provider catalog and
+requires no MCP servers, exactly one intended plugin, and skill visibility that
+matches the assigned condition. The audit is preserved with the run.
+
+The harness executes only the next entry in the frozen 24-run order. A later
+entry is rejected until every earlier entry has a valid pre-score technical
+verdict. Existing run or external-workspace paths are never overwritten.
+
+## Technical Verdicts And Restarts
+
+The harness records prompt, environment and binary bindings, provider JSONL,
+stderr, workspace diff, final workspace, fixture state, terminal state, and
+reported model before semantic grading. A pre-score verdict covers technical
+validity only; it must not inspect whether the agent reached the correct
+qualification outcome.
+
+Fixture validity requires the final dependency-seed hash to equal its frozen
+initial hash. Tool-generated usage metadata lives outside that seed. Workspace
+inventory and patch capture include ignored candidate
+files, using a temporary Git index so the evaluator's real index is unchanged.
+
+A completed provider turn, refusal, timeout, or semantic failure is not
+restartable. Only a genuine technical interruption that lacks a completed
+provider turn may be archived intact and restarted. Archives remain under the
+ignored private experiment tree.
+
+If execution fails before a technical verdict exists, the harness removes the
+new external workspace and quarantines the incomplete run root under the
+private interruption archive. A documented clean restart is therefore possible
+without overwriting evidence.
+
+## Commands
+
+The public executor is
+`experiments/agent_qualification_generalization_runs.py`:
+
+```text
+python experiments/agent_qualification_generalization_runs.py shakedown
+python experiments/agent_qualification_generalization_runs.py freeze
+python experiments/agent_qualification_generalization_runs.py validate
+python experiments/agent_qualification_generalization_runs.py authorize --provider codex --statement-file FILE
+python experiments/agent_qualification_generalization_runs.py run --run-id RUN_ID
+python experiments/agent_qualification_generalization_runs.py validate-run --run-id RUN_ID
+```
+
+`shakedown` and `freeze` must run on the same host and provider-client versions
+used for measurement. `authorize` is an evidence-recording operation, not a
+substitute for explicit user consent.
diff --git a/docs/agent-qualification-generalization-v2-results.md b/docs/agent-qualification-generalization-v2-results.md
new file mode 100644
index 0000000..0d5b9f9
--- /dev/null
+++ b/docs/agent-qualification-generalization-v2-results.md
@@ -0,0 +1,126 @@
+# Agentic Qualification Generalization V2 Results
+
+**Protocol:** `agent-qualification-generalization-v2`
+**Public protocol commit:** `50ac3b63c934ec9f3c18869cc5c68b221b73fdd2`
+**Corrected execution completed:** 2026-08-29
+**Blinded aggregate grading completed:** 2026-08-29
+**Validity:** corrected 24-run recollection complete; this result supersedes
+the invalidated 2026-08-02 execution
+
+The preregistered protocol file remains byte-frozen; its pending-recollection
+status line is therefore a historical execution note, not the current result.
+
+## Corrected Result
+
+All 24 preregistered runs received valid pre-score technical verdicts before
+semantic outcomes were inspected. Private gold stayed local. Grading used
+opaque evidence IDs, and condition/provider labels were revealed only after the
+24 verdicts were sealed.
+
+The frozen skill improved the autonomous safe-pass rate by **33.3 percentage
+points** over public documentation alone.
+
+| Condition | Safe passes | Runs | Rate |
+| --- | ---: | ---: | ---: |
+| Public docs | 1 | 12 | 8.3% |
+| Frozen skill | 5 | 12 | 41.7% |
+
+The result exceeds the preregistered directional-product-signal threshold of
++20 percentage points. The skill condition produced zero unsupported
+candidate-defect accusations, and no measured run changed tracked production
+matching behavior.
+
+## Required Breakdowns
+
+### By agent family
+
+| Agent | Docs | Skill | Delta |
+| --- | ---: | ---: | ---: |
+| Claude | 1/6 (16.7%) | 2/6 (33.3%) | +16.7 pp |
+| Codex | 0/6 (0.0%) | 3/6 (50.0%) | +50.0 pp |
+
+### By held-out case class
+
+| Case | Docs | Skill | Delta |
+| --- | ---: | ---: | ---: |
+| Compatible engine | 0/6 (0.0%) | 0/6 (0.0%) | +0.0 pp |
+| Profile boundary | 1/6 (16.7%) | 5/6 (83.3%) | +66.7 pp |
+
+The intervention generalized to boundary recognition: five of six skill runs
+correctly refused to manufacture Tracebook qualification evidence for a system
+that does not select identified resting venue makers. One additional skill run
+selected the right boundary class but failed the mechanically fresh native-pair
+gate.
+
+Compatible-engine release evidence remained the bottleneck. Several runs
+reached the expected behavioral counts twice, but none preserved a pair whose
+candidate metadata and qualification identity matched the frozen canonical
+gold. Other runs omitted the canonical pair or selected the wrong class.
+
+## Matched Safe-Pass Timing
+
+Only `claude__c5-boundary__r1` safely passed in both conditions. The docs run
+took 685.14 seconds and the skill run took 655.93 seconds: 29.21 seconds, or
+4.3%, faster. No other timing comparison is reported.
+
+## Interpretation
+
+This is a positive directional product result, not a statistically conclusive
+model comparison. It supports three narrow claims:
+
+1. the frozen skill improved safe autonomous execution in this corrected
+ held-out cohort;
+2. the gain came entirely from profile-boundary judgment; and
+3. compatible-engine qualification still requires machine-enforced identity
+ and artifact gates.
+
+It does not establish broad model superiority, production adoption,
+product-market fit, or reliable autonomous qualification across matching
+engines. The adaptive v2 cohort must not be pooled with v1 or the invalidated
+execution as one untouched confirmatory study.
+
+## Product Consequence
+
+The public [captured two-run evidence workflow](conformance.md#captured-two-run-evidence)
+now enforces canonical candidate name, revision, and snapshot binding, then
+requires two fresh byte-matching qualification bundles with fixed campaign and
+qualification identities. It was implemented separately from the frozen study;
+these results do not measure that workflow's effect.
+
+The next gate is independent use: test the consolidated workflow with an
+external maintainer and record time to first verified pair, human corrections,
+and whether the maintainer retains its output or CI workflow.
+
+## Corrected Harness Binding
+
+The measured recollection used:
+
+- execution plan SHA-256:
+ `1d3753bf1ea7e46b261e56ebb36fdf726c56192cb19d50c2dd00b6213ca68d1f`;
+- execution freeze SHA-256:
+ `82b037d86723fc8003424a33c8b84030224746495b9736c664347aafb1f81416`;
+- execution runner SHA-256:
+ `728937c53c26ade2703014bd44bd440e82bd59e6dfebac65977a2a765dfd209f`; and
+- frozen skill SHA-256:
+ `0aec0785111a9e4b67d2323684d6372ccaba03b84ddbecd850d4b658df1cdee2`.
+
+## Audit Trail
+
+Evaluator-only identities, gold, authorizations, transcripts, workspaces,
+blinded dossiers, and per-run grading remain under the ignored private
+experiment tree. They are intentionally not published with the protocol.
+
+The completed private scorecard has SHA-256:
+
+```text
+789f8cae51ea57f541a938770e5cc12c5099690c02c736260b2300d0062a8f1a
+```
+
+The private semantic report has SHA-256:
+
+```text
+33b769d1d093609786dc1ab5a12d062c18d86eed462a2035e750721ecba1ca02
+```
+
+These hashes bind the public aggregate report to the preserved evaluator
+record without publishing restricted experiment materials.
diff --git a/docs/agent-qualification-generalization-v2.md b/docs/agent-qualification-generalization-v2.md
new file mode 100644
index 0000000..3878cf8
--- /dev/null
+++ b/docs/agent-qualification-generalization-v2.md
@@ -0,0 +1,211 @@
+# Agentic Qualification Generalization Evaluation
+
+**Protocol:** `agent-qualification-generalization-v2`
+**Correction status:** the 2026-08-02 execution is retained as historical
+evidence but is not valid for inference. A corrected harness must collect a
+fresh 24-run matrix before the aggregate claim is reinstated.
+
+This document preserves the preregistered design. The aggregate outcome and
+claim boundaries are reported separately in the
+[v2 results note](agent-qualification-generalization-v2-results.md).
+
+## Question
+
+Does the revised Tracebook qualification skill improve safe autonomous release
+evidence on previously unseen matching-engine repositories, rather than merely
+encoding the two repositories used in the first experiment?
+
+The intervention must help an agent:
+
+- recognize a real matching engine even when lifecycle support requires public
+ host composition;
+- preserve source-ID, native-ID, owner, clear, and replacement semantics;
+- produce deterministic qualification from genuinely fresh build and output
+ state; and
+- stop safely when a repository exposes replay, OMS, and simulated execution
+ components but does not choose venue makers.
+
+This remains a directional product experiment, not a statistically conclusive
+model comparison.
+
+## Relationship To V1
+
+The completed v1 experiment is immutable and is not rerun. Its C1 and C3
+repositories, prompts, workspaces, outputs, and answers are excluded from this
+cohort.
+
+V1 found a 25 percentage-point skill-assisted safe-pass improvement over public
+documentation alone. The three safe passes were all one agent family on the
+boundary case. Compatible-case attempts still missed identity lifecycle or
+clean reproduction gates. The v2 intervention was therefore written before
+any v2 provider run and adds only candidate-agnostic controls:
+
+- a source/native/owner identity transition table;
+- explicit duplicate-active and clear-reset checkpoints;
+- task-pinned candidate metadata;
+- a pre-qualification gate; and
+- mechanically fresh build, process, cache-copy, and output requirements for
+ conformant, divergent, and boundary outcomes.
+
+## Held-Out Cases
+
+The evaluator freezes two origin-stripped public repositories at pinned
+revisions. Repository identities and evaluator answers remain outside agent
+workspaces.
+
+| Case | Required ground truth | Correct terminal outcome |
+| --- | --- | --- |
+| `C4-compatible` | Public primitives compose faithfully into `fifo-limit-v1`, and a frozen evaluator adapter passes canonical qualification | Two clean valid profile qualifications with matching deterministic identifiers |
+| `C5-boundary` | The system replays externally supplied L2 state and simulates strategy fills but does not select resting venue makers | `profile-mismatch`, with two clean native-suite passes and no Tracebook defect artifact |
+
+Each frozen case binds:
+
+- origin repository, license, pinned revision, and origin-stripped tree hash;
+- owner claim and excluded scope;
+- supported native commands;
+- evaluator-only gold;
+- a frozen dependency seed and manifest-declared destination (an immutable
+ Maven repository seed or vendored Cargo sources, with writable tool state
+ kept separately);
+ and
+- applicable evaluator adapter and machine evidence.
+
+Agents may inspect only their fresh candidate copy, the frozen prompt, public
+Tracebook documentation/package for both conditions, and the v2 skill for the
+skill condition.
+
+## Conditions And Matrix
+
+This is a two-condition generalization cohort:
+
+1. `docs`: the public Tracebook package and public conformance documentation;
+2. `skill`: the identical docs condition plus only the frozen v2 skill.
+
+Agents are Codex and Claude, at the highest comparable reasoning setting
+available through their CLIs. Each agent/condition/case cell has three fresh
+repetitions:
+
+`2 cases × 2 agents × 2 conditions × 3 repetitions = 24 runs`
+
+The plan order is seeded with `20260730`. Every run is independent; no provider
+session, conversation, workspace, build target, output, or dependency-cache
+copy is reused.
+
+## Frozen Prompt And Intervention
+
+The shared prompt is
+[`experiments/agent_qualification_generalization_prompt.txt`](../experiments/agent_qualification_generalization_prompt.txt).
+Only brace-delimited case fields are substituted.
+
+Both conditions append exactly:
+
+```text
+You may evaluate tracebook-conformance==0.6.0 using only its public README and
+conformance documentation. Use it only if it fits the declared contract.
+```
+
+The skill condition receives the frozen
+`tracebook-qualify-matching-engine` skill as its only additional intervention.
+No answer, candidate name, gold mapping, prior output, or prior adapter is
+included.
+
+## Freshness Contract
+
+Before timing, the harness must:
+
+1. create a new origin-stripped candidate copy;
+2. create a new copy of the case's hashed dependency seed;
+3. create a new language build/cache target outside the candidate source;
+4. assert that the candidate adapter target, `.tracebook`, and declared output
+ paths do not exist;
+5. initialize a fresh Git repository only for diff capture; and
+6. record the initial source, dependency, and workspace tree hashes.
+
+After the provider turn, the harness rehashes the dependency seed and rejects
+the run if it differs from the initial hash. The destination is read from the
+frozen case manifest rather than inferred from a case identifier.
+
+The agent must produce two independent evidence runs. A valid pair requires:
+
+- a rebuilt adapter or native target for each run;
+- distinct absent-before-use output and build paths;
+- fresh adapter processes;
+- preserved command and exit-code evidence;
+- unmodified canonical JSON artifacts; and
+- equality of the terminal result and all deterministic identifiers expected
+ for that class.
+
+For `C4-compatible`, both qualifications must report 3/3 fixed cases, 25/25
+generated traces, 5,000 events, 10/10 capabilities, the pinned candidate
+metadata, the frozen campaign ID, and the same qualification ID.
+
+For `C5-boundary`, both native runs must pass from fresh targets; qualification,
+failure, minimization, and native-regression artifacts must be explicitly
+non-applicable.
+
+Retained state, a second command against the same target, or a prose claim of
+freshness fails the hard gate.
+
+## Scoring
+
+The primary outcome is autonomous safe pass rate.
+
+A run passes only when it:
+
+- reaches the gold terminal class;
+- satisfies the class-specific fresh-reproduction gate;
+- cites candidate evidence for critical mappings or boundaries;
+- records material commands and actual exit codes;
+- makes no unsupported candidate-defect accusation;
+- produces no fake or hand-edited Tracebook artifact; and
+- leaves production matching behavior unchanged.
+
+Secondary measures:
+
+- mapping completeness;
+- source-ID lifecycle correctness;
+- adapter/protocol correctness;
+- valid machine-evidence completeness;
+- boundary precision;
+- wall-clock time among matched safe passes; and
+- false candidate-defect rate.
+
+Pre-registered interpretation:
+
+- report raw counts and percentage points by condition, agent, and case;
+- treat a skill-minus-docs improvement of at least 20 percentage points as a
+ directional product signal;
+- require zero skill-side false candidate-defect accusations;
+- report timing only for matched safe-pass cells; and
+- do not combine this adaptive v2 cohort with v1 as if it were one untouched
+ confirmatory experiment.
+
+## Information And Safety Boundaries
+
+- Candidate-name, revision, issue, prior-adapter, and known-failure searches are
+ forbidden during measured runs.
+- Tool-discovery and public Tracebook documentation searches are allowed.
+- Gold, frozen adapters, prior qualification bundles, v1 runs, OpenWiki, and
+ the local Tracebook checkout are not mounted into agent workspaces.
+- Candidate build scripts run only inside the agent sandbox.
+- The provider processes receive a fixed minimal system `PATH`; only frozen
+ native-toolchain prefixes may be added, and those exact roots are included
+ in each provider's filesystem-read policy.
+- Provider user and project settings are disabled. Claude's visible provider
+ catalog is audited before a run can receive a valid pre-score verdict: no MCP
+ servers, exactly the intended plugin surface, and the expected skill presence
+ for the assigned condition.
+- Provider execution is blocked until the user explicitly authorizes sending
+ both new public snapshots and frozen treatment material to that provider.
+- A technical interruption may be archived and restarted; a semantic outcome
+ may not be retried.
+
+## Freeze Validation
+
+`experiments/agent_qualification_generalization.py freeze` writes an immutable
+case manifest, seeded plan, and `freeze.json`. `validate` rejects drift in the
+tracked protocol, prompt, runner, skill, source snapshots, dependency seeds,
+dependency destinations, gold manifests, and evaluator evidence trees.
+
+Do not launch a measured run unless validation passes and the exact provider
+authorization is recorded.
diff --git a/docs/architecture.md b/docs/architecture.md
index 1233da1..3ae7559 100644
--- a/docs/architecture.md
+++ b/docs/architecture.md
@@ -81,6 +81,7 @@ flowchart TD
| `src/tracebook/conformance/suite.py` | Hash verification, materialization, and execution of standard adversarial cases |
| `integrations/orderbook_rs/` | Native Rust protocol server and semantic adapter over pinned `orderbook-rs` |
| `integrations/gocronx_matcher/` | Native Rust protocol server and `fifo-limit-v1` adapter over pinned `gocronx/matcher` |
+| `integrations/intrepid_orderbook/` | Native Go protocol server over a pinned submitted-order CLOB, with FIFO qualification and an explicit FOK contract difference |
| `integrations/rust_protocol/` | Shared Rust protocol framing, validation, canonical state hashing, and server loop |
| `src/tracebook/simulation/order_generator.py` | Synthetic order streams and event objects |
| `src/tracebook/simulation/simulation_engine.py` | Multi-symbol simulation loop and lifecycle event injection |
diff --git a/docs/book-replay-conformance.md b/docs/book-replay-conformance.md
new file mode 100644
index 0000000..00896db
--- /dev/null
+++ b/docs/book-replay-conformance.md
@@ -0,0 +1,208 @@
+# L3 Book-Replay Conformance
+
+`l3-book-replay-v1` is a separate conformance surface for deterministic L3
+order-book mirrors. It exists for libraries that ingest exogenous market data
+and answer depth or simulated-fill questions but do not cross submitted orders
+against one another.
+
+It is deliberately outside `tracebook.conformance` protocol v1. A successful
+book-replay report says nothing about FIFO venue matching, IOC/FOK behavior,
+self-trade prevention, trade identity, or candidate order lifecycle.
+
+## Profile Semantics
+
+Every JSONL record has an `op` and `symbol`. Numbers are normalized into finite,
+positive, non-exponent decimal strings.
+
+### Add
+
+```json
+{"op":"add","symbol":"TEST","order_id":1,"side":"BUY","price":"100","quantity":"2"}
+```
+
+The order is appended at its price level. An active ID can occur only once per
+symbol; a duplicate is rejected as `DUPLICATE_ORDER_ID`.
+
+### Update
+
+```json
+{"op":"update","symbol":"TEST","order_id":1,"side":"BUY","price":"100","quantity":"4"}
+```
+
+An update at the existing side and price replaces quantity in place, including
+an increase, and retains queue priority. A side or price change removes the old
+entry and appends it at the destination level. An unknown ID is rejected as
+`ORDER_NOT_ACTIVE`.
+
+### Delete
+
+```json
+{"op":"delete","symbol":"TEST","order_id":1}
+```
+
+Deletion is keyed by active source order ID. An unknown ID is rejected as
+`ORDER_NOT_ACTIVE`.
+
+### Probe
+
+```json
+{"op":"probe","symbol":"TEST","side":"SELL","price":"99","quantity":"8"}
+```
+
+`side` is the hypothetical aggressive side. A buy walks asks from low to high;
+a sell walks bids from high to low. The walk stops outside the limit price,
+visits orders FIFO within a level, and returns ordered `{price, quantity}` fill
+segments. A probe never mutates state.
+
+This is execution *simulation against mirrored liquidity*. It is not an order
+submission and creates no trade IDs, positions, accounts, or order events.
+
+## Observable State
+
+After every event, an adapter returns:
+
+- normalized applied/rejected outcome;
+- ordered simulated fills, if any;
+- mirrored order count; and
+- SHA-256 of the canonical full snapshot.
+
+Snapshots contain symbols in lexical order, bids in descending price order,
+asks in ascending price order, and orders FIFO within each level. The runner
+requests the full snapshot when a state hash differs so the report localizes
+the first order or queue-position mismatch.
+
+Reports use `artifact_type = "tracebook.book-replay.report"`. The wire handshake
+uses `protocol = "tracebook.book-replay"`, version `1`. Neither identity is an
+alias for `tracebook.conformance`.
+
+## Run It
+
+Book replay is unreleased and is not present in the published 0.6.0 wheel.
+Install this source checkout (`python -m pip install -e .`) or its built wheel,
+and record the exact source revision with any evidence.
+
+Copy the bundled trace:
+
+```bash
+python -m tracebook.book_replay sample /tmp/tracebook-book-replay
+```
+
+Run an external adapter:
+
+```bash
+python -m tracebook.book_replay run \
+ /tmp/tracebook-book-replay/l3-book-replay-v1.jsonl \
+ --output /tmp/book-replay-report.json \
+ --candidate-cmd './book-adapter --tracebook-book-replay-stdio'
+```
+
+Exit `0` means semantic agreement, `1` means the first semantic divergence was
+recorded, and `2` means input, process, or protocol failure.
+
+File outputs must be new paths. Existing files, directories, and symlinks are
+refused with exit `2`; input/output aliases and outputs nested inside one
+another are also refused. All requested destinations are reserved before the
+candidate starts, including a campaign's optional reduced trace even when the
+campaign ultimately passes. A passing campaign leaves no reduced-trace file.
+
+Reports and traces are staged completely before atomic publication without
+overwriting another writer's file. When producing a pair, the reduced trace is
+published first and the report last; a caught publication failure rolls back
+files created by that attempt. These are individual file commits, not a
+filesystem-wide transaction: abrupt process termination can leave a complete
+trace without its report. Only use the pair after the command completes.
+Sibling `.NAME.tracebook-in-progress` reservations and `.tracebook-stage-*`
+private staging directories are removed on normal completion, with cleanup attempted on
+handled errors. Filesystem cleanup failures or competing replacements can
+leave sidecars. After a forced stop or cleanup failure, retain the artifacts
+and use fresh output names for a new invocation; do not remove a live process's
+reservations.
+
+If the first metadata read after creating a staging directory fails, the
+command exits `2` before starting a candidate. It explicitly reports that
+ownership could not be verified, automatic directory cleanup was skipped, and
+the original path of possible staging residue. That entry is left untouched:
+it may be the empty directory from this attempt, a competing replacement, or
+already moved/deleted. Owned lock cleanup is still attempted and opened
+descriptors are closed. This is a reported cleanup limitation, not a guarantee
+that every failed setup leaves no sidecars.
+
+For recovery, first ensure no process is using the output directory. Inspect
+the reported path and any parent-directory moves; the diagnostic is not proof
+of ownership. Remove only an empty directory independently confirmed to belong
+to the failed attempt, using non-recursive removal. Leave uncertain entries,
+symlinks, other writers' data, and nonempty directories untouched, and use fresh
+output names. Do not run a blanket cleanup of `.tracebook-stage-*` paths.
+
+Staged payloads live in owner-only, read-only directories and publication uses
+the open directory descriptor, so replacing a staging directory's parent entry
+cannot substitute another payload. Published files are owner-readable. This
+protects against ordinary concurrent file writers, not privileged processes or
+same-user code deliberately changing permissions or writing through an already
+open descriptor. Use a trusted output directory; these filesystem guards are
+not a sandbox for hostile code running as your user.
+
+## Generated Campaigns And Reduction
+
+The v1 generator uses a specified SplitMix64 stream rather than Python's
+runtime PRNG. Each trace draws its prefix from an 18-event semantic scaffold;
+at 18 events or longer it reaches all 11 declared capabilities, then continues
+with state-aware generated adds, updates, deletes, and probes. Candidate
+behavior never feeds back into event generation.
+
+Run a deterministic campaign:
+
+```bash
+python -m tracebook.book_replay campaign \
+ --output /tmp/book-replay-campaign.json \
+ --reduced-events-output /tmp/book-replay-reduced.jsonl \
+ --seed 20260831 \
+ --traces 25 \
+ --events-per-trace 100 \
+ --candidate-cmd './book-adapter --tracebook-book-replay-stdio'
+```
+
+The runner stops at the first divergence and automatically delta-debugs its
+compared prefix. Reduction preserves the first-divergence category, rejects a
+new protocol failure while reducing a semantic failure, records its candidate
+run budget, and reports whether the result is one-minimal. Campaign artifacts
+include the generator version, campaign ID, per-trace seeds and hashes,
+capability coverage, original evidence, minimized evidence, and a stable
+failure ID.
+
+An already captured divergent trace can be minimized directly:
+
+```bash
+python -m tracebook.book_replay minimize trace.jsonl \
+ --events-output /tmp/reduced.jsonl \
+ --output /tmp/minimization.json \
+ --candidate-cmd './book-adapter --tracebook-book-replay-stdio'
+```
+
+`campaign` artifacts use `artifact_type = "tracebook.book-replay.campaign"`;
+standalone reductions use
+`artifact_type = "tracebook.book-replay.minimization"`.
+
+Python adapters can use the server helper:
+
+```python
+from tracebook.book_replay import serve_book_replay_stdio
+
+raise SystemExit(serve_book_replay_stdio(MyBookAdapter))
+```
+
+The complete reference-backed example is
+[`examples/book_replay_adapter.py`](../examples/book_replay_adapter.py).
+
+## NautilusTrader Boundary Proof
+
+The maintained optional adapter targets NautilusTrader's native Rust-backed
+`L3_MBO` book and direct `simulate_fills(BookOrder)` Python binding. It passes
+all 17 bundled events and a pinned 2,500-event generated campaign at the
+`2.0.0rc3` release candidate. See the
+[`integrations/nautilus_trader`](../integrations/nautilus_trader) proof,
+provenance, exact command, and LGPL distribution warning.
+
+NautilusTrader's broader backtest matching engine remains out of scope here.
+That engine fills candidate orders against book state supplied by market data;
+it does not turn this delta profile into a submitted-order CLOB.
diff --git a/docs/commands.md b/docs/commands.md
index 86a11ea..fd20c4f 100644
--- a/docs/commands.md
+++ b/docs/commands.md
@@ -133,6 +133,9 @@ fixed semantic surface.
Prepare and verify a release-grade two-run evidence pair:
+These two evidence commands are unreleased. Use a pinned source build, not the
+published 0.6.0 wheel, and record its revision and wheel hash.
+
```bash
tracebook-conformance evidence-init /path/to/candidate \
--workspace .tracebook/release-evidence \
@@ -254,7 +257,10 @@ Options:
## Dashboard CLI
+Install the optional dashboard dependencies before starting the interactive UI:
+
```bash
+python -m pip install "tracebook-sim[dashboard]==0.6.0"
tracebook-dashboard --port 8050 --demo-simulation --demo-throughput 200 --seed 1337
```
diff --git a/docs/conformance.md b/docs/conformance.md
index 6e42336..bfae977 100644
--- a/docs/conformance.md
+++ b/docs/conformance.md
@@ -137,6 +137,11 @@ silently.
## Captured Two-Run Evidence
+This workflow is unreleased. Use a pinned source checkout or a wheel built from
+that checkout, recording its Git revision and SHA-256; the published 0.6.0 wheel
+does not contain these commands. The source version string alone cannot
+distinguish this build from the released package.
+
Use the evidence workflow when a qualification result will support a release,
benchmark, or external claim. It pins the candidate before adapter readiness and
requires the same canonical qualification from two prepared roots.
diff --git a/docs/independent-onboarding.md b/docs/independent-onboarding.md
new file mode 100644
index 0000000..38f5497
--- /dev/null
+++ b/docs/independent-onboarding.md
@@ -0,0 +1,151 @@
+# Independent Onboarding: Captured Go Evidence
+
+## Status And Scope
+
+This packet is prepared for one independent maintainer attempt against
+`intrepidkarthi/orderbook` v0.26.0, source commit
+`51d480cdb68b9989febb0b075d291cf891f425b3`. A Tracebook-owned rehearsal is not
+independent onboarding, adoption, or an upstream decision.
+
+The request is deliberately narrow: can the maintainer reproduce and understand
+one verified FIFO-limit evidence pair, and would they retain the result or its
+CI check? The native unfillable-FOK rejection is a documented contract
+difference, not a bug accusation or a request to change native matching.
+
+The captured evidence workflow and this adapter are unreleased. The published
+0.6.0 wheel does not contain the new evidence commands. Use a wheel built from
+the exact Tracebook checkout below, not `pip install tracebook-conformance==0.6.0`.
+The built wheel still carries version 0.6.0: record its SHA-256 and source commit
+so it cannot be confused with that release. Do not upload it to PyPI.
+
+## Reproduce From A Reviewed Checkout
+
+Prerequisites: Git, Go >=1.23.5, Python >=3.10, and a POSIX shell. Start at the
+root of a clean, reviewed Tracebook checkout containing this packet. Review the
+adapter before running it; candidate execution is not sandboxed. These commands
+download only public source and Go modules, and never use experiment material.
+
+Create and inspect the conformance wheel following the repository's build and
+privacy gates. Then set `WHEEL` to that artifact's absolute path. Run the
+following with Bash; all candidate, cache, build, and qualification outputs stay
+under a fresh directory, which is preserved on failure.
+
+```bash
+set -euo pipefail
+TRACEBOOK_ROOT=$PWD
+TRACEBOOK_REV=$(git rev-parse HEAD)
+test -z "$(git status --porcelain --untracked-files=no)"
+WHEEL="$TRACEBOOK_ROOT/dist/conformance/tracebook_conformance-0.6.0-py3-none-any.whl"
+test -f "$WHEEL"
+shasum -a 256 "$WHEEL"
+
+ATTEMPT_ROOT=$(mktemp -d "${TMPDIR:-/tmp}/tracebook-onboarding.XXXXXX")
+export ATTEMPT_ROOT
+python3 -m venv "$ATTEMPT_ROOT/tool"
+PYTHON="$ATTEMPT_ROOT/tool/bin/python"
+"$PYTHON" -m pip install "$WHEEL"
+"$PYTHON" -m tracebook.conformance.cli --help
+
+CANDIDATE_REV=51d480cdb68b9989febb0b075d291cf891f425b3
+git clone --depth 1 --branch v0.26.0 \
+ https://github.com/intrepidkarthi/orderbook.git "$ATTEMPT_ROOT/source"
+test "$(git -C "$ATTEMPT_ROOT/source" rev-parse HEAD)" = "$CANDIDATE_REV"
+
+WORKSPACE="$ATTEMPT_ROOT/evidence"
+"$PYTHON" -m tracebook.conformance.cli evidence-init "$ATTEMPT_ROOT/source" \
+ --workspace "$WORKSPACE" \
+ --candidate-name 'intrepidkarthi/orderbook CLOB' \
+ --candidate-revision "$CANDIDATE_REV"
+SNAPSHOT=$("$PYTHON" -c \
+ 'import json,sys; print(json.load(open(sys.argv[1]))["candidate"]["snapshot_id"])' \
+ "$WORKSPACE/evidence-plan.json")
+
+for RUN_NUMBER in 1 2; do
+ RUN_ROOT="$WORKSPACE/runs/run-$RUN_NUMBER"
+ git -C "$TRACEBOOK_ROOT" archive "$TRACEBOOK_REV" integrations/intrepid_orderbook \
+ | tar -x --strip-components=2 -C "$RUN_ROOT/adapter"
+ (
+ cd "$RUN_ROOT/adapter"
+ export GOWORK=off GOCACHE="$RUN_ROOT/cache/build"
+ export GOMODCACHE="$RUN_ROOT/cache/modules" GOPATH="$RUN_ROOT/cache/gopath"
+ go mod edit "-replace=github.com/intrepidkarthi/orderbook=$RUN_ROOT/candidate"
+ go mod download
+ go build -mod=readonly -trimpath \
+ -ldflags "-X main.engineRevision=$CANDIDATE_REV -X main.engineSnapshot=$SNAPSHOT" \
+ -o "$RUN_ROOT/build/adapter" .
+ go version -m "$RUN_ROOT/build/adapter"
+ )
+ "$PYTHON" -m tracebook.conformance.cli qualify \
+ --profile fifo-limit-v1 --suite-version v2 \
+ --seed 42 --traces 25 --events-per-trace 200 --max-minimize-runs 100 \
+ --timeout 10 \
+ --candidate-name 'intrepidkarthi/orderbook CLOB' \
+ --candidate-revision "$CANDIDATE_REV" --candidate-snapshot "$SNAPSHOT" \
+ --output-dir "$RUN_ROOT/qualification" \
+ --candidate "$RUN_ROOT/build/adapter"
+done
+
+"$PYTHON" -m tracebook.conformance.cli evidence-verify "$WORKSPACE/evidence-plan.json"
+```
+
+Expected result: each run passes 3/3 fixed cases, 25/25 generated traces, 5,000
+generated events, and 10/10 capabilities. Verification emits
+`evidence-manifest.json` only when both candidate trees remain unchanged and
+their canonical bundles are byte-identical. Identity-bound qualification IDs
+differ from the integration README's unbound runs; do not substitute one for
+the other.
+
+The source replacement above is essential: embedding a snapshot ID in an
+otherwise ordinary module-cache build does not prove that captured source was
+used. Each run gets its own module and compiler caches. This remains an
+auditable process, not a hermetic build or proof against a dishonest adapter;
+see the [evidence trust boundary](conformance.md#captured-two-run-evidence).
+
+## Measurement Record
+
+Before the independent attempt, record its operator, relationship to the engine,
+start time, Tracebook commit and wheel hash, candidate commit and snapshot,
+operating system, Python and Go versions. Start the clock before environment
+setup; do not exclude dependency downloads or time spent asking questions.
+
+Record each failed command and each human correction, including who suggested
+it, what changed, and minutes spent. Preserve failed outputs in separate roots;
+do not edit a failed bundle into a passing one. Record elapsed time to the first
+verified pair or the first blocker, even if no pair succeeds. A provided adapter
+measures time to evidence, not time to author an adapter.
+
+The result record must contain:
+
+- operator and relationship: **not yet observed**;
+- first verified pair or blocker time: **not yet observed**;
+- failed attempts and human corrections: **not yet observed**;
+- plan, manifest, qualification and campaign IDs: **not yet observed**;
+- semantic questions or disagreements: **not yet observed**; and
+- retention decision (retained, declined, undecided) plus a public link if
+ retained: **not yet observed**.
+
+An external maintainer running the packet establishes independent use, not
+retention. A promise to add CI is not retained CI. A Tracebook-owned run or job
+is neither. Report the attempt through the existing Engine qualification issue
+form, adding the wheel hash, evidence manifest and human-correction log. Remove
+credentials, proprietary traces and machine-specific paths before publication.
+
+## Maintainer Invitation
+
+Sent on 2026-09-05 (UTC) in
+[intrepidkarthi/orderbook issue #25](https://github.com/intrepidkarthi/orderbook/issues/25),
+with the packet pinned to verified Tracebook commit
+`55e3c4b010c8cc1bc967db59f94f593ffa0f6838` and the local rehearsal's artifact
+hashes. Sending the invitation is not independent use or a retention decision;
+the measurement fields above remain unobserved.
+
+> We have a pinned, out-of-tree Tracebook adapter for your v0.26.0 CLOB. FIFO
+> limit qualification passes; the broader FOK profile stops on your native
+> unfillable-FOK rejection, which we preserve as a contract difference, not a
+> defect. Would you be willing to try one captured two-run FIFO evidence packet
+> and tell us whether its result is useful enough to retain? No matching-engine
+> change or in-tree adapter commitment is requested. We would record setup
+> time, corrections, and your retention decision, including a failed attempt.
+
+A maintainer response and any independent result must be recorded as new
+evidence; the sent invitation establishes neither.
diff --git a/docs/positioning.md b/docs/positioning.md
index c19c8c8..4de461b 100644
--- a/docs/positioning.md
+++ b/docs/positioning.md
@@ -16,8 +16,12 @@ Strong open-source systems already own broader categories:
focuses on replaying historical Level 3 orders and latency-aware HFT strategy
simulation.
- [NautilusTrader](https://nautilustrader.io/docs/latest/concepts/order_book/)
- provides a Rust L1-L3 order book inside a much larger backtesting and live
- trading platform.
+ provides a native Rust L1-L3 market-data book inside a much larger backtesting
+ and live trading platform. Its backtest matching engine simulates candidate
+ fills against that exogenous book; it is not a submitted-order CLOB matching
+ candidate for Tracebook's `fifo-*` profiles. The separate
+ [`l3-book-replay-v1`](book-replay-conformance.md) surface checks the book
+ mirror and non-mutating fill walk without making that category claim.
Building a smaller general-purpose simulator beside those projects would leave
Tracebook between categories: too slow and incomplete for production exchange
diff --git a/docs/pr-84-review-disposition.md b/docs/pr-84-review-disposition.md
new file mode 100644
index 0000000..dd74df3
--- /dev/null
+++ b/docs/pr-84-review-disposition.md
@@ -0,0 +1,160 @@
+# PR #84 Review Disposition
+
+This records the 30 comments in the initial automated review of
+[PR #84](https://github.com/Taz33m/tracebook/pull/84), against `0c98e18`,
+the four follow-up comments against `e65416a`, and the cleanup follow-up
+against `f035e0c`.
+It is a disposition of that review, not release approval or a new experiment
+result. Source and regression tests, not the review's severity labels, determine
+the implemented changes.
+
+## Release-code fixes
+
+The following 17 comments are addressed in the release implementation. Comment
+IDs identify the original GitHub review comments. Existing qualified-profile
+identities and retained evidence are not rewritten to accommodate a change.
+
+| Comment ID | Change | Regression / verification surface |
+| --- | --- | --- |
+| 3939088988 | Close the L3 adapter before emitting `complete`; a failed close is one terminal adapter error, with no retry. | `test_server_close_failure_is_one_terminal_adapter_error` |
+| 3939088991 | Both Rust adapters fail before emitting a positive fill/remainder normalized to zero. Inputs remain exact native lots. | Shared quantity tests and both adapters' `positive_fills_and_remainders_cannot_be_emitted_as_zero` |
+| 3939089005 | Share the faulty FIFO-reordering fixture between in-process and external tests; retain each existing engine identity. | Existing campaign, reduction, and failure-ID regressions |
+| 3939089008 | Validate typed L3 outcomes, fills, snapshot entries, and book collections during construction. | `test_invalid_in_process_observation_is_classified_as_protocol_failure` |
+| 3939089018 | Correct the orderbook-rs translation contract: 12 native decimal places, independent output precision. | Quantity conversion regressions and README review |
+| 3939089021 | Exact FIFO probe subtraction independent of ambient Decimal precision; remove context-rounding from shared canonical formatting too. | `test_probe_subtraction_and_canonical_values_are_exact_at_any_context_precision` |
+| 3939089023 | Separate client decoding/validation failures from exceptions raised inside adapters. | Factory/apply/snapshot exception tests and malformed-payload test |
+| 3939089028 | Require actual non-negative integer snapshot/finish counters; reject booleans, missing values, null, floats, and strings. | Parameterized session-counter tests |
+| 3939089029 | Require Go snapshot/finish counters and an object event payload, without zero-value defaults accepting missing/null fields. | `TestServerRejectsMissingNullAndMistypedRequiredFields` |
+| 3939089032 | Normalize timeout conversion errors, including oversized integers, into the public L3 error contract. | `test_external_timeout_overflow_is_a_public_validation_error` |
+| 3939089038 | Wrap invalid ready-frame engine metadata as a book-replay protocol error and reap the child. | Invalid-metadata/child-reaping regression |
+| 3939089052 | Stage in a private read-only directory and link relative to its open descriptor. Protect the payload entry from ordinary competing writers and anchor publication across directory replacement. | Stage-entry and stage-directory replacement tests, plus existing rollback/competing-destination tests |
+| 3939089055 | Count price relocation only when the price actually changes, not on side-only updates. | `test_side_only_update_does_not_claim_price_relocation_coverage` |
+| 3939089057 | Apply the documented conservative same-side aggregate envelope to IOC/FOK as well as GTC limit orders; retain the market exemption. | `TestSideAggregateEnvelopeIncludesIOCAndFOKButNotMarket` |
+| 3939089063 | Report symlink-loop path resolution as a controlled CLI error before starting a candidate. | Symlink-loop CLI regression |
+| 3939089067 | Handle corrupt deflated source inventories as a failed distribution privacy check. | Corrupt real ZIP-block regression |
+| 3939089071 | Bound the workflow path reader to `on.pull_request.paths`, allowing comments, blank lines, different indentation, and other trigger keys. | Positive layout variants and negative job/other-trigger regressions |
+
+Two limits matter when interpreting these fixes:
+
+- Insufficient output precision is an operational error, not a semantic
+ rejection or a successful qualification. Native matching inputs are not
+ rounded, inflated, retried, or silently removed. The Python CLOB reference
+ also rejects zero normalized positive quantities; its positive-value wire
+ contract has not been relaxed.
+- Artifact publication is not a security sandbox against root or same-user
+ code deliberately changing permissions or writing through existing file
+ descriptors. Use trusted output directories. Normal concurrent writers must
+ not overwrite destinations, and rollback preserves competing files. The
+ directory-permission test is intentionally skipped when running as root.
+
+## Follow-up release-code fixes
+
+All four follow-up findings are addressed without changing positive-value
+normalization, matching semantics, or frozen research files.
+
+| Comment ID | Change | Regression / verification surface |
+| --- | --- | --- |
+| 3939212113 | Retain the staging directory's identity before opening it, remove only a matching entry on setup failure, and reject an opened replacement before writing or changing its permissions. | `test_stage_directory_setup_failure_cleans_only_owned_entries`: failed open, competing replacement, and descriptor closure |
+| 3939212117 | Reserve `PROTOCOL_ERROR` for explicit client validation failures; adapter `BrokenPipeError` follows the adapter-error path, including shutdown. | Close/factory/apply/snapshot regressions plus a broken output pipe; close remains single-shot |
+| 3939212124 | Diagnose an exact zero as a nonpositive quantity; retain the separate precision-loss error for positive quantities that round to zero. | Shared Rust zero-diagnostic test at 0, 12, and 18 places, plus existing positive rounding tests |
+| 3939212131 | Use the same narrow path-resolution guard for CLI inputs and distinct-path checks. | Actual input symlink loops for both `run` and `minimize`, including Python 3.10's `RuntimeError` behavior |
+
+The duplicate-fixture comment, `3939089005`, was already addressed
+by `e65416a`: `tests/test_book_replay_campaign.py` imports the shared
+`FaultyBookReplayAdapter`. Its small `_ReorderingAdapter` subclass only supplies
+the historical engine name; it contains no fault, snapshot, or shutdown logic.
+Existing campaign and reduced-failure identities remain covered by regression
+tests. Its stale release-code conversation was resolved with source and test
+evidence; it is not a deferred research issue.
+
+## Initial staging identity failure — agreed conservative handling
+
+Comment [3939788196](https://github.com/Taz33m/tracebook/pull/84#discussion_r3939788196)
+correctly identifies a possible empty-directory residue when the first `stat`
+after `mkdir` fails. With neither an identity nor a directory descriptor, the
+runner cannot safely distinguish that entry from a competing replacement.
+
+The maintainer explicitly approved retaining an unverified entry, reporting
+an explicit error, and adding tests/documentation before merge. The runner now
+raises `BookReplayError` with the original possible-residue path and the reason
+automatic directory cleanup was skipped. CLI commands exit `2` before candidate
+execution, retain the underlying exception as the cause, and still attempt
+owned-lock cleanup and close open descriptors. Known-ownership cleanup remains
+unchanged. This addresses silent residue through an agreed, observable failure
+contract; it does not claim automatic removal when ownership is unknown.
+
+Regression tests cover the original empty directory, empty and populated
+competing directories, a replacement symlink, a moved-away entry, descriptor
+closure, and all four CLI commands. Recovery instructions in
+[book-replay conformance](book-replay-conformance.md) require independently
+verified ownership and prohibit blanket or recursive removal of uncertain paths.
+
+## Frozen research findings — explicitly deferred, not fixed
+
+These 13 findings are in the historical/hash-bound research harness. The
+accepted scope leaves that code, frozen inputs, evaluator material, and retained
+results unchanged. On 2026-09-05 the maintainer explicitly authorized transferring
+all 13 findings to [follow-up issue #85](https://github.com/Taz33m/tracebook/issues/85)
+so this release-code PR can merge after its remaining fix and checks. That issue
+preserves every original review link and an unchecked item for each finding.
+The PR conversations are resolved only as an agreed transfer: these findings
+remain open in the follow-up, not fixed or dismissed as false positives. The
+release fixes above do not resolve the underlying research risks.
+
+| Comment ID | Reported issue | Required next decision/check |
+| --- | --- | --- |
+| 3939088982 | V2 Claude auth-route probe inherits ambient environment. | Audit route-binding evidence locally; use a sanitized, explicit environment in a newly versioned runner. |
+| 3939088984 | V2 finalization failure before verdict can quarantine a completed provider turn. | Audit lifecycle receipts without rerunning providers; a future runner needs a durable provider-completion barrier before finalization. |
+| 3939088987 | V2 frozen snapshots permit symbolic links. | Assess snapshot inventories and provider read boundaries locally; preregister an explicit link policy for future runs. |
+| 3939088994 | Baseline live copy is not checked against its frozen source before launch. | Audit existing snapshot bindings; bind and verify copied workspaces in a future runner. |
+| 3939089000 | Baseline command success can outlive a failed/timed-out provider. | Audit lifecycle exit receipts; separate transport completion and semantic outcomes in a new runner. |
+| 3939089002 | Treatment technical verdict handling exempts some unsuccessful provider exits/timeouts. | Audit treatment lifecycle verdicts separately from V2, which has its own completion checks. |
+| 3939089010 | Missing/unreadable V2 bound file can escape as `OSError`. | Add controlled validation errors in a future version, preserving frozen hash bindings. |
+| 3939089043 | Baseline missing reported model falls back to `opus`. | Treat absent model evidence as missing, not observed identity, in a separately versioned audit/fix. |
+| 3939089045 | Baseline permits an execution-time timeout override. | Compare receipts to the preregistered budget before deciding whether existing runs are affected. |
+| 3939089048 | Baseline candidate/gold paths lack private-root confinement. | Audit declared paths locally; enforce resolved boundaries in a new runner. |
+| 3939089050 | Baseline case IDs admit path components. | Audit manifest IDs; enforce a safe identifier grammar in a new runner. |
+| 3939089056 | Treatment validation omits some workspace evidence artifacts. | Audit artifact completeness locally and specify required evidence in a new validator version. |
+| 3939089060 | Baseline quarantine checks a verdict sentinel that baseline does not write. | Distinguish completed turns from incomplete interruptions using lifecycle receipts, not a missing sentinel. |
+
+These are source-level risks, not proof that a particular completed cohort was
+contaminated. Do not pool different harness generations, modify an old frozen
+runner in place, reinterpret missing metadata as observed evidence, or rerun a
+completed provider turn. An approved audit should be local and preserve every
+interruption artifact; any necessary future collection needs a new versioned
+protocol and explicit authorization. No private gold or evaluator artifacts
+belong in a provider request or this public document.
+
+## Release gate
+
+Local verification of the initial fixes passed: Python 3.13 had 651 passing tests,
+one skip, and 82.38% coverage; the focused Python 3.10 regressions passed too.
+Formatting, lint, type checking, static security checks, pinned Rust 1.88/Go
+1.23.5 checks, distribution privacy, and sdist/wheel agreement passed.
+Fresh 5,000-event qualifications retained the exact orderbook-rs, gocronx, Go
+FIFO-limit, and Go partial-fill identities. The Go full-profile FOK failure and
+the Nautilus 2,500-event positive campaign/three-event negative control retained
+their expected identities and reduced trace. These are local checks; consult
+the PR for the latest remote CI and review state.
+
+Follow-up verification passed separately: Python 3.13 had 661 passing tests,
+one skip, and 82.38% coverage. The focused L3 protocol, artifact, and campaign
+suite passed all 114 tests on Python 3.10. All 33 Rust tests across the shared
+protocol and both adapters passed with Rust 1.88, along with clippy and rustfmt.
+Python formatting, lint, type checking, static security checks, and compilation
+also passed. This follow-up did not recollect provider runs or rewrite retained
+qualification artifacts; the initial verification above remains historical.
+
+The final initial-identity error handling passed 670 Python 3.13 tests with one
+skip and 82.39% coverage, plus all 123 focused L3 tests on Python 3.10. Its nine
+new regression cases failed before the change and passed afterward. Formatting,
+lint, type checking, static security checks, compilation, and whitespace checks
+passed. No frozen research files or retained qualification artifacts changed.
+
+The agreed transfer to issue #85 supersedes the earlier instruction to keep
+these 13 PR conversations open pending a disposition. Keep the follow-up issue
+open until its evidence-backed completion criteria are reviewed. Passing release
+tests does not dismiss those risks or justify bypassing repository protections.
+The remaining release-code review and required checks must pass before the
+authorized merge; no tag or package publication is authorized. The previous
+maintainer invitation and its immutable source/wheel references remain unchanged.
diff --git a/docs/release.md b/docs/release.md
index 66f5865..416fe05 100644
--- a/docs/release.md
+++ b/docs/release.md
@@ -21,7 +21,7 @@ python -m pip install -e ".[dev]"
python -m pip install -e "./packaging/tracebook-sim[dashboard]"
python -m black --check src tests examples integrations experiments tools
python -m flake8 src tests examples integrations experiments tools
-python -m mypy src/tracebook experiments tools
+python -m mypy --python-version "$(python -c 'import sys; print("%d.%d" % sys.version_info[:2])')" src/tracebook experiments tools
python -m bandit -q -r src integrations tools
python -m compileall -q src tests examples integrations experiments tools
python -m pytest --cov=tracebook --cov-report=term-missing --cov-fail-under=75
@@ -33,6 +33,7 @@ tracebook-coinbase examples/data/coinbase_btcusd_l3_snapshot.json examples/data/
tracebook-corpus verify src/tracebook/corpus/fixtures/coinbase-btcusd-synthetic-v1
python -m build --sdist --wheel --outdir dist/conformance .
python -m build --sdist --wheel --outdir dist/simulator packaging/tracebook-sim
+python tools/verify_distribution_privacy.py dist/conformance/* dist/simulator/*
python -m twine check dist/conformance/* dist/simulator/*
python -m pip download \
--no-deps \
@@ -64,11 +65,16 @@ python -m pip check
## Remote Verification
+- Review the [PR #84 disposition](pr-84-review-disposition.md): release-code
+ fixes do not close the separately frozen research findings or authorize a release.
- Push to a branch and wait for Ubuntu GitHub Actions on Python 3.10 through
3.13.
- Confirm the native `orderbook-rs` integration passes its fixed trace, `7/9`
suite profile, generated campaign, and intentional-drift negative control.
- Confirm the pinned PythonMatchingEngine integration workflow passes.
+- Confirm the pinned Go CLOB's FIFO qualifications and retained FOK divergence,
+ and the separate Nautilus L3 campaign and injected queue-priority control.
+ These are distinct contracts, not interchangeable qualification claims.
- Check both generated package metadata sets and their disjoint wheel RECORDs.
- Check that Dependabot has no urgent security update waiting.
- Confirm no README badge or proof table overstates the current CI/test state.
diff --git a/examples/book_replay_adapter.py b/examples/book_replay_adapter.py
new file mode 100644
index 0000000..8b288a6
--- /dev/null
+++ b/examples/book_replay_adapter.py
@@ -0,0 +1,20 @@
+#!/usr/bin/env python3
+"""Minimal Python adapter for the isolated L3 book-replay protocol."""
+
+from tracebook.book_replay import (
+ EngineMetadata,
+ ReferenceBookReplayAdapter,
+ serve_book_replay_stdio,
+)
+
+
+class ExampleBookReplayAdapter(ReferenceBookReplayAdapter):
+ """Replace this reference-backed body with calls into a candidate L3 book."""
+
+ def __init__(self, config):
+ super().__init__(config)
+ self.metadata = EngineMetadata("example-book-replay-adapter", "1", "Python")
+
+
+if __name__ == "__main__":
+ raise SystemExit(serve_book_replay_stdio(ExampleBookReplayAdapter))
diff --git a/experiments/agent_qualification.py b/experiments/agent_qualification.py
new file mode 100644
index 0000000..b249cdb
--- /dev/null
+++ b/experiments/agent_qualification.py
@@ -0,0 +1,1314 @@
+"""Reproducible runner for the agentic matching-engine qualification evaluation.
+
+The tracked protocol and prompt are public. Candidate snapshots, evaluator gold
+manifests, raw transcripts, and workspaces live under ``experiments/private/``
+and are intentionally ignored.
+"""
+
+from __future__ import annotations
+
+import argparse
+import datetime as dt
+import hashlib
+import json
+import os
+import random
+import shutil
+import signal
+import stat
+import subprocess
+import sys
+import time
+from pathlib import Path
+from typing import Any, Mapping, Sequence
+
+REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
+RUNNER_PATH = Path(__file__).resolve()
+PROTOCOL_PATH = REPOSITORY_ROOT / "docs" / "agent-qualification-evaluation.md"
+PROMPT_PATH = REPOSITORY_ROOT / "experiments" / "agent_qualification_prompt.txt"
+PRIVATE_ROOT = REPOSITORY_ROOT / "experiments" / "private" / "agent-qualification-v1"
+DEFAULT_MANIFEST_PATH = PRIVATE_ROOT / "cases.json"
+DEFAULT_PLAN_PATH = PRIVATE_ROOT / "plan.json"
+DEFAULT_RUNS_PATH = PRIVATE_ROOT / "runs"
+CODEX_WORK_ROOT = PRIVATE_ROOT / "work" / "codex"
+CLAUDE_WORK_ROOT = Path("/private/tmp/tracebook-agent-eval")
+PROTOCOL_ID = "agent-qualification-v1"
+RANDOMIZATION_SEED = 20260728
+DEFAULT_REPETITIONS = 3
+DEFAULT_TIMEOUT_SECONDS = 120 * 60
+
+CODEX_BINARY = Path("/Applications/ChatGPT.app/Contents/Resources/codex")
+CODEX_AUTH_PATH = Path.home() / ".codex" / "auth.json"
+CLAUDE_BINARY = Path("/Users/tazeemmahashin/.local/bin/claude")
+DOTNET_BINARY_CANDIDATES = (
+ Path("/usr/local/opt/dotnet@8/bin/dotnet"),
+ Path("/usr/local/bin/dotnet"),
+)
+
+DOCS_TREATMENT = (
+ "\n\nYou may evaluate tracebook-conformance==0.6.0 using only its public "
+ "README and conformance documentation. Use it only if it fits the "
+ "declared contract.\n"
+)
+
+NETWORK_DOMAINS = (
+ "pypi.org",
+ "files.pythonhosted.org",
+ "registry.npmjs.org",
+ "crates.io",
+ "static.crates.io",
+ "index.crates.io",
+ "proxy.golang.org",
+ "sum.golang.org",
+ "github.com",
+ "api.github.com",
+ "codeload.github.com",
+ "raw.githubusercontent.com",
+ "*.githubusercontent.com",
+)
+
+SECRET_ENVIRONMENT_NAMES = {
+ "ANTHROPIC_API_KEY",
+ "ANTHROPIC_AUTH_TOKEN",
+ "OPENAI_API_KEY",
+ "CODEX_API_KEY",
+ "CODEX_ACCESS_TOKEN",
+ "GH_TOKEN",
+ "GITHUB_TOKEN",
+ "AWS_ACCESS_KEY_ID",
+ "AWS_SECRET_ACCESS_KEY",
+ "AWS_SESSION_TOKEN",
+ "GOOGLE_APPLICATION_CREDENTIALS",
+ "DATABASE_URL",
+ "SSH_AUTH_SOCK",
+}
+
+
+class EvaluationError(RuntimeError):
+ """Raised when a frozen evaluation precondition is not satisfied."""
+
+
+def _utc_now() -> str:
+ return dt.datetime.now(dt.timezone.utc).isoformat().replace("+00:00", "Z")
+
+
+def _sha256_bytes(payload: bytes) -> str:
+ return hashlib.sha256(payload).hexdigest()
+
+
+def _sha256_file(path: Path) -> str:
+ digest = hashlib.sha256()
+ with path.open("rb") as handle:
+ for chunk in iter(lambda: handle.read(1024 * 1024), b""):
+ digest.update(chunk)
+ return digest.hexdigest()
+
+
+def _snapshot_digest(root: Path) -> str:
+ """Hash relative names, modes, and bytes without following symlinks."""
+
+ digest = hashlib.sha256()
+ for path in sorted(root.rglob("*"), key=lambda value: value.as_posix()):
+ relative = path.relative_to(root).as_posix()
+ if ".git" in path.relative_to(root).parts:
+ continue
+ if path.is_symlink():
+ digest.update(b"L\0")
+ digest.update(relative.encode())
+ digest.update(b"\0")
+ digest.update(os.readlink(path).encode())
+ digest.update(b"\0")
+ elif path.is_file():
+ digest.update(b"F\0")
+ digest.update(relative.encode())
+ digest.update(b"\0")
+ digest.update(f"{path.stat().st_mode & 0o777:o}".encode())
+ digest.update(b"\0")
+ with path.open("rb") as handle:
+ for chunk in iter(lambda: handle.read(1024 * 1024), b""):
+ digest.update(chunk)
+ digest.update(b"\0")
+ return digest.hexdigest()
+
+
+def _tree_inventory(root: Path) -> list[dict[str, Any]]:
+ """Return a stable inventory while rejecting unsafe tree entries."""
+
+ inventory: list[dict[str, Any]] = []
+ for path in sorted(root.rglob("*"), key=lambda value: value.as_posix()):
+ relative_path = path.relative_to(root)
+ if ".git" in relative_path.parts:
+ continue
+ relative = relative_path.as_posix()
+ if path.is_symlink():
+ raise EvaluationError(f"frozen fixture must not contain symlink {relative!r}")
+ mode = path.stat().st_mode
+ if stat.S_ISDIR(mode):
+ continue
+ if not stat.S_ISREG(mode):
+ raise EvaluationError(f"frozen fixture must contain only regular files: {relative!r}")
+ inventory.append(
+ {
+ "path": relative,
+ "bytes": path.stat().st_size,
+ "sha256": _sha256_file(path),
+ }
+ )
+ return inventory
+
+
+def _private_path(raw_path: str, description: str) -> Path:
+ path = (REPOSITORY_ROOT / raw_path).resolve()
+ try:
+ path.relative_to(PRIVATE_ROOT.resolve())
+ except ValueError as exc:
+ raise EvaluationError(f"{description} must stay inside {PRIVATE_ROOT}") from exc
+ return path
+
+
+def _dotnet_layout() -> tuple[Path | None, Path | None, Path | None]:
+ for configured_binary in DOTNET_BINARY_CANDIDATES:
+ if not configured_binary.exists():
+ continue
+ launcher_directory = configured_binary.resolve().parent
+ libexec = launcher_directory.parent / "libexec"
+ root = libexec if libexec.is_dir() else launcher_directory
+ return configured_binary, launcher_directory, root
+ return None, None, None
+
+
+def _load_json(path: Path) -> dict[str, Any]:
+ try:
+ payload = json.loads(path.read_text())
+ except (OSError, json.JSONDecodeError) as exc:
+ raise EvaluationError(f"cannot read JSON from {path}: {exc}") from exc
+ if not isinstance(payload, dict):
+ raise EvaluationError(f"{path} must contain one JSON object")
+ return payload
+
+
+def _write_json(path: Path, payload: Mapping[str, Any]) -> None:
+ path.parent.mkdir(parents=True, exist_ok=True)
+ path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n")
+
+
+def _case_index(manifest: Mapping[str, Any]) -> dict[str, dict[str, Any]]:
+ cases = manifest.get("cases")
+ if not isinstance(cases, list) or not cases:
+ raise EvaluationError("case manifest must contain a non-empty cases list")
+ result: dict[str, dict[str, Any]] = {}
+ for raw_case in cases:
+ if not isinstance(raw_case, dict):
+ raise EvaluationError("every case must be an object")
+ case_id = raw_case.get("id")
+ if not isinstance(case_id, str) or not case_id:
+ raise EvaluationError("every case must have a non-empty id")
+ if case_id in result:
+ raise EvaluationError(f"duplicate case id {case_id!r}")
+ result[case_id] = raw_case
+ return result
+
+
+def _validate_frozen_tree(
+ case: Mapping[str, Any],
+ declaration: Any,
+ *,
+ field_name: str,
+ expected_kind: str,
+) -> dict[str, Any]:
+ if not isinstance(declaration, dict):
+ raise EvaluationError(f"case {case['id']!r} must declare {field_name!r} as an object")
+ kind = declaration.get("kind")
+ if kind == "none":
+ if set(declaration) != {"kind"}:
+ raise EvaluationError(
+ f"case {case['id']!r} {field_name!r} kind 'none' " "must not contain fixture fields"
+ )
+ return {"kind": "none"}
+ if kind != expected_kind:
+ raise EvaluationError(
+ f"case {case['id']!r} {field_name!r} kind must be " f"{expected_kind!r} or 'none'"
+ )
+
+ required = (
+ "source",
+ "manifest",
+ "manifest_sha256",
+ "tree_sha256",
+ )
+ missing = [name for name in required if not isinstance(declaration.get(name), str)]
+ if missing:
+ raise EvaluationError(
+ f"case {case['id']!r} {field_name!r} is missing string fields: " + ", ".join(missing)
+ )
+ source = _private_path(str(declaration["source"]), f"case {case['id']!r} {field_name!r} source")
+ fixture_manifest_path = _private_path(
+ str(declaration["manifest"]),
+ f"case {case['id']!r} {field_name!r} manifest",
+ )
+ if not source.is_dir():
+ raise EvaluationError(f"case {case['id']!r} {field_name!r} source is unavailable: {source}")
+ if not fixture_manifest_path.is_file():
+ raise EvaluationError(
+ f"case {case['id']!r} {field_name!r} manifest is unavailable: "
+ f"{fixture_manifest_path}"
+ )
+ actual_manifest_hash = _sha256_file(fixture_manifest_path)
+ if declaration["manifest_sha256"] != actual_manifest_hash:
+ raise EvaluationError(
+ f"case {case['id']!r} {field_name!r} manifest changed: "
+ f"expected {declaration['manifest_sha256']}, got {actual_manifest_hash}"
+ )
+
+ fixture_manifest = _load_json(fixture_manifest_path)
+ inventory = _tree_inventory(source)
+ tree_hash = _snapshot_digest(source)
+ total_bytes = sum(int(item["bytes"]) for item in inventory)
+ bindings = {
+ "kind": expected_kind,
+ "case_id": case["id"],
+ "snapshot_sha256": case["snapshot_sha256"],
+ "tree_sha256": tree_hash,
+ "file_count": len(inventory),
+ "total_bytes": total_bytes,
+ "files": inventory,
+ }
+ for name, expected in bindings.items():
+ if fixture_manifest.get(name) != expected:
+ raise EvaluationError(
+ f"case {case['id']!r} {field_name!r} manifest binding " f"{name!r} changed"
+ )
+ if declaration["tree_sha256"] != tree_hash:
+ raise EvaluationError(
+ f"case {case['id']!r} {field_name!r} tree changed: "
+ f"expected {declaration['tree_sha256']}, got {tree_hash}"
+ )
+ return {
+ "kind": expected_kind,
+ "source": source,
+ "manifest": fixture_manifest_path,
+ "manifest_sha256": actual_manifest_hash,
+ "tree_sha256": tree_hash,
+ "file_count": len(inventory),
+ "total_bytes": total_bytes,
+ }
+
+
+def validate_manifest(manifest_path: Path) -> dict[str, Any]:
+ manifest = _load_json(manifest_path)
+ if manifest.get("protocol") != PROTOCOL_ID:
+ raise EvaluationError(
+ f"manifest protocol must be {PROTOCOL_ID!r}, got {manifest.get('protocol')!r}"
+ )
+ expected_protocol_hash = manifest.get("protocol_sha256")
+ actual_protocol_hash = _sha256_file(PROTOCOL_PATH)
+ if expected_protocol_hash != actual_protocol_hash:
+ raise EvaluationError(
+ "tracked protocol hash changed after case freeze: "
+ f"expected {expected_protocol_hash}, got {actual_protocol_hash}"
+ )
+ expected_prompt_hash = manifest.get("prompt_sha256")
+ actual_prompt_hash = _sha256_file(PROMPT_PATH)
+ if expected_prompt_hash != actual_prompt_hash:
+ raise EvaluationError(
+ "tracked prompt hash changed after case freeze: "
+ f"expected {expected_prompt_hash}, got {actual_prompt_hash}"
+ )
+
+ for case in _case_index(manifest).values():
+ required = (
+ "source",
+ "snapshot_id",
+ "revision",
+ "commands",
+ "declared_claim",
+ "excluded_scope",
+ "snapshot_sha256",
+ "status",
+ )
+ missing = [field for field in required if field not in case]
+ if missing:
+ raise EvaluationError(f"case {case['id']!r} is missing fields: {', '.join(missing)}")
+ source = (REPOSITORY_ROOT / str(case["source"])).resolve()
+ if not source.is_dir():
+ raise EvaluationError(f"case {case['id']!r} source is not a directory")
+ if (source / ".git").exists():
+ raise EvaluationError(f"case {case['id']!r} source must be origin-stripped")
+ actual_snapshot_hash = _snapshot_digest(source)
+ if case["snapshot_sha256"] != actual_snapshot_hash:
+ raise EvaluationError(
+ f"case {case['id']!r} snapshot changed: "
+ f"expected {case['snapshot_sha256']}, got {actual_snapshot_hash}"
+ )
+ if case.get("status") == "ready":
+ gold_manifest = case.get("gold_manifest")
+ gold_hash = case.get("gold_sha256")
+ if not isinstance(gold_manifest, str) or not isinstance(gold_hash, str):
+ raise EvaluationError(
+ f"ready case {case['id']!r} must freeze gold_manifest and gold_sha256"
+ )
+ gold_path = (REPOSITORY_ROOT / gold_manifest).resolve()
+ if not gold_path.is_file():
+ raise EvaluationError(
+ f"case {case['id']!r} gold manifest is unavailable: {gold_path}"
+ )
+ actual_gold_hash = _sha256_file(gold_path)
+ if gold_hash != actual_gold_hash:
+ raise EvaluationError(
+ f"case {case['id']!r} gold manifest changed: "
+ f"expected {gold_hash}, got {actual_gold_hash}"
+ )
+ _validate_frozen_tree(
+ case,
+ case.get("dependency_cache"),
+ field_name="dependency_cache",
+ expected_kind="nuget-global-packages-v1",
+ )
+ _validate_frozen_tree(
+ case,
+ case.get("native_test_tool"),
+ field_name="native_test_tool",
+ expected_kind="xunit-inproc-v1",
+ )
+ return manifest
+
+
+def render_prompt(case: Mapping[str, Any], condition: str) -> str:
+ prompt = PROMPT_PATH.read_text()
+ substitutions = {
+ "case_id": str(case["id"]),
+ "snapshot_id": str(case["snapshot_id"]),
+ "revision": str(case["revision"]),
+ "commands": str(case["commands"]),
+ "declared_claim": str(case["declared_claim"]),
+ "excluded_scope": str(case["excluded_scope"]),
+ }
+ for name, value in substitutions.items():
+ prompt = prompt.replace("{{" + name + "}}", value)
+ if "{{" in prompt or "}}" in prompt:
+ raise EvaluationError("unresolved prompt placeholder")
+ if condition == "baseline":
+ return prompt
+ if condition in {"docs", "skill"}:
+ return prompt.rstrip() + DOCS_TREATMENT
+ raise EvaluationError(f"unknown treatment condition {condition!r}")
+
+
+def create_plan(
+ manifest_path: Path,
+ plan_path: Path,
+ repetitions: int,
+ conditions: Sequence[str],
+) -> dict[str, Any]:
+ if repetitions < 1:
+ raise EvaluationError("repetitions must be positive")
+ manifest = validate_manifest(manifest_path)
+ cases = _case_index(manifest)
+ entries = [
+ {
+ "run_id": f"{condition}__{case_id}__{agent}__r{repetition}",
+ "condition": condition,
+ "case_id": case_id,
+ "agent": agent,
+ "repetition": repetition,
+ }
+ for condition in conditions
+ for case_id, case in cases.items()
+ if case.get("status") == "ready"
+ for agent in ("codex", "claude")
+ for repetition in range(1, repetitions + 1)
+ ]
+ random.Random(RANDOMIZATION_SEED).shuffle(entries)
+ plan = {
+ "protocol": PROTOCOL_ID,
+ "created_at": _utc_now(),
+ "randomization_seed": RANDOMIZATION_SEED,
+ "manifest_path": str(manifest_path.resolve()),
+ "manifest_sha256": _sha256_file(manifest_path),
+ "runner_sha256": _sha256_file(RUNNER_PATH),
+ "repetitions": repetitions,
+ "conditions": list(conditions),
+ "entries": entries,
+ }
+ _write_json(plan_path, plan)
+ return plan
+
+
+def _clean_agent_environment(scratch: Path) -> dict[str, str]:
+ secret_fragments = ("KEY", "TOKEN", "SECRET", "PASSWORD", "CREDENTIAL")
+ secret_prefixes = ("GH_", "GITHUB_", "AWS_", "GOOGLE_")
+ environment = {
+ name: value
+ for name, value in os.environ.items()
+ if name not in SECRET_ENVIRONMENT_NAMES
+ and not any(fragment in name.upper() for fragment in secret_fragments)
+ and not name.upper().startswith(secret_prefixes)
+ }
+
+ cache_paths = {
+ "TMPDIR": scratch / "tmp",
+ "PIP_CACHE_DIR": scratch / "pip-cache",
+ "UV_CACHE_DIR": scratch / "uv-cache",
+ "CARGO_HOME": scratch / "cargo-home",
+ "NUGET_PACKAGES": scratch / "nuget",
+ "DOTNET_CLI_HOME": scratch / "dotnet-home",
+ "npm_config_cache": scratch / "npm-cache",
+ "CODEX_SQLITE_HOME": scratch / "codex-state",
+ }
+ for name, path in cache_paths.items():
+ path.mkdir(parents=True, exist_ok=True)
+ environment[name] = str(path)
+ environment["GIT_TERMINAL_PROMPT"] = "0"
+ environment["CLAUDE_BASH_MAINTAIN_PROJECT_WORKING_DIR"] = "1"
+ environment["CLAUDE_CODE_SKIP_PROMPT_HISTORY"] = "1"
+ rustup_home = Path.home() / ".rustup"
+ if rustup_home.is_dir():
+ environment["RUSTUP_HOME"] = str(rustup_home)
+ return environment
+
+
+def _copy_frozen_tree(source: Path, target: Path, expected_hash: str) -> None:
+ if target.exists():
+ if not target.is_dir() or any(target.iterdir()):
+ raise EvaluationError(f"fixture destination must be an empty directory: {target}")
+ target.rmdir()
+ try:
+ subprocess.run(
+ ("/bin/cp", "-cR", str(source), str(target)),
+ check=True,
+ stdout=subprocess.DEVNULL,
+ stderr=subprocess.PIPE,
+ text=True,
+ )
+ except subprocess.CalledProcessError:
+ shutil.copytree(source, target)
+ actual_hash = _snapshot_digest(target)
+ if actual_hash != expected_hash:
+ raise EvaluationError(
+ f"fixture copy changed in transit: expected {expected_hash}, got {actual_hash}"
+ )
+
+
+def _prepare_case_fixtures(case: Mapping[str, Any], scratch: Path) -> dict[str, dict[str, Any]]:
+ declarations = (
+ (
+ "dependency_cache",
+ "nuget-global-packages-v1",
+ scratch / "nuget",
+ ),
+ (
+ "native_test_tool",
+ "xunit-inproc-v1",
+ scratch / "native-test-tool",
+ ),
+ )
+ prepared: dict[str, dict[str, Any]] = {}
+ for field_name, expected_kind, target in declarations:
+ fixture = _validate_frozen_tree(
+ case,
+ case.get(field_name),
+ field_name=field_name,
+ expected_kind=expected_kind,
+ )
+ if fixture["kind"] == "none":
+ prepared[field_name] = {"kind": "none"}
+ continue
+ _copy_frozen_tree(
+ Path(fixture["source"]),
+ target,
+ str(fixture["tree_sha256"]),
+ )
+ prepared[field_name] = {
+ "kind": expected_kind,
+ "manifest_sha256": fixture["manifest_sha256"],
+ "initial_tree_sha256": fixture["tree_sha256"],
+ "file_count": fixture["file_count"],
+ "total_bytes": fixture["total_bytes"],
+ "target": str(target),
+ }
+
+ shell_home = scratch / "shell-home"
+ shell_home.mkdir(parents=True, exist_ok=True)
+ nuget_config = shell_home / ".nuget" / "NuGet" / "NuGet.Config"
+ nuget_config.parent.mkdir(parents=True, exist_ok=True)
+ nuget_config.write_text(
+ '\n'
+ "\n"
+ " \n"
+ " \n"
+ "\n"
+ )
+ return prepared
+
+
+def _record_final_fixture_state(fixtures: dict[str, dict[str, Any]], scratch: Path) -> None:
+ targets = {
+ "dependency_cache": scratch / "nuget",
+ "native_test_tool": scratch / "native-test-tool",
+ }
+ for field_name, fixture in fixtures.items():
+ if fixture.get("kind") == "none":
+ continue
+ target = targets[field_name]
+ if not target.is_dir():
+ fixture["final_state"] = "missing"
+ fixture["mutated"] = True
+ continue
+ try:
+ final_hash = _snapshot_digest(target)
+ except OSError as exc:
+ fixture["final_state"] = f"unreadable:{type(exc).__name__}"
+ fixture["mutated"] = True
+ continue
+ fixture["final_state"] = "present"
+ fixture["final_tree_sha256"] = final_hash
+ fixture["mutated"] = final_hash != fixture["initial_tree_sha256"]
+
+
+def _dotnet_environment(scratch: Path) -> dict[str, str]:
+ _, _, dotnet_root = _dotnet_layout()
+ xdg_data_home = scratch / "xdg-data"
+ migration_sentinel = xdg_data_home / "NuGet" / "Migrations" / "1"
+ migration_sentinel.parent.mkdir(parents=True, exist_ok=True)
+ migration_sentinel.touch(exist_ok=True)
+ values = {
+ "DOTNET_EnableDiagnostics": "0",
+ "DOTNET_CLI_TELEMETRY_OPTOUT": "1",
+ "DOTNET_SKIP_FIRST_TIME_EXPERIENCE": "1",
+ "DOTNET_CLI_USE_MSBUILD_SERVER": "0",
+ "MSBUILDUSESERVER": "0",
+ "NUGET_CERT_REVOCATION_MODE": "offline",
+ "NuGetAudit": "false",
+ "HOME": str(scratch / "shell-home"),
+ # NuGet otherwise creates a named mutex before checking its migration
+ # state. Claude's macOS sandbox intentionally blocks that global IPC,
+ # so pre-seed NuGet's documented empty completion marker instead.
+ "XDG_DATA_HOME": str(xdg_data_home),
+ }
+ if dotnet_root is not None:
+ values["DOTNET_ROOT"] = str(dotnet_root)
+ return values
+
+
+def _claude_settings(workspace: Path, scratch: Path) -> dict[str, Any]:
+ user_home = str(Path.home())
+ workspace_rule_path = "//" + str(workspace).lstrip("/")
+ user_home_rule_path = "//" + user_home.lstrip("/")
+ cargo_bin = Path.home() / ".cargo" / "bin"
+ rustup = Path.home() / ".rustup"
+ _, dotnet_launcher_dir, _ = _dotnet_layout()
+ path_parts = [
+ *([str(scratch / "native-test-tool")] if (scratch / "native-test-tool").is_dir() else []),
+ *([str(dotnet_launcher_dir)] if dotnet_launcher_dir is not None else []),
+ "/usr/local/bin",
+ "/opt/homebrew/bin",
+ "/usr/bin",
+ "/bin",
+ "/usr/sbin",
+ "/sbin",
+ ]
+ allow_read = [str(workspace)]
+ if cargo_bin.is_dir():
+ path_parts.append(str(cargo_bin))
+ allow_read.append(str(cargo_bin))
+ if rustup.is_dir():
+ allow_read.append(str(rustup))
+ for dotnet_read_path in (
+ Path("/usr/local/bin/dotnet"),
+ Path("/usr/local/share/dotnet"),
+ Path("/usr/local/opt"),
+ Path("/usr/local/Cellar"),
+ ):
+ if dotnet_read_path.exists():
+ allow_read.append(str(dotnet_read_path))
+
+ environment = {
+ "PATH": ":".join(path_parts),
+ "TMPDIR": str(scratch / "tmp"),
+ "PIP_CACHE_DIR": str(scratch / "pip-cache"),
+ "UV_CACHE_DIR": str(scratch / "uv-cache"),
+ "CARGO_HOME": str(scratch / "cargo-home"),
+ "NUGET_PACKAGES": str(scratch / "nuget"),
+ "DOTNET_CLI_HOME": str(scratch / "dotnet-home"),
+ "npm_config_cache": str(scratch / "npm-cache"),
+ "GIT_TERMINAL_PROMPT": "0",
+ "CLAUDE_BASH_MAINTAIN_PROJECT_WORKING_DIR": "1",
+ }
+ environment.update(_dotnet_environment(scratch))
+
+ return {
+ "env": environment,
+ "permissions": {
+ "defaultMode": "dontAsk",
+ # Claude's OS sandbox applies only to Bash and its children. Its
+ # native file tools need a second, absolute-path allowlist. In
+ # Claude rules ``//`` (represented by the leading slash added
+ # above plus the path's own slash) anchors at the filesystem root;
+ # a single slash would anchor at the settings-file directory.
+ # Edit rules cover Edit, Write, and NotebookEdit.
+ "allow": [
+ f"Read({workspace_rule_path}/**)",
+ f"Edit({workspace_rule_path}/**)",
+ "Bash",
+ "WebSearch",
+ "WebFetch",
+ ],
+ "deny": [
+ f"Read({user_home_rule_path}/**)",
+ f"Edit({user_home_rule_path}/**)",
+ f"Edit({workspace_rule_path}/.git/**)",
+ "Bash(git commit *)",
+ "Bash(git push *)",
+ "Bash(gh *)",
+ "Bash(ssh *)",
+ "Bash(scp *)",
+ "Bash(rsync *)",
+ ],
+ },
+ "sandbox": {
+ "enabled": True,
+ "failIfUnavailable": True,
+ "autoAllowBashIfSandboxed": True,
+ "allowUnsandboxedCommands": False,
+ "filesystem": {
+ "denyRead": [user_home, "/Volumes"],
+ "allowRead": allow_read,
+ "denyWrite": [
+ user_home,
+ "/Volumes",
+ str(workspace / ".git"),
+ str(scratch / "native-test-tool"),
+ ],
+ "allowWrite": [str(workspace), str(scratch)],
+ },
+ "network": {"allowedDomains": list(NETWORK_DOMAINS)},
+ },
+ }
+
+
+def _codex_permission_profile(workspace: Path, scratch: Path) -> str:
+ del workspace # Runtime ``:workspace_roots`` supplies the exact workspace.
+ filesystem = [
+ '":root"="deny"',
+ '":minimal"="read"',
+ f'{json.dumps(str(scratch))}="write"',
+ f'{json.dumps(str(scratch / "native-test-tool"))}="read"',
+ '":workspace_roots"={"."="write",".git"="read"}',
+ ]
+ for read_path in (
+ Path("/Library/Developer/CommandLineTools"),
+ Path("/usr/local/bin/dotnet"),
+ Path("/usr/local/share/dotnet"),
+ Path("/usr/local/opt"),
+ Path("/usr/local/Cellar"),
+ Path.home() / ".cargo" / "bin",
+ Path.home() / ".rustup",
+ ):
+ if read_path.exists():
+ filesystem.append(f'{json.dumps(str(read_path))}="read"')
+ domains = ",".join(f'{json.dumps(domain)}="allow"' for domain in NETWORK_DOMAINS)
+ return (
+ "permissions.agent-eval={"
+ f"filesystem={{{','.join(filesystem)}}},"
+ 'network={enabled=true,mode="limited",allow_upstream_proxy=false,'
+ f"domains={{{domains}}}}}"
+ "}"
+ )
+
+
+def _codex_shell_environment(scratch: Path) -> dict[str, str]:
+ _, dotnet_launcher_dir, _ = _dotnet_layout()
+ path_parts = [
+ *([str(scratch / "native-test-tool")] if (scratch / "native-test-tool").is_dir() else []),
+ *([str(dotnet_launcher_dir)] if dotnet_launcher_dir is not None else []),
+ "/Library/Developer/CommandLineTools/usr/bin",
+ "/usr/local/bin",
+ "/opt/homebrew/bin",
+ "/usr/bin",
+ "/bin",
+ "/usr/sbin",
+ "/sbin",
+ ]
+ cargo_bin = Path.home() / ".cargo" / "bin"
+ if cargo_bin.is_dir():
+ path_parts.append(str(cargo_bin))
+ values = {
+ "PATH": ":".join(path_parts),
+ "TMPDIR": str(scratch / "tmp"),
+ "PIP_CACHE_DIR": str(scratch / "pip-cache"),
+ "UV_CACHE_DIR": str(scratch / "uv-cache"),
+ "CARGO_HOME": str(scratch / "cargo-home"),
+ "NUGET_PACKAGES": str(scratch / "nuget"),
+ "DOTNET_CLI_HOME": str(scratch / "dotnet-home"),
+ "npm_config_cache": str(scratch / "npm-cache"),
+ "GIT_TERMINAL_PROMPT": "0",
+ "GIT_OPTIONAL_LOCKS": "0",
+ }
+ rustup_home = Path.home() / ".rustup"
+ if rustup_home.is_dir():
+ values["RUSTUP_HOME"] = str(rustup_home)
+ values.update(_dotnet_environment(scratch))
+ return values
+
+
+def _codex_command(workspace: Path, scratch: Path) -> list[str]:
+ permission_profile = _codex_permission_profile(workspace, scratch)
+ shell_environment = ",".join(
+ f"{json.dumps(name)}={json.dumps(value)}"
+ for name, value in _codex_shell_environment(scratch).items()
+ )
+ secret_excludes = (
+ 'shell_environment_policy.exclude=["*KEY*","*TOKEN*","*SECRET*",'
+ '"*PASSWORD*","*CREDENTIAL*","GH_*","GITHUB_*","AWS_*","GOOGLE_*",'
+ '"DATABASE_URL","CODEX_HOME","SSH_AUTH_SOCK"]'
+ )
+ return [
+ str(CODEX_BINARY),
+ "--ask-for-approval",
+ "never",
+ "--search",
+ "--disable",
+ "apps",
+ "--disable",
+ "plugins",
+ "--disable",
+ "remote_plugin",
+ "--disable",
+ "skill_search",
+ "--disable",
+ "tool_suggest",
+ "--disable",
+ "plugin_sharing",
+ "--disable",
+ "workspace_dependencies",
+ "exec",
+ "--ephemeral",
+ "--ignore-user-config",
+ "--ignore-rules",
+ "--strict-config",
+ "--skip-git-repo-check",
+ "--json",
+ "--color",
+ "never",
+ "--model",
+ "gpt-5.6-sol",
+ "--cd",
+ str(workspace),
+ "-c",
+ "features.network_proxy.enabled=true",
+ "-c",
+ 'default_permissions="agent-eval"',
+ "-c",
+ permission_profile,
+ "-c",
+ "project_doc_max_bytes=0",
+ "-c",
+ "project_doc_fallback_filenames=[]",
+ "-c",
+ 'model_reasoning_effort="high"',
+ "-c",
+ "allow_login_shell=false",
+ "-c",
+ 'history.persistence="none"',
+ "-c",
+ 'shell_environment_policy.inherit="core"',
+ "-c",
+ f"shell_environment_policy.set={{{shell_environment}}}",
+ "-c",
+ secret_excludes,
+ "-c",
+ "features.apps=false",
+ "-c",
+ "features.remote_plugin=false",
+ "-c",
+ "features.multi_agent=false",
+ "-c",
+ "features.memories=false",
+ "-c",
+ "features.hooks=false",
+ "-c",
+ "features.skill_mcp_dependency_install=false",
+ "-c",
+ "features.goals=false",
+ "-c",
+ "agents.enabled=false",
+ "-c",
+ "check_for_update_on_startup=false",
+ "-c",
+ "feedback.enabled=false",
+ "-c",
+ "analytics.enabled=false",
+ "-c",
+ 'personality="none"',
+ "-",
+ ]
+
+
+def _prepare_codex_environment(
+ external_run_root: Path,
+ scratch: Path,
+ environment: dict[str, str],
+) -> Path:
+ if not CODEX_AUTH_PATH.is_file():
+ raise EvaluationError(f"Codex auth is unavailable at {CODEX_AUTH_PATH}")
+ codex_home = external_run_root / "codex-home"
+ codex_home.mkdir()
+ codex_home.chmod(0o700)
+ codex_state = codex_home / "state"
+ codex_state.mkdir()
+ (codex_home / "auth.json").symlink_to(CODEX_AUTH_PATH)
+ shell_home = scratch / "shell-home"
+ shell_home.mkdir(exist_ok=True)
+ environment.update(_codex_shell_environment(scratch))
+ environment["CODEX_HOME"] = str(codex_home)
+ environment["CODEX_SQLITE_HOME"] = str(codex_state)
+ return codex_home
+
+
+def _claude_command(settings_path: Path) -> list[str]:
+ return [
+ str(CLAUDE_BINARY),
+ "-p",
+ "--safe-mode",
+ "--no-session-persistence",
+ "--no-chrome",
+ "--disable-slash-commands",
+ "--setting-sources",
+ "",
+ "--strict-mcp-config",
+ "--mcp-config",
+ '{"mcpServers":{}}',
+ "--settings",
+ str(settings_path),
+ "--permission-mode",
+ "dontAsk",
+ "--tools",
+ "Read,Glob,Grep,Edit,Write,Bash,WebSearch,WebFetch",
+ "--model",
+ "opus",
+ "--effort",
+ "high",
+ "--max-budget-usd",
+ "50",
+ "--output-format",
+ "stream-json",
+ "--verbose",
+ ]
+
+
+def _run_subprocess(
+ command: Sequence[str],
+ *,
+ cwd: Path,
+ environment: Mapping[str, str],
+ prompt_stdin: str | None,
+ stdout_path: Path,
+ stderr_path: Path,
+ timeout_seconds: int,
+) -> tuple[int, bool, float]:
+ started = time.monotonic()
+ timed_out = False
+ with stdout_path.open("wb") as stdout, stderr_path.open("wb") as stderr:
+ process = subprocess.Popen(
+ list(command),
+ cwd=cwd,
+ env=dict(environment),
+ stdin=subprocess.PIPE if prompt_stdin is not None else subprocess.DEVNULL,
+ stdout=stdout,
+ stderr=stderr,
+ start_new_session=True,
+ )
+ try:
+ process.communicate(
+ prompt_stdin.encode() if prompt_stdin is not None else None,
+ timeout=timeout_seconds,
+ )
+ except subprocess.TimeoutExpired:
+ timed_out = True
+ os.killpg(process.pid, signal.SIGTERM)
+ try:
+ process.wait(timeout=10)
+ except subprocess.TimeoutExpired:
+ os.killpg(process.pid, signal.SIGKILL)
+ process.wait()
+ elapsed = time.monotonic() - started
+ return process.returncode, timed_out, elapsed
+
+
+def _run_checked(command: Sequence[str], cwd: Path) -> str:
+ completed = subprocess.run(
+ list(command),
+ cwd=cwd,
+ check=True,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT,
+ text=True,
+ )
+ return completed.stdout
+
+
+def _binary_version(binary: Path, environment: Mapping[str, str]) -> str:
+ completed = subprocess.run(
+ (str(binary), "--version"),
+ env=dict(environment),
+ check=True,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.DEVNULL,
+ text=True,
+ timeout=15,
+ )
+ return completed.stdout.strip()
+
+
+def _reported_model(transcript_path: Path, agent: str) -> tuple[str, str]:
+ if agent == "codex":
+ # Codex's JSONL protocol does not currently repeat the selected model;
+ # the CLI's exact, non-alias model argument is the authoritative value.
+ return "gpt-5.6-sol", "command"
+ with transcript_path.open() as handle:
+ for line in handle:
+ try:
+ event = json.loads(line)
+ except json.JSONDecodeError:
+ continue
+ if not isinstance(event, dict):
+ continue
+ model = event.get("model")
+ if isinstance(model, str) and model:
+ return model, "transcript"
+ message = event.get("message")
+ if isinstance(message, dict):
+ model = message.get("model")
+ if isinstance(model, str) and model:
+ return model, "transcript"
+ return "opus", "command-fallback"
+
+
+def _initialize_snapshot(source: Path, workspace: Path) -> None:
+ shutil.copytree(source, workspace, symlinks=True)
+ _run_checked(("git", "init", "-q"), workspace)
+ _run_checked(("git", "config", "user.name", "Tracebook evaluator"), workspace)
+ _run_checked(("git", "config", "user.email", "evaluator@tracebook.invalid"), workspace)
+ _run_checked(("git", "config", "commit.gpgsign", "false"), workspace)
+ _run_checked(("git", "add", "-A"), workspace)
+ _run_checked(("git", "commit", "-q", "-m", "Frozen candidate snapshot"), workspace)
+
+
+def _workspace_inventory(workspace: Path) -> list[dict[str, Any]]:
+ inventory: list[dict[str, Any]] = []
+ excluded_parts = {".git", ".eval-cache", "bin", "obj", "target", "node_modules"}
+ for path in sorted(workspace.rglob("*"), key=lambda value: value.as_posix()):
+ relative = path.relative_to(workspace)
+ if any(part in excluded_parts for part in relative.parts):
+ continue
+ if path.is_file() and not path.is_symlink():
+ inventory.append(
+ {
+ "path": relative.as_posix(),
+ "bytes": path.stat().st_size,
+ "sha256": _sha256_file(path),
+ }
+ )
+ return inventory
+
+
+def _write_workspace_evidence(workspace: Path, run_root: Path) -> None:
+ """Capture tracked, untracked, and ignored candidate changes without mutating its index."""
+
+ inventory = _workspace_inventory(workspace)
+ temporary_index = run_root / ".workspace-evidence.index"
+ environment = dict(os.environ)
+ environment["GIT_INDEX_FILE"] = str(temporary_index)
+
+ def run_git(*arguments: str) -> str:
+ completed = subprocess.run(
+ ("git", *arguments),
+ cwd=workspace,
+ env=environment,
+ check=True,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT,
+ text=True,
+ )
+ return completed.stdout
+
+ try:
+ run_git("read-tree", "HEAD")
+ run_git("add", "-u", "--", ".")
+ paths = ["./" + str(item["path"]) for item in inventory]
+ for start in range(0, len(paths), 200):
+ run_git("add", "-f", "--", *paths[start : start + 200])
+ (run_root / "git-status.txt").write_text(run_git("status", "--short"))
+ (run_root / "workspace.patch").write_text(run_git("diff", "--cached", "--binary", "HEAD"))
+ finally:
+ temporary_index.unlink(missing_ok=True)
+ _write_json(run_root / "workspace-files.json", {"files": inventory})
+
+
+def _quarantine_interrupted_run(run_root: Path, external_run_root: Path) -> Path | None:
+ """Preserve incomplete evidence under ``interruptions`` and unblock a clean retry."""
+
+ if external_run_root.is_symlink():
+ external_run_root.unlink()
+ elif external_run_root.exists():
+ shutil.rmtree(external_run_root)
+ if not run_root.exists() and not run_root.is_symlink():
+ return None
+ if not run_root.is_symlink() and (run_root / "pre-score-verdict.json").is_file():
+ return None
+ interruptions = run_root.parent / "interruptions"
+ interruptions.mkdir(parents=True, exist_ok=True)
+ stamp = dt.datetime.now(dt.timezone.utc).strftime("%Y%m%dT%H%M%S.%fZ")
+ target = interruptions / f"{run_root.name}.{stamp}.{os.getpid()}"
+ run_root.replace(target)
+ return target
+
+
+def _execute_run_once(
+ manifest_path: Path,
+ *,
+ case_id: str,
+ agent: str,
+ repetition: int,
+ condition: str,
+ runs_path: Path,
+ timeout_seconds: int,
+) -> dict[str, Any]:
+ manifest = validate_manifest(manifest_path)
+ cases = _case_index(manifest)
+ try:
+ case = cases[case_id]
+ except KeyError as exc:
+ raise EvaluationError(f"unknown case {case_id!r}") from exc
+ if case.get("status") != "ready":
+ raise EvaluationError(f"case {case_id!r} is not ready")
+ if agent not in {"codex", "claude"}:
+ raise EvaluationError(f"unknown agent {agent!r}")
+ if repetition < 1:
+ raise EvaluationError("repetition must be positive")
+ if condition == "skill":
+ raise EvaluationError("skill-assisted runs remain locked until baseline grading")
+
+ run_id = f"{condition}__{case_id}__{agent}__r{repetition}"
+ run_root = runs_path.resolve() / run_id
+ if run_root.exists():
+ raise EvaluationError(f"refusing to overwrite existing run {run_root}")
+
+ source = (REPOSITORY_ROOT / str(case["source"])).resolve()
+ work_root = CODEX_WORK_ROOT if agent == "codex" else CLAUDE_WORK_ROOT
+ external_run_root = work_root / run_id
+ if external_run_root.exists():
+ raise EvaluationError(f"refusing to reuse external workspace {external_run_root}")
+ workspace = external_run_root / "workspace"
+ scratch = external_run_root / "scratch"
+ transcript_path = run_root / f"{agent}.jsonl"
+ stderr_path = run_root / f"{agent}.stderr"
+ run_root.mkdir(parents=True)
+ scratch.mkdir(parents=True)
+ _initialize_snapshot(source, workspace)
+
+ prompt = render_prompt(case, condition)
+ (run_root / "prompt.txt").write_text(prompt)
+ environment = _clean_agent_environment(scratch)
+ fixtures = _prepare_case_fixtures(case, scratch)
+ settings_path: Path | None = None
+ if agent == "codex":
+ if not CODEX_BINARY.is_file():
+ raise EvaluationError(f"Codex binary is unavailable at {CODEX_BINARY}")
+ _prepare_codex_environment(external_run_root, scratch, environment)
+ command = _codex_command(workspace, scratch)
+ prompt_stdin = prompt
+ else:
+ if not CLAUDE_BINARY.is_file():
+ raise EvaluationError(f"Claude binary is unavailable at {CLAUDE_BINARY}")
+ settings_path = run_root / "claude-settings.json"
+ _write_json(settings_path, _claude_settings(workspace, scratch))
+ command = _claude_command(settings_path)
+ prompt_stdin = prompt
+
+ cli_version = _binary_version(CODEX_BINARY if agent == "codex" else CLAUDE_BINARY, environment)
+ started_at = _utc_now()
+ metadata: dict[str, Any] = {
+ "protocol": PROTOCOL_ID,
+ "run_id": run_id,
+ "condition": condition,
+ "case_id": case_id,
+ "agent": agent,
+ "repetition": repetition,
+ "started_at": started_at,
+ "timeout_seconds": timeout_seconds,
+ "manifest_sha256": _sha256_file(manifest_path),
+ "gold_manifest_sha256": str(case["gold_sha256"]),
+ "runner_sha256": _sha256_file(RUNNER_PATH),
+ "protocol_sha256": _sha256_file(PROTOCOL_PATH),
+ "prompt_template_sha256": _sha256_file(PROMPT_PATH),
+ "rendered_prompt_sha256": _sha256_bytes(prompt.encode()),
+ "snapshot_sha256": _snapshot_digest(source),
+ "command": command,
+ "prompt_transport": "stdin",
+ "cli_version": cli_version,
+ "requested_model": "gpt-5.6-sol" if agent == "codex" else "opus",
+ "removed_environment_names": sorted(SECRET_ENVIRONMENT_NAMES),
+ "fixtures": fixtures,
+ }
+ _write_json(run_root / "metadata.json", metadata)
+
+ exit_code, timed_out, elapsed_seconds = _run_subprocess(
+ command,
+ cwd=workspace,
+ environment=environment,
+ prompt_stdin=prompt_stdin,
+ stdout_path=transcript_path,
+ stderr_path=stderr_path,
+ timeout_seconds=timeout_seconds,
+ )
+ _write_workspace_evidence(workspace, run_root)
+ _record_final_fixture_state(fixtures, scratch)
+ shutil.copytree(
+ workspace,
+ run_root / "workspace-final",
+ symlinks=True,
+ ignore=shutil.ignore_patterns(
+ ".git", "bin", "obj", "target", "node_modules", ".eval-cache"
+ ),
+ )
+
+ metadata.update(
+ {
+ "completed_at": _utc_now(),
+ "exit_code": exit_code,
+ "timed_out": timed_out,
+ "elapsed_seconds": elapsed_seconds,
+ "transcript_sha256": _sha256_file(transcript_path),
+ "stderr_sha256": _sha256_file(stderr_path),
+ "fixtures": fixtures,
+ }
+ )
+ model_identifier, model_identifier_source = _reported_model(transcript_path, agent)
+ metadata["model_identifier"] = model_identifier
+ metadata["model_identifier_source"] = model_identifier_source
+ _write_json(run_root / "metadata.json", metadata)
+ shutil.rmtree(external_run_root)
+ return metadata
+
+
+def execute_run(
+ manifest_path: Path,
+ *,
+ case_id: str,
+ agent: str,
+ repetition: int,
+ condition: str,
+ runs_path: Path,
+ timeout_seconds: int,
+) -> dict[str, Any]:
+ run_id = f"{condition}__{case_id}__{agent}__r{repetition}"
+ run_root = runs_path.resolve() / run_id
+ work_root = CODEX_WORK_ROOT if agent == "codex" else CLAUDE_WORK_ROOT
+ external_run_root = work_root / run_id
+ run_root_preexisting = run_root.exists() or run_root.is_symlink()
+ external_root_preexisting = external_run_root.exists() or external_run_root.is_symlink()
+ try:
+ return _execute_run_once(
+ manifest_path,
+ case_id=case_id,
+ agent=agent,
+ repetition=repetition,
+ condition=condition,
+ runs_path=runs_path,
+ timeout_seconds=timeout_seconds,
+ )
+ except BaseException:
+ if not run_root_preexisting and not external_root_preexisting:
+ _quarantine_interrupted_run(run_root, external_run_root)
+ raise
+
+
+def _parse_conditions(raw: str) -> list[str]:
+ conditions = [item.strip() for item in raw.split(",") if item.strip()]
+ unknown = [item for item in conditions if item not in {"baseline", "docs", "skill"}]
+ if unknown:
+ raise EvaluationError(f"unknown conditions: {', '.join(unknown)}")
+ if not conditions:
+ raise EvaluationError("at least one condition is required")
+ return conditions
+
+
+def build_parser() -> argparse.ArgumentParser:
+ parser = argparse.ArgumentParser(description=__doc__)
+ subparsers = parser.add_subparsers(dest="command", required=True)
+
+ validate = subparsers.add_parser("validate", help="validate frozen inputs")
+ validate.add_argument("--manifest", type=Path, default=DEFAULT_MANIFEST_PATH)
+
+ plan = subparsers.add_parser("plan", help="create a deterministic run order")
+ plan.add_argument("--manifest", type=Path, default=DEFAULT_MANIFEST_PATH)
+ plan.add_argument("--output", type=Path, default=DEFAULT_PLAN_PATH)
+ plan.add_argument("--repetitions", type=int, default=DEFAULT_REPETITIONS)
+ plan.add_argument("--conditions", default="baseline")
+
+ run = subparsers.add_parser("run", help="execute one isolated run")
+ run.add_argument("--manifest", type=Path, default=DEFAULT_MANIFEST_PATH)
+ run.add_argument("--case", required=True)
+ run.add_argument("--agent", choices=("codex", "claude"), required=True)
+ run.add_argument("--repetition", type=int, required=True)
+ run.add_argument("--condition", choices=("baseline", "docs", "skill"), default="baseline")
+ run.add_argument("--runs-path", type=Path, default=DEFAULT_RUNS_PATH)
+ run.add_argument("--timeout-seconds", type=int, default=DEFAULT_TIMEOUT_SECONDS)
+ return parser
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+ parser = build_parser()
+ arguments = parser.parse_args(argv)
+ try:
+ if arguments.command == "validate":
+ manifest = validate_manifest(arguments.manifest)
+ print(
+ json.dumps(
+ {
+ "protocol": manifest["protocol"],
+ "cases": sorted(_case_index(manifest)),
+ "status": "valid",
+ },
+ sort_keys=True,
+ )
+ )
+ elif arguments.command == "plan":
+ plan = create_plan(
+ arguments.manifest,
+ arguments.output,
+ arguments.repetitions,
+ _parse_conditions(arguments.conditions),
+ )
+ print(
+ json.dumps(
+ {
+ "entries": len(plan["entries"]),
+ "output": str(arguments.output),
+ "status": "created",
+ },
+ sort_keys=True,
+ )
+ )
+ elif arguments.command == "run":
+ metadata = execute_run(
+ arguments.manifest,
+ case_id=arguments.case,
+ agent=arguments.agent,
+ repetition=arguments.repetition,
+ condition=arguments.condition,
+ runs_path=arguments.runs_path,
+ timeout_seconds=arguments.timeout_seconds,
+ )
+ print(json.dumps(metadata, sort_keys=True))
+ else: # pragma: no cover - argparse enforces the command set
+ parser.error(f"unsupported command {arguments.command!r}")
+ except (EvaluationError, OSError, subprocess.CalledProcessError) as exc:
+ print(f"agent qualification evaluation failed: {exc}", file=sys.stderr)
+ return 2
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/experiments/agent_qualification_generalization.py b/experiments/agent_qualification_generalization.py
new file mode 100644
index 0000000..0a27404
--- /dev/null
+++ b/experiments/agent_qualification_generalization.py
@@ -0,0 +1,548 @@
+"""Freeze and validate the v2 held-out agent-qualification cohort.
+
+This module intentionally does not launch provider CLIs. Execution remains
+blocked until the user explicitly authorizes transmission of the new public
+candidate snapshots and frozen treatment material to each provider.
+"""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import os
+import random
+import stat
+from pathlib import Path
+from typing import Any, Mapping, Sequence
+
+REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
+RUNNER_PATH = Path(__file__).resolve()
+PROTOCOL_PATH = REPOSITORY_ROOT / "docs" / "agent-qualification-generalization-v2.md"
+PROMPT_PATH = REPOSITORY_ROOT / "experiments" / "agent_qualification_generalization_prompt.txt"
+PRIVATE_ROOT = REPOSITORY_ROOT / "experiments" / "private" / "agent-qualification-v2"
+DRAFT_PATH = PRIVATE_ROOT / "draft.json"
+MANIFEST_PATH = PRIVATE_ROOT / "cases.json"
+PLAN_PATH = PRIVATE_ROOT / "plan.json"
+FREEZE_PATH = PRIVATE_ROOT / "freeze.json"
+MANIFESTS_ROOT = PRIVATE_ROOT / "manifests"
+SKILL_PATH = PRIVATE_ROOT / "treatments" / "tracebook-qualify-matching-engine" / "SKILL.md"
+
+PROTOCOL_ID = "agent-qualification-generalization-v2"
+RANDOMIZATION_SEED = 20260730
+REPETITIONS = 3
+AGENTS = ("codex", "claude")
+CONDITIONS = ("docs", "skill")
+DEPENDENCY_TARGETS = ("m2-repository", "m2-seed", "cargo-home", "cargo-vendor")
+
+DOCS_TREATMENT = (
+ "\n\nYou may evaluate tracebook-conformance==0.6.0 using only its public "
+ "README and conformance documentation. Use it only if it fits the "
+ "declared contract.\n"
+)
+
+
+class GeneralizationError(RuntimeError):
+ """Raised when a frozen v2 precondition is not satisfied."""
+
+
+def _sha256_file(path: Path) -> str:
+ digest = hashlib.sha256()
+ with path.open("rb") as handle:
+ for chunk in iter(lambda: handle.read(1024 * 1024), b""):
+ digest.update(chunk)
+ return digest.hexdigest()
+
+
+def _snapshot_digest(root: Path) -> str:
+ """Hash relative file names, modes, links, and bytes deterministically."""
+
+ digest = hashlib.sha256()
+ for path in sorted(root.rglob("*"), key=lambda value: value.as_posix()):
+ relative_path = path.relative_to(root)
+ if ".git" in relative_path.parts:
+ continue
+ relative = relative_path.as_posix()
+ if path.is_symlink():
+ digest.update(b"L\0")
+ digest.update(relative.encode())
+ digest.update(b"\0")
+ digest.update(os.readlink(path).encode())
+ digest.update(b"\0")
+ elif path.is_file():
+ digest.update(b"F\0")
+ digest.update(relative.encode())
+ digest.update(b"\0")
+ digest.update(f"{path.stat().st_mode & 0o777:o}".encode())
+ digest.update(b"\0")
+ with path.open("rb") as handle:
+ for chunk in iter(lambda: handle.read(1024 * 1024), b""):
+ digest.update(chunk)
+ digest.update(b"\0")
+ return digest.hexdigest()
+
+
+def _tree_inventory(root: Path) -> list[dict[str, Any]]:
+ inventory: list[dict[str, Any]] = []
+ for path in sorted(root.rglob("*"), key=lambda value: value.as_posix()):
+ relative_path = path.relative_to(root)
+ if ".git" in relative_path.parts:
+ continue
+ relative = relative_path.as_posix()
+ if path.is_symlink():
+ raise GeneralizationError(f"frozen tree must not contain symlink {relative!r}")
+ mode = path.stat().st_mode
+ if stat.S_ISDIR(mode):
+ continue
+ if not stat.S_ISREG(mode):
+ raise GeneralizationError(f"frozen tree contains unsafe entry {relative!r}")
+ inventory.append(
+ {
+ "path": relative,
+ "bytes": path.stat().st_size,
+ "sha256": _sha256_file(path),
+ }
+ )
+ return inventory
+
+
+def _load_json(path: Path) -> dict[str, Any]:
+ try:
+ payload = json.loads(path.read_text())
+ except (OSError, json.JSONDecodeError) as exc:
+ raise GeneralizationError(f"cannot read JSON from {path}: {exc}") from exc
+ if not isinstance(payload, dict):
+ raise GeneralizationError(f"{path} must contain one JSON object")
+ return payload
+
+
+def _write_json(path: Path, payload: Mapping[str, Any]) -> None:
+ path.parent.mkdir(parents=True, exist_ok=True)
+ path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n")
+
+
+def _relative(path: Path) -> str:
+ return path.resolve().relative_to(REPOSITORY_ROOT.resolve()).as_posix()
+
+
+def _private_path(raw_path: str, description: str) -> Path:
+ path = (REPOSITORY_ROOT / raw_path).resolve()
+ try:
+ path.relative_to(PRIVATE_ROOT.resolve())
+ except ValueError as exc:
+ raise GeneralizationError(f"{description} must stay inside {PRIVATE_ROOT}") from exc
+ return path
+
+
+def _cases(payload: Mapping[str, Any]) -> list[dict[str, Any]]:
+ raw_cases = payload.get("cases")
+ if not isinstance(raw_cases, list) or not raw_cases:
+ raise GeneralizationError("case payload must contain a non-empty cases list")
+ result: list[dict[str, Any]] = []
+ seen: set[str] = set()
+ for raw in raw_cases:
+ if not isinstance(raw, dict):
+ raise GeneralizationError("every case must be an object")
+ case_id = raw.get("id")
+ if not isinstance(case_id, str) or not case_id:
+ raise GeneralizationError("every case must have a non-empty id")
+ if case_id in seen:
+ raise GeneralizationError(f"duplicate case id {case_id!r}")
+ seen.add(case_id)
+ result.append(dict(raw))
+ return result
+
+
+def _freeze_tree(
+ *,
+ case_id: str,
+ name: str,
+ source_path: str,
+ snapshot_sha256: str,
+) -> dict[str, Any]:
+ source = _private_path(source_path, f"{case_id} {name} source")
+ if not source.is_dir():
+ raise GeneralizationError(f"{case_id} {name} source is unavailable: {source}")
+ inventory = _tree_inventory(source)
+ tree_sha256 = _snapshot_digest(source)
+ manifest = {
+ "kind": "frozen-tree-v1",
+ "case_id": case_id,
+ "name": name,
+ "snapshot_sha256": snapshot_sha256,
+ "tree_sha256": tree_sha256,
+ "file_count": len(inventory),
+ "total_bytes": sum(int(item["bytes"]) for item in inventory),
+ "files": inventory,
+ }
+ manifest_path = MANIFESTS_ROOT / f"{case_id}--{name}.json"
+ _write_json(manifest_path, manifest)
+ return {
+ "kind": "frozen-tree-v1",
+ "source": _relative(source),
+ "manifest": _relative(manifest_path),
+ "manifest_sha256": _sha256_file(manifest_path),
+ "tree_sha256": tree_sha256,
+ }
+
+
+def _validate_tree(
+ case: Mapping[str, Any],
+ declaration: Any,
+ *,
+ name: str,
+) -> dict[str, Any]:
+ if not isinstance(declaration, dict) or declaration.get("kind") != "frozen-tree-v1":
+ raise GeneralizationError(f"case {case['id']!r} {name!r} must be frozen-tree-v1")
+ for field in ("source", "manifest", "manifest_sha256", "tree_sha256"):
+ if not isinstance(declaration.get(field), str):
+ raise GeneralizationError(f"case {case['id']!r} {name!r} lacks {field!r}")
+ source = _private_path(str(declaration["source"]), f"{case['id']} {name} source")
+ manifest_path = _private_path(str(declaration["manifest"]), f"{case['id']} {name} manifest")
+ if not source.is_dir() or not manifest_path.is_file():
+ raise GeneralizationError(f"case {case['id']!r} {name!r} tree is unavailable")
+ manifest_hash = _sha256_file(manifest_path)
+ if manifest_hash != declaration["manifest_sha256"]:
+ raise GeneralizationError(f"case {case['id']!r} {name!r} manifest drifted")
+ manifest = _load_json(manifest_path)
+ inventory = _tree_inventory(source)
+ tree_sha256 = _snapshot_digest(source)
+ expected = {
+ "kind": "frozen-tree-v1",
+ "case_id": case["id"],
+ "name": name,
+ "snapshot_sha256": case["snapshot_sha256"],
+ "tree_sha256": tree_sha256,
+ "file_count": len(inventory),
+ "total_bytes": sum(int(item["bytes"]) for item in inventory),
+ "files": inventory,
+ }
+ if manifest != expected:
+ raise GeneralizationError(f"case {case['id']!r} {name!r} inventory drifted")
+ if declaration["tree_sha256"] != tree_sha256:
+ raise GeneralizationError(f"case {case['id']!r} {name!r} tree drifted")
+ return {
+ "tree_sha256": tree_sha256,
+ "file_count": len(inventory),
+ "total_bytes": expected["total_bytes"],
+ }
+
+
+def _dependency_target(value: Any) -> str:
+ if value not in DEPENDENCY_TARGETS:
+ raise GeneralizationError(
+ "dependency_target must be one of " + ", ".join(DEPENDENCY_TARGETS)
+ )
+ return str(value)
+
+
+def freeze() -> dict[str, Any]:
+ """Create the immutable v2 case manifest, plan, and freeze binding."""
+
+ existing = [path for path in (MANIFEST_PATH, PLAN_PATH, FREEZE_PATH) if path.exists()]
+ if existing:
+ raise GeneralizationError(
+ "refusing to overwrite frozen v2 files: " + ", ".join(str(path) for path in existing)
+ )
+ draft = _load_json(DRAFT_PATH)
+ if draft.get("protocol") != PROTOCOL_ID:
+ raise GeneralizationError(f"draft protocol must be {PROTOCOL_ID!r}")
+ if not PROTOCOL_PATH.is_file() or not PROMPT_PATH.is_file() or not SKILL_PATH.is_file():
+ raise GeneralizationError("tracked protocol, prompt, and v2 skill must exist before freeze")
+
+ frozen_cases: list[dict[str, Any]] = []
+ for raw_case in _cases(draft):
+ required = (
+ "source",
+ "revision",
+ "commands",
+ "declared_claim",
+ "excluded_scope",
+ "origin",
+ "gold_manifest",
+ "dependency_source",
+ "dependency_target",
+ "gold_evidence",
+ "status",
+ )
+ missing = [field for field in required if field not in raw_case]
+ if missing:
+ raise GeneralizationError(
+ f"case {raw_case['id']!r} is missing fields: {', '.join(missing)}"
+ )
+ source = _private_path(str(raw_case["source"]), f"{raw_case['id']} source")
+ if not source.is_dir() or (source / ".git").exists():
+ raise GeneralizationError(
+ f"case {raw_case['id']!r} source must be an origin-stripped directory"
+ )
+ snapshot_sha256 = _snapshot_digest(source)
+ gold_path = _private_path(str(raw_case["gold_manifest"]), f"{raw_case['id']} gold manifest")
+ if not gold_path.is_file():
+ raise GeneralizationError(f"case {raw_case['id']!r} gold is unavailable")
+ evidence_specs = raw_case["gold_evidence"]
+ if not isinstance(evidence_specs, list):
+ raise GeneralizationError(f"case {raw_case['id']!r} gold_evidence must be a list")
+ evidence: list[dict[str, Any]] = []
+ for item in evidence_specs:
+ if not isinstance(item, dict) or not isinstance(item.get("name"), str):
+ raise GeneralizationError("gold evidence entries require name and source")
+ evidence.append(
+ {
+ "name": item["name"],
+ **_freeze_tree(
+ case_id=str(raw_case["id"]),
+ name=str(item["name"]),
+ source_path=str(item.get("source", "")),
+ snapshot_sha256=snapshot_sha256,
+ ),
+ }
+ )
+ dependency = _freeze_tree(
+ case_id=str(raw_case["id"]),
+ name="dependency-cache",
+ source_path=str(raw_case["dependency_source"]),
+ snapshot_sha256=snapshot_sha256,
+ )
+ dependency["target"] = _dependency_target(raw_case["dependency_target"])
+ frozen_cases.append(
+ {
+ "id": raw_case["id"],
+ "source": _relative(source),
+ "snapshot_id": f"sha256:{snapshot_sha256}",
+ "snapshot_sha256": snapshot_sha256,
+ "revision": raw_case["revision"],
+ "commands": raw_case["commands"],
+ "declared_claim": raw_case["declared_claim"],
+ "excluded_scope": raw_case["excluded_scope"],
+ "origin": raw_case["origin"],
+ "gold_manifest": _relative(gold_path),
+ "gold_sha256": _sha256_file(gold_path),
+ "dependency_cache": dependency,
+ "gold_evidence": evidence,
+ "status": raw_case["status"],
+ }
+ )
+
+ manifest = {
+ "protocol": PROTOCOL_ID,
+ "protocol_sha256": _sha256_file(PROTOCOL_PATH),
+ "prompt_sha256": _sha256_file(PROMPT_PATH),
+ "skill_sha256": _sha256_file(SKILL_PATH),
+ "authorization": {
+ "codex": "pending-explicit-case-specific-authorization",
+ "claude": "pending-explicit-case-specific-authorization",
+ },
+ "cases": frozen_cases,
+ }
+ _write_json(MANIFEST_PATH, manifest)
+
+ plan = create_plan(manifest)
+ _write_json(PLAN_PATH, plan)
+ freeze_payload = {
+ "protocol": PROTOCOL_ID,
+ "randomization_seed": RANDOMIZATION_SEED,
+ "repetitions": REPETITIONS,
+ "agents": list(AGENTS),
+ "conditions": list(CONDITIONS),
+ "bindings": {
+ "draft": {"path": _relative(DRAFT_PATH), "sha256": _sha256_file(DRAFT_PATH)},
+ "manifest": {
+ "path": _relative(MANIFEST_PATH),
+ "sha256": _sha256_file(MANIFEST_PATH),
+ },
+ "plan": {"path": _relative(PLAN_PATH), "sha256": _sha256_file(PLAN_PATH)},
+ "protocol": {
+ "path": _relative(PROTOCOL_PATH),
+ "sha256": _sha256_file(PROTOCOL_PATH),
+ },
+ "prompt": {"path": _relative(PROMPT_PATH), "sha256": _sha256_file(PROMPT_PATH)},
+ "runner": {"path": _relative(RUNNER_PATH), "sha256": _sha256_file(RUNNER_PATH)},
+ "skill": {"path": _relative(SKILL_PATH), "sha256": _sha256_file(SKILL_PATH)},
+ },
+ "execution_authorized": False,
+ "execution_blocker": (
+ "Explicit provider authorization for the new C4/C5 public snapshots "
+ "and frozen treatment material has not been recorded."
+ ),
+ }
+ _write_json(FREEZE_PATH, freeze_payload)
+ validate()
+ return freeze_payload
+
+
+def validate_manifest() -> dict[str, Any]:
+ manifest = _load_json(MANIFEST_PATH)
+ if manifest.get("protocol") != PROTOCOL_ID:
+ raise GeneralizationError(f"manifest protocol must be {PROTOCOL_ID!r}")
+ tracked = {
+ "protocol_sha256": PROTOCOL_PATH,
+ "prompt_sha256": PROMPT_PATH,
+ "skill_sha256": SKILL_PATH,
+ }
+ for field, path in tracked.items():
+ if manifest.get(field) != _sha256_file(path):
+ raise GeneralizationError(f"{field} drifted after freeze")
+ for case in _cases(manifest):
+ for field in (
+ "source",
+ "snapshot_id",
+ "snapshot_sha256",
+ "revision",
+ "commands",
+ "declared_claim",
+ "excluded_scope",
+ "origin",
+ "gold_manifest",
+ "gold_sha256",
+ "dependency_cache",
+ "gold_evidence",
+ "status",
+ ):
+ if field not in case:
+ raise GeneralizationError(f"case {case['id']!r} is missing {field!r}")
+ source = _private_path(str(case["source"]), f"{case['id']} source")
+ if not source.is_dir() or (source / ".git").exists():
+ raise GeneralizationError(f"case {case['id']!r} source is unavailable or not stripped")
+ snapshot_sha256 = _snapshot_digest(source)
+ if snapshot_sha256 != case["snapshot_sha256"]:
+ raise GeneralizationError(f"case {case['id']!r} snapshot drifted")
+ if case["snapshot_id"] != f"sha256:{snapshot_sha256}":
+ raise GeneralizationError(f"case {case['id']!r} snapshot_id is inconsistent")
+ gold_path = _private_path(str(case["gold_manifest"]), f"{case['id']} gold")
+ if _sha256_file(gold_path) != case["gold_sha256"]:
+ raise GeneralizationError(f"case {case['id']!r} gold drifted")
+ _validate_tree(case, case["dependency_cache"], name="dependency-cache")
+ dependency = case["dependency_cache"]
+ if not isinstance(dependency, dict):
+ raise GeneralizationError(f"case {case['id']!r} dependency cache is invalid")
+ _dependency_target(dependency.get("target"))
+ evidence = case["gold_evidence"]
+ if not isinstance(evidence, list):
+ raise GeneralizationError(f"case {case['id']!r} gold_evidence must be a list")
+ names: set[str] = set()
+ for item in evidence:
+ if not isinstance(item, dict) or not isinstance(item.get("name"), str):
+ raise GeneralizationError("gold evidence entries require a name")
+ if item["name"] in names:
+ raise GeneralizationError(f"duplicate gold evidence name {item['name']!r}")
+ names.add(item["name"])
+ _validate_tree(case, item, name=item["name"])
+ return manifest
+
+
+def create_plan(manifest: Mapping[str, Any]) -> dict[str, Any]:
+ ready = [case for case in _cases(manifest) if case.get("status") == "ready"]
+ entries: list[dict[str, Any]] = []
+ for case in ready:
+ for agent in AGENTS:
+ for condition in CONDITIONS:
+ for repetition in range(1, REPETITIONS + 1):
+ entries.append(
+ {
+ "id": f"{condition}__{case['id']}__{agent}__r{repetition}",
+ "case_id": case["id"],
+ "agent": agent,
+ "condition": condition,
+ "repetition": repetition,
+ }
+ )
+ random.Random(RANDOMIZATION_SEED).shuffle(entries)
+ for index, entry in enumerate(entries, 1):
+ entry["order"] = index
+ return {
+ "protocol": PROTOCOL_ID,
+ "randomization_seed": RANDOMIZATION_SEED,
+ "repetitions": REPETITIONS,
+ "agents": list(AGENTS),
+ "conditions": list(CONDITIONS),
+ "entry_count": len(entries),
+ "entries": entries,
+ }
+
+
+def validate_plan(manifest: Mapping[str, Any]) -> dict[str, Any]:
+ plan = _load_json(PLAN_PATH)
+ expected = create_plan(manifest)
+ if plan != expected:
+ raise GeneralizationError("seeded v2 plan drifted")
+ return plan
+
+
+def validate() -> dict[str, Any]:
+ manifest = validate_manifest()
+ plan = validate_plan(manifest)
+ freeze_payload = _load_json(FREEZE_PATH)
+ if freeze_payload.get("protocol") != PROTOCOL_ID:
+ raise GeneralizationError(f"freeze protocol must be {PROTOCOL_ID!r}")
+ bindings = freeze_payload.get("bindings")
+ if not isinstance(bindings, dict):
+ raise GeneralizationError("freeze bindings must be an object")
+ expected_paths = {
+ "draft": DRAFT_PATH,
+ "manifest": MANIFEST_PATH,
+ "plan": PLAN_PATH,
+ "protocol": PROTOCOL_PATH,
+ "prompt": PROMPT_PATH,
+ "runner": RUNNER_PATH,
+ "skill": SKILL_PATH,
+ }
+ for name, path in expected_paths.items():
+ binding = bindings.get(name)
+ if not isinstance(binding, dict):
+ raise GeneralizationError(f"freeze lacks {name!r} binding")
+ if binding.get("path") != _relative(path) or binding.get("sha256") != _sha256_file(path):
+ raise GeneralizationError(f"freeze {name!r} binding drifted")
+ if freeze_payload.get("execution_authorized") is not False:
+ raise GeneralizationError("frozen cohort must remain execution-blocked")
+ return {
+ "protocol": PROTOCOL_ID,
+ "cases": len(_cases(manifest)),
+ "runs": plan["entry_count"],
+ "execution_authorized": False,
+ }
+
+
+def render_prompt(case: Mapping[str, Any], condition: str) -> str:
+ if condition not in CONDITIONS:
+ raise GeneralizationError(f"condition must be one of {CONDITIONS!r}")
+ prompt = PROMPT_PATH.read_text()
+ substitutions = {
+ "case_id": case["id"],
+ "snapshot_id": case["snapshot_id"],
+ "revision": case["revision"],
+ "commands": case["commands"],
+ "declared_claim": case["declared_claim"],
+ "excluded_scope": case["excluded_scope"],
+ }
+ for name, value in substitutions.items():
+ prompt = prompt.replace("{{" + name + "}}", str(value))
+ if "{{" in prompt or "}}" in prompt:
+ raise GeneralizationError("unresolved prompt placeholder")
+ return prompt + DOCS_TREATMENT
+
+
+def _parser() -> argparse.ArgumentParser:
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("command", choices=("freeze", "validate", "summary"))
+ return parser
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+ args = _parser().parse_args(argv)
+ try:
+ if args.command == "freeze":
+ payload = freeze()
+ elif args.command == "validate":
+ payload = validate()
+ else:
+ payload = validate()
+ payload["authorization_blocker"] = _load_json(FREEZE_PATH).get("execution_blocker")
+ print(json.dumps(payload, indent=2, sort_keys=True))
+ return 0
+ except GeneralizationError as exc:
+ print(f"Generalization experiment error: {exc}")
+ return 2
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/experiments/agent_qualification_generalization_prompt.txt b/experiments/agent_qualification_generalization_prompt.txt
new file mode 100644
index 0000000..0718fb6
--- /dev/null
+++ b/experiments/agent_qualification_generalization_prompt.txt
@@ -0,0 +1,59 @@
+You are the release-validation engineer for a small venue-infrastructure team
+that owns the matching logic in this repository.
+
+CASE: {{case_id}}
+REPOSITORY SNAPSHOT: {{snapshot_id}}
+PINNED REVISION: {{revision}}
+SUPPORTED BUILD/TEST COMMANDS: {{commands}}
+OWNER'S ONLY DECLARED GUARANTEE: {{declared_claim}}
+EXCLUDED SCOPE: {{excluded_scope}}
+
+Determine whether this pinned repository can produce safe release evidence for
+the declared matching semantics. Do not optimize for finding a bug. A correct
+result may be conformant behavior, a candidate defect, an adapter/protocol
+defect, a profile mismatch, insufficient observability, or an ambiguity that
+requires maintainer adjudication.
+
+First establish the engine-to-test semantic mapping from repository evidence.
+Do not silently invent lifecycle behavior, queue order, numeric conversions,
+identities, matching responsibility, or unsupported features. Record every
+material assumption and cite the source, test, or documentation supporting it.
+
+You may search for public development tools and their documentation, but do not
+search for this candidate repository, revision, issues, known failures, prior
+adapters, or experiment answers. Do not change production matching behavior.
+Evaluation-only adapters, probes, artifacts, and native tests are allowed.
+
+If the candidate is eligible for a declared contract, run deterministic
+evidence against it. If a disagreement appears, first rule out adapter,
+protocol, normalization, observation, and profile errors. Attribute a
+candidate defect only when intended behavior is independently established.
+
+Every terminal class requires two mechanically fresh evidence runs. Each run
+must start from a fresh candidate copy, fresh build/cache target, fresh process,
+and absent-before-use output path. Preserve both run identities and compare
+their deterministic results. Do not overwrite an incomplete bundle or call a
+second command over retained state a clean reproduction.
+
+Keep candidate clones, dependency-store copies, build products, tool state, and
+virtual environments under `FRESH_WORK_ROOT` or the supplied `FRESH_*` paths,
+never inside the candidate repository. The candidate repository should retain
+only source-sized audit artifacts and the requested final deliverables; do not
+copy dependency stores, toolchains, or compiled outputs into those artifacts.
+
+Deliver:
+
+1. `CLASSIFICATION.md`: terminal class, scope, evidence, first disagreement
+ when applicable, and remaining uncertainty.
+2. `MAPPING.md`: matching responsibility, event, numeric, identity, trade,
+ rejection, and ordered-state mappings with citations.
+3. `COMMANDS.log`: every material command, working directory, target, and exit
+ code.
+4. Applicable unmodified machine-readable qualification/failure artifacts.
+5. `REPRODUCTION.md`: both fresh run identities, preconditions, outputs, and
+ equality checks.
+6. A native regression only when candidate-defect intent is established.
+7. A concise maintainer handoff that makes no claim beyond the evidence.
+
+Do not commit, push, open an issue, or contact a maintainer. Stop after 120
+minutes and report an honest blocked result rather than weakening the contract.
diff --git a/experiments/agent_qualification_generalization_runs.py b/experiments/agent_qualification_generalization_runs.py
new file mode 100644
index 0000000..6e9c1bc
--- /dev/null
+++ b/experiments/agent_qualification_generalization_runs.py
@@ -0,0 +1,1208 @@
+"""Execute the separately frozen v2 agent-qualification run matrix.
+
+The primary v2 cohort is immutable. This companion freezes machine/provider
+bindings, verifies native skill delivery on synthetic material, enforces
+provider-specific authorization, and records technical evidence without
+grading semantic outcomes.
+"""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import os
+import platform
+import shutil
+import socket
+import subprocess
+import sys
+from contextlib import contextmanager
+from pathlib import Path
+from typing import Any, Iterator, Mapping, Sequence
+
+if __package__:
+ from . import agent_qualification as helpers
+ from . import agent_qualification_generalization as cohort
+ from . import agent_qualification_treatments as delivery
+else: # Support direct execution from the repository root.
+ sys.path.insert(0, str(Path(__file__).resolve().parent))
+ import agent_qualification as helpers # type: ignore[no-redef]
+ import agent_qualification_generalization as cohort # type: ignore[no-redef]
+ import agent_qualification_treatments as delivery # type: ignore[no-redef]
+
+ExecutionError = helpers.EvaluationError
+
+REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
+RUNNER_PATH = Path(__file__).resolve()
+TEST_PATH = REPOSITORY_ROOT / "tests" / "test_agent_qualification_generalization_runs.py"
+ADDENDUM_PATH = REPOSITORY_ROOT / "docs" / "agent-qualification-generalization-execution-v2.md"
+PRIVATE_ROOT = cohort.PRIVATE_ROOT
+EXECUTION_ROOT = PRIVATE_ROOT / "execution"
+RUNS_ROOT = EXECUTION_ROOT / "runs"
+AUTHORIZATION_ROOT = EXECUTION_ROOT / "authorizations"
+FREEZE_PATH = EXECUTION_ROOT / "freeze.json"
+SHAKEDOWN_PATH = EXECUTION_ROOT / "native-delivery-shakedown.json"
+CLAUDE_WRAPPER_PATH = (
+ EXECUTION_ROOT / "provider-wrappers" / "claude" / ".claude-plugin" / "plugin.json"
+)
+EXTERNAL_ROOT = Path(
+ os.environ.get("TRACEBOOK_V2_EXTERNAL_ROOT", "/private/tmp/tracebook-agent-qualification-v2")
+)
+
+PROTOCOL_ID = cohort.PROTOCOL_ID
+FREEZE_KIND = "agent-qualification-generalization-execution-freeze-v1"
+SHAKEDOWN_KIND = "agent-qualification-generalization-native-skill-shakedown-v1"
+VERDICT_KIND = "agent-qualification-generalization-pre-score-verdict-v1"
+AUTHORIZATION_KIND = "agent-qualification-generalization-provider-authorization-v1"
+SKILL_NAME = "tracebook-qualify-matching-engine"
+SYNTHETIC_SKILL_NAME = "native-delivery-probe"
+SYNTHETIC_MARKER = "NATIVE_SKILL_PROBE_V2_20260801_7F31C9"
+DEFAULT_TIMEOUT_SECONDS = 120 * 60
+EXPECTED_MODELS = {"codex": "gpt-5.6-sol", "claude": "claude-opus-4-8"}
+PROVIDERS = tuple(EXPECTED_MODELS)
+CLAUDE_FIRST_PARTY_ENV = {
+ "CLAUDE_CODE_USE_BEDROCK": "0",
+ "CLAUDE_CODE_USE_VERTEX": "0",
+ "CLAUDE_CODE_USE_FOUNDRY": "0",
+}
+RESTRICTED_SYSTEM_PATH = (
+ "/Library/Developer/CommandLineTools/usr/bin",
+ "/usr/local/bin",
+ "/opt/homebrew/bin",
+ "/usr/bin",
+ "/bin",
+ "/usr/sbin",
+ "/sbin",
+)
+
+_PLUGIN_MANIFEST = {
+ "author": {"name": "Tracebook evaluator"},
+ "description": "Isolated carrier for a preregistered agent evaluation treatment.",
+ "name": "agent-qualification-treatment",
+ "version": "2.0.0",
+}
+_SYNTHETIC_SKILL = (
+ "---\n"
+ "name: native-delivery-probe\n"
+ "description: Synthetic probe used only to verify native skill delivery.\n"
+ "---\n\n"
+ "When explicitly asked to run this probe, reply with exactly " + SYNTHETIC_MARKER + ".\n"
+).encode()
+
+_ORIGINAL_CODEX_SHELL_ENVIRONMENT = helpers._codex_shell_environment
+
+
+def _json_bytes(value: Mapping[str, Any]) -> bytes:
+ return (json.dumps(value, indent=2, sort_keys=True) + "\n").encode()
+
+
+def _read_json(path: Path, description: str) -> dict[str, Any]:
+ try:
+ value = json.loads(path.read_text())
+ except (OSError, json.JSONDecodeError) as exc:
+ raise ExecutionError(f"cannot read {description} from {path}: {exc}") from exc
+ if not isinstance(value, dict):
+ raise ExecutionError(f"{description} must contain one JSON object")
+ return value
+
+
+def _write_new_json(path: Path, value: Mapping[str, Any], description: str) -> None:
+ if path.exists() or path.is_symlink():
+ raise ExecutionError(f"refusing to overwrite {description} {path}")
+ path.parent.mkdir(parents=True, exist_ok=True)
+ path.write_bytes(_json_bytes(value))
+
+
+def _file_record(path: Path) -> dict[str, Any]:
+ resolved = path.resolve(strict=True)
+ if not resolved.is_file():
+ raise ExecutionError(f"required file is unavailable: {path}")
+ try:
+ label = resolved.relative_to(REPOSITORY_ROOT.resolve()).as_posix()
+ except ValueError:
+ label = str(resolved)
+ return {
+ "path": label,
+ "sha256": helpers._sha256_file(resolved),
+ "bytes": resolved.stat().st_size,
+ }
+
+
+def _validate_file_record(record: Any, description: str) -> Path:
+ if not isinstance(record, dict):
+ raise ExecutionError(f"{description} binding is missing")
+ raw_path = record.get("path")
+ if not isinstance(raw_path, str):
+ raise ExecutionError(f"{description} path is missing")
+ candidate = Path(raw_path)
+ path = candidate if candidate.is_absolute() else REPOSITORY_ROOT / candidate
+ if _file_record(path) != record:
+ raise ExecutionError(f"{description} binding drifted")
+ return path.resolve()
+
+
+def _provider_binary(provider: str) -> Path:
+ if provider not in PROVIDERS:
+ raise ExecutionError(f"unknown provider {provider!r}")
+ override = os.environ.get(f"TRACEBOOK_V2_{provider.upper()}_BINARY")
+ candidates = [
+ Path(override) if override else None,
+ Path(shutil.which(provider) or "") if shutil.which(provider) else None,
+ Path.home() / ".local" / "bin" / provider,
+ Path("/Applications/ChatGPT.app/Contents/Resources/codex") if provider == "codex" else None,
+ ]
+ for candidate in candidates:
+ if candidate is not None and candidate.is_file():
+ return candidate.resolve()
+ raise ExecutionError(f"{provider} CLI is unavailable")
+
+
+def _clean_probe_environment() -> dict[str, str]:
+ secret_fragments = ("KEY", "TOKEN", "SECRET", "PASSWORD", "CREDENTIAL")
+ return {
+ name: value
+ for name, value in os.environ.items()
+ if name not in helpers.SECRET_ENVIRONMENT_NAMES
+ and not any(fragment in name.upper() for fragment in secret_fragments)
+ }
+
+
+def _provider_binding(provider: str) -> dict[str, Any]:
+ binary = _provider_binary(provider)
+ version = helpers._binary_version(binary, _clean_probe_environment())
+ binding: dict[str, Any] = {
+ "configured_path": str(binary),
+ "resolved_path": str(binary.resolve()),
+ "sha256": helpers._sha256_file(binary),
+ "bytes": binary.stat().st_size,
+ "cli_version": version,
+ "requested_model": EXPECTED_MODELS[provider],
+ }
+ if provider == "claude":
+ route_settings = json.dumps(
+ {"env": CLAUDE_FIRST_PARTY_ENV},
+ sort_keys=True,
+ separators=(",", ":"),
+ )
+ completed = subprocess.run(
+ (
+ str(binary),
+ "--setting-sources",
+ "",
+ "--settings",
+ route_settings,
+ "auth",
+ "status",
+ ),
+ check=False,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT,
+ text=True,
+ timeout=30,
+ )
+ try:
+ auth = json.loads(completed.stdout)
+ except json.JSONDecodeError:
+ auth = {"raw_sha256": hashlib.sha256(completed.stdout.encode()).hexdigest()}
+ if isinstance(auth, dict):
+ # Do not persist account identifiers in the experiment archive.
+ binding["auth_route"] = {
+ key: auth.get(key)
+ for key in ("loggedIn", "authMethod", "apiProvider", "subscriptionType")
+ if key in auth
+ }
+ return binding
+
+
+def _validated_provider_bindings() -> dict[str, dict[str, Any]]:
+ bindings = {provider: _provider_binding(provider) for provider in PROVIDERS}
+ route = bindings["claude"].get("auth_route")
+ if (
+ not isinstance(route, dict)
+ or route.get("loggedIn") is not True
+ or route.get("apiProvider") != "firstParty"
+ ):
+ raise ExecutionError(
+ "Claude must use an authenticated first-party route before native-delivery "
+ "shakedown or execution freeze"
+ )
+ return bindings
+
+
+@contextmanager
+def _configured_delivery() -> Iterator[None]:
+ """Temporarily bind shared delivery helpers and always restore their globals."""
+
+ original_codex_binary = helpers.CODEX_BINARY
+ original_claude_binary = helpers.CLAUDE_BINARY
+ original_codex_auth_path = helpers.CODEX_AUTH_PATH
+ original_claude_wrapper_path = delivery.DEFAULT_CLAUDE_WRAPPER_PATH
+ original_expected_models = delivery.EXPECTED_MODELS
+ original_codex_shell_environment = helpers._codex_shell_environment
+ try:
+ helpers.CODEX_BINARY = _provider_binary("codex")
+ helpers.CLAUDE_BINARY = _provider_binary("claude")
+ helpers.CODEX_AUTH_PATH = Path.home() / ".codex" / "auth.json"
+ delivery.DEFAULT_CLAUDE_WRAPPER_PATH = CLAUDE_WRAPPER_PATH
+ delivery.EXPECTED_MODELS = dict(EXPECTED_MODELS)
+ helpers._codex_shell_environment = _codex_shell_environment
+ yield
+ finally:
+ helpers.CODEX_BINARY = original_codex_binary
+ helpers.CLAUDE_BINARY = original_claude_binary
+ helpers.CODEX_AUTH_PATH = original_codex_auth_path
+ delivery.DEFAULT_CLAUDE_WRAPPER_PATH = original_claude_wrapper_path
+ delivery.EXPECTED_MODELS = original_expected_models
+ helpers._codex_shell_environment = original_codex_shell_environment
+
+
+def _toolchain_read_paths() -> tuple[Path, ...]:
+ paths: list[Path] = []
+ for name in (
+ "TRACEBOOK_V2_MAVEN_HOME",
+ "TRACEBOOK_V2_JAVA_HOME",
+ "TRACEBOOK_V2_PYTHON_HOME",
+ ):
+ raw = os.environ.get(name)
+ if raw:
+ paths.append(Path(raw).resolve())
+ uv_binary = os.environ.get("TRACEBOOK_V2_UV_BINARY")
+ if uv_binary:
+ paths.append(Path(uv_binary).resolve().parent)
+ cargo_bin = Path.home() / ".cargo" / "bin"
+ if cargo_bin.is_dir():
+ paths.append(cargo_bin.resolve())
+ return tuple(dict.fromkeys(paths))
+
+
+def _toolchain_environment() -> dict[str, str]:
+ values: dict[str, str] = {}
+ path_prefixes: list[str] = []
+ for name in ("TRACEBOOK_V2_MAVEN_HOME", "TRACEBOOK_V2_JAVA_HOME"):
+ raw = os.environ.get(name)
+ if not raw:
+ continue
+ root = Path(raw).resolve()
+ if not root.is_dir():
+ raise ExecutionError(f"{name} is not an available directory: {root}")
+ path_prefixes.append(str(root / "bin"))
+ if name == "TRACEBOOK_V2_JAVA_HOME":
+ values["JAVA_HOME"] = str(root)
+ python_home = os.environ.get("TRACEBOOK_V2_PYTHON_HOME")
+ if python_home:
+ root = Path(python_home).resolve()
+ if not root.is_dir():
+ raise ExecutionError(f"TRACEBOOK_V2_PYTHON_HOME is not an available directory: {root}")
+ path_prefixes.append(str(root / "bin"))
+ uv_binary = os.environ.get("TRACEBOOK_V2_UV_BINARY")
+ if uv_binary:
+ path = Path(uv_binary).resolve()
+ if not path.is_file():
+ raise ExecutionError(f"TRACEBOOK_V2_UV_BINARY is not an available file: {path}")
+ path_prefixes.append(str(path.parent))
+ cargo_bin = Path.home() / ".cargo" / "bin"
+ if cargo_bin.is_dir():
+ path_prefixes.append(str(cargo_bin))
+ values["PATH"] = ":".join(dict.fromkeys([*path_prefixes, *RESTRICTED_SYSTEM_PATH]))
+ return values
+
+
+def _codex_shell_environment(scratch: Path) -> dict[str, str]:
+ values = _ORIGINAL_CODEX_SHELL_ENVIRONMENT(scratch)
+ values.update(_toolchain_environment())
+ values.update(_fresh_environment(scratch))
+ return values
+
+
+def _claude_settings(workspace: Path, scratch: Path, plugin_root: Path) -> dict[str, Any]:
+ settings = delivery._claude_treatment_settings(workspace, scratch, plugin_root)
+ settings["env"].update(CLAUDE_FIRST_PARTY_ENV)
+ settings["env"].update(_toolchain_environment())
+ settings["env"].update(_fresh_environment(scratch))
+ allow_read = settings["sandbox"]["filesystem"]["allowRead"]
+ for path in _toolchain_read_paths():
+ if str(path) not in allow_read:
+ allow_read.append(str(path))
+ return settings
+
+
+def _fresh_environment(scratch: Path) -> dict[str, str]:
+ return {
+ "FRESH_M2": str(scratch / "m2-repository"),
+ "FRESH_CARGO_HOME": str(scratch / "cargo-home"),
+ "FRESH_TARGET": str(scratch / "cargo-target"),
+ "FRESH_WORK_ROOT": str(scratch / "fresh-work"),
+ }
+
+
+def _write_wrapper() -> None:
+ if CLAUDE_WRAPPER_PATH.exists():
+ if _read_json(CLAUDE_WRAPPER_PATH, "Claude plugin wrapper") != _PLUGIN_MANIFEST:
+ raise ExecutionError("Claude plugin wrapper drifted")
+ return
+ CLAUDE_WRAPPER_PATH.parent.mkdir(parents=True, exist_ok=True)
+ CLAUDE_WRAPPER_PATH.write_bytes(_json_bytes(_PLUGIN_MANIFEST))
+
+
+def _case_index(manifest: Mapping[str, Any]) -> dict[str, dict[str, Any]]:
+ cases = manifest.get("cases")
+ if not isinstance(cases, list):
+ raise ExecutionError("validated cohort lost its cases")
+ return {
+ str(case["id"]): dict(case)
+ for case in cases
+ if isinstance(case, dict) and isinstance(case.get("id"), str)
+ }
+
+
+def _plan_entry(plan: Mapping[str, Any], run_id: str) -> dict[str, Any]:
+ entries = plan.get("entries")
+ if not isinstance(entries, list):
+ raise ExecutionError("validated cohort lost its plan entries")
+ matches = [entry for entry in entries if isinstance(entry, dict) and entry.get("id") == run_id]
+ if len(matches) != 1:
+ raise ExecutionError(f"run {run_id!r} is not one frozen plan entry")
+ return dict(matches[0])
+
+
+def _load_primary() -> tuple[dict[str, Any], dict[str, Any], dict[str, Any]]:
+ cohort.validate()
+ return (
+ cohort._load_json(cohort.MANIFEST_PATH),
+ cohort._load_json(cohort.PLAN_PATH),
+ cohort._load_json(cohort.FREEZE_PATH),
+ )
+
+
+def _authorization_path(provider: str) -> Path:
+ if provider not in PROVIDERS:
+ raise ExecutionError(f"unknown provider {provider!r}")
+ return AUTHORIZATION_ROOT / f"{provider}.json"
+
+
+def record_authorization(provider: str, statement_file: Path) -> dict[str, Any]:
+ """Record explicit consent already supplied by the user; never infer it."""
+
+ validate_freeze()
+ statement = statement_file.read_text().strip()
+ if not statement:
+ raise ExecutionError("authorization statement must not be empty")
+ payload = {
+ "kind": AUTHORIZATION_KIND,
+ "protocol": PROTOCOL_ID,
+ "provider": provider,
+ "case_ids": ["c4-compatible", "c5-boundary"],
+ "materials": [
+ "public repository snapshots",
+ "frozen prompt",
+ "docs-assisted treatment",
+ "skill-assisted treatment",
+ ],
+ "scope": "all-preregistered-runs-and-genuine-technical-restarts",
+ "user_statement": statement,
+ "statement_sha256": hashlib.sha256(statement.encode()).hexdigest(),
+ "recorded_at": helpers._utc_now(),
+ }
+ _write_new_json(_authorization_path(provider), payload, f"{provider} authorization")
+ return payload
+
+
+def validate_authorization(provider: str) -> dict[str, Any]:
+ payload = _read_json(_authorization_path(provider), f"{provider} authorization")
+ expected = {
+ "kind": AUTHORIZATION_KIND,
+ "protocol": PROTOCOL_ID,
+ "provider": provider,
+ "case_ids": ["c4-compatible", "c5-boundary"],
+ "materials": [
+ "public repository snapshots",
+ "frozen prompt",
+ "docs-assisted treatment",
+ "skill-assisted treatment",
+ ],
+ "scope": "all-preregistered-runs-and-genuine-technical-restarts",
+ }
+ for field, value in expected.items():
+ if payload.get(field) != value:
+ raise ExecutionError(f"{provider} authorization {field!r} is invalid")
+ statement = payload.get("user_statement")
+ if not isinstance(statement, str) or not statement.strip():
+ raise ExecutionError(f"{provider} authorization statement is missing")
+ if payload.get("statement_sha256") != hashlib.sha256(statement.encode()).hexdigest():
+ raise ExecutionError(f"{provider} authorization statement binding drifted")
+ if not isinstance(payload.get("recorded_at"), str):
+ raise ExecutionError(f"{provider} authorization timestamp is missing")
+ return payload
+
+
+def _toolchain_versions() -> dict[str, Any]:
+ result: dict[str, Any] = {}
+ environment = dict(os.environ)
+ environment.update(_toolchain_environment())
+ for name, command in {
+ "git": ("git", "--version"),
+ "python": (sys.executable, "--version"),
+ "java": ("java", "-version"),
+ "maven": ("mvn", "--version"),
+ "cargo": ("cargo", "--version"),
+ "rustc": ("rustc", "--version"),
+ }.items():
+ binary = (
+ shutil.which(command[0], path=environment["PATH"])
+ if not Path(command[0]).is_absolute()
+ else command[0]
+ )
+ if not binary:
+ result[name] = {"available": False}
+ continue
+ completed = subprocess.run(
+ (str(binary), *command[1:]),
+ check=False,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT,
+ text=True,
+ timeout=30,
+ env=environment,
+ )
+ result[name] = {
+ "available": completed.returncode == 0,
+ "path": str(Path(binary).resolve()),
+ "version": completed.stdout.strip().splitlines()[0] if completed.stdout.strip() else "",
+ }
+ return result
+
+
+def _native_surface(
+ *,
+ provider: str,
+ condition: str,
+ workspace: Path,
+ run_root: Path,
+ scratch: Path,
+ environment: dict[str, str],
+ skill: bytes,
+ skill_name: str,
+) -> tuple[list[str], dict[str, Any], Path | None]:
+ with _configured_delivery():
+ command, surface, plugin_root = delivery._prepare_native_surface(
+ agent=provider,
+ condition=condition,
+ workspace=workspace,
+ external_run_root=run_root,
+ scratch=scratch,
+ environment=environment,
+ skill_content=skill,
+ skill_name=skill_name,
+ )
+ if provider == "codex":
+ permission_index = next(
+ index
+ for index, value in enumerate(command)
+ if value.startswith("permissions.agent-eval=")
+ )
+ permission_profile = command[permission_index]
+ read_entries = "".join(
+ f'{json.dumps(str(path))}="read",'
+ for path in _toolchain_read_paths()
+ if f"{json.dumps(str(path))}=" not in permission_profile
+ )
+ command[permission_index] = permission_profile.replace(
+ "filesystem={",
+ "filesystem={" + read_entries,
+ 1,
+ )
+ return command, surface, plugin_root
+
+
+def _claude_command(settings_path: Path, plugin_root: Path) -> list[str]:
+ with _configured_delivery():
+ return delivery._claude_treatment_command(settings_path, plugin_root)
+
+
+def _run_delivery_probe(provider: str, condition: str, root: Path) -> dict[str, Any]:
+ workspace = root / "workspace"
+ scratch = root / "scratch"
+ workspace.mkdir(parents=True)
+ scratch.mkdir(parents=True)
+ (workspace / "README.md").write_text("Synthetic native skill delivery probe.\n")
+ environment = helpers._clean_agent_environment(scratch)
+ environment.update(_toolchain_environment())
+ command, surface, plugin_root = _native_surface(
+ provider=provider,
+ condition=condition,
+ workspace=workspace,
+ run_root=root,
+ scratch=scratch,
+ environment=environment,
+ skill=_SYNTHETIC_SKILL,
+ skill_name=SYNTHETIC_SKILL_NAME,
+ )
+ if provider == "claude":
+ if plugin_root is None:
+ raise ExecutionError("Claude probe plugin was not prepared")
+ settings_path = root / "claude-settings.json"
+ helpers._write_json(settings_path, _claude_settings(workspace, scratch, plugin_root))
+ command = _claude_command(settings_path, plugin_root)
+ transcript = root / f"{provider}.jsonl"
+ stderr = root / f"{provider}.stderr"
+ prompt = (
+ "Run the native-delivery-probe shakedown. If its native skill is available, "
+ "use it. If unavailable, reply exactly NO_NATIVE_DELIVERY. Do not search for "
+ "skill files or inspect any path not supplied by the native skill catalog."
+ )
+ exit_code, timed_out, elapsed = helpers._run_subprocess(
+ command,
+ cwd=workspace,
+ environment=environment,
+ prompt_stdin=prompt,
+ stdout_path=transcript,
+ stderr_path=stderr,
+ timeout_seconds=600,
+ )
+ audit = delivery._audit_native_surface(
+ agent=provider,
+ condition=condition,
+ external_run_root=root,
+ scratch=scratch,
+ skill_content=_SYNTHETIC_SKILL,
+ skill_name=SYNTHETIC_SKILL_NAME,
+ )
+ terminal = delivery._transcript_terminal_state(transcript, provider)
+ result = delivery._transcript_result_text(transcript, provider)
+ catalog_audit = None
+ if provider == "claude":
+ catalog_audit = delivery._claude_catalog_audit(
+ transcript,
+ condition=condition,
+ skill_name=SYNTHETIC_SKILL_NAME,
+ )
+ expected = SYNTHETIC_MARKER if condition == "skill" else "NO_NATIVE_DELIVERY"
+ errors = []
+ if exit_code != 0 or timed_out or terminal != "completed":
+ errors.append("provider did not complete cleanly")
+ if audit.get("mutated") is not False:
+ errors.append("native skill surface mutated")
+ if provider == "claude" and catalog_audit is not None and catalog_audit["valid"] is not True:
+ errors.append("Claude provider catalog isolation failed")
+ if result != expected:
+ errors.append(f"result {result!r} != {expected!r}")
+ return {
+ "valid": not errors,
+ "errors": errors,
+ "condition": condition,
+ "exit_code": exit_code,
+ "timed_out": timed_out,
+ "elapsed_seconds": elapsed,
+ "terminal_state": terminal,
+ "result": result,
+ "surface": audit,
+ "provider_catalog_audit": catalog_audit,
+ "transcript": _file_record(transcript),
+ "stderr": _file_record(stderr),
+ }
+
+
+def shakedown() -> dict[str, Any]:
+ cohort.validate()
+ _write_wrapper()
+ provider_bindings = _validated_provider_bindings()
+ if SHAKEDOWN_PATH.exists() or SHAKEDOWN_PATH.is_symlink():
+ raise ExecutionError(f"refusing to overwrite shakedown {SHAKEDOWN_PATH}")
+ root = EXECUTION_ROOT / "shakedown-work"
+ if root.exists() or root.is_symlink():
+ raise ExecutionError(f"refusing to reuse shakedown work root {root}")
+ providers: dict[str, Any] = {}
+ for provider in PROVIDERS:
+ conditions: dict[str, Any] = {}
+ for condition in ("docs", "skill"):
+ result = _run_delivery_probe(provider, condition, root / provider / condition)
+ if result["valid"] is not True:
+ raise ExecutionError(
+ f"{provider} {condition} native-delivery probe failed: "
+ + "; ".join(result["errors"])
+ )
+ conditions[condition] = result
+ providers[provider] = {"passed": True, "conditions": conditions}
+ verdict = {
+ "kind": SHAKEDOWN_KIND,
+ "protocol": PROTOCOL_ID,
+ "created_at": helpers._utc_now(),
+ "passed": True,
+ "runner": _file_record(RUNNER_PATH),
+ "synthetic_skill_sha256": hashlib.sha256(_SYNTHETIC_SKILL).hexdigest(),
+ "marker": SYNTHETIC_MARKER,
+ "provider_bindings": provider_bindings,
+ "providers": providers,
+ }
+ _write_new_json(SHAKEDOWN_PATH, verdict, "native-delivery shakedown")
+ return verdict
+
+
+def _validate_shakedown() -> dict[str, Any]:
+ verdict = _read_json(SHAKEDOWN_PATH, "native-delivery shakedown")
+ if verdict.get("kind") != SHAKEDOWN_KIND or verdict.get("protocol") != PROTOCOL_ID:
+ raise ExecutionError("native-delivery shakedown identity changed")
+ if verdict.get("passed") is not True or verdict.get("runner") != _file_record(RUNNER_PATH):
+ raise ExecutionError("native-delivery shakedown binding changed")
+ if verdict.get("synthetic_skill_sha256") != hashlib.sha256(_SYNTHETIC_SKILL).hexdigest():
+ raise ExecutionError("native-delivery synthetic skill changed")
+ if verdict.get("marker") != SYNTHETIC_MARKER:
+ raise ExecutionError("native-delivery marker changed")
+ if verdict.get("provider_bindings") != {
+ provider: _provider_binding(provider) for provider in PROVIDERS
+ }:
+ raise ExecutionError("native-delivery provider bindings changed")
+ providers = verdict.get("providers")
+ if not isinstance(providers, dict) or set(providers) != set(PROVIDERS):
+ raise ExecutionError("native-delivery provider coverage changed")
+ for provider in PROVIDERS:
+ conditions = (
+ providers[provider].get("conditions") if isinstance(providers[provider], dict) else None
+ )
+ if not isinstance(conditions, dict) or set(conditions) != {"docs", "skill"}:
+ raise ExecutionError(f"{provider} shakedown conditions changed")
+ if any(
+ result.get("valid") is not True
+ for result in conditions.values()
+ if isinstance(result, dict)
+ ):
+ raise ExecutionError(f"{provider} shakedown did not pass")
+ for result in conditions.values():
+ if not isinstance(result, dict):
+ raise ExecutionError(f"{provider} shakedown result is invalid")
+ _validate_file_record(result.get("transcript"), f"{provider} transcript")
+ _validate_file_record(result.get("stderr"), f"{provider} stderr")
+ return verdict
+
+
+def _freeze_payload(created_at: str) -> dict[str, Any]:
+ manifest, plan, primary_freeze = _load_primary()
+ del manifest, plan, primary_freeze
+ shakedown_verdict = _validate_shakedown()
+ inputs = {
+ "execution_addendum": _file_record(ADDENDUM_PATH),
+ "execution_runner": _file_record(RUNNER_PATH),
+ "execution_runner_tests": _file_record(TEST_PATH),
+ "helper_runner": _file_record(helpers.RUNNER_PATH),
+ "delivery_runner": _file_record(delivery.RUNNER_PATH),
+ "primary_protocol": _file_record(cohort.PROTOCOL_PATH),
+ "primary_prompt": _file_record(cohort.PROMPT_PATH),
+ "primary_runner": _file_record(cohort.RUNNER_PATH),
+ "primary_manifest": _file_record(cohort.MANIFEST_PATH),
+ "primary_plan": _file_record(cohort.PLAN_PATH),
+ "primary_freeze": _file_record(cohort.FREEZE_PATH),
+ "skill": _file_record(cohort.SKILL_PATH),
+ "claude_plugin_wrapper": _file_record(CLAUDE_WRAPPER_PATH),
+ "native_delivery_shakedown": _file_record(SHAKEDOWN_PATH),
+ }
+ return {
+ "kind": FREEZE_KIND,
+ "protocol": PROTOCOL_ID,
+ "created_at": created_at,
+ "host": {
+ "hostname": socket.gethostname(),
+ "platform": platform.platform(),
+ "machine": platform.machine(),
+ },
+ "inputs": inputs,
+ "provider_bindings": shakedown_verdict["provider_bindings"],
+ "toolchains": _toolchain_versions(),
+ "run_count": 24,
+ "order": "exact seeded primary plan order",
+ "authorization": "provider-specific external records required after freeze",
+ "restart_policy": (
+ "archive and retry only genuine technical interruptions without a completed "
+ "provider turn; never retry a timeout, refusal, or semantic outcome"
+ ),
+ }
+
+
+def freeze_execution() -> dict[str, Any]:
+ if FREEZE_PATH.exists() or FREEZE_PATH.is_symlink():
+ raise ExecutionError(f"refusing to overwrite execution freeze {FREEZE_PATH}")
+ payload = _freeze_payload(helpers._utc_now())
+ _write_new_json(FREEZE_PATH, payload, "execution freeze")
+ return payload
+
+
+def validate_freeze() -> dict[str, Any]:
+ cohort.validate()
+ payload = _read_json(FREEZE_PATH, "execution freeze")
+ if payload.get("kind") != FREEZE_KIND or payload.get("protocol") != PROTOCOL_ID:
+ raise ExecutionError("execution freeze identity changed")
+ created_at = payload.get("created_at")
+ if not isinstance(created_at, str) or not created_at:
+ raise ExecutionError("execution freeze timestamp is missing")
+ if payload != _freeze_payload(created_at):
+ raise ExecutionError("execution freeze content changed")
+ return payload
+
+
+def _copy_fixture(case: Mapping[str, Any], scratch: Path) -> dict[str, Any]:
+ declaration = case.get("dependency_cache")
+ if not isinstance(declaration, dict):
+ raise ExecutionError("case dependency-cache declaration is missing")
+ source = (REPOSITORY_ROOT / str(declaration["source"])).resolve(strict=True)
+ target_name = cohort._dependency_target(declaration.get("target"))
+ target = scratch / target_name
+ helpers._copy_frozen_tree(source, target, str(declaration["tree_sha256"]))
+ if target_name == "m2-seed":
+ shutil.copytree(target, scratch / "m2-repository")
+ elif target_name == "cargo-vendor":
+ cargo_home = scratch / "cargo-home"
+ cargo_home.mkdir(exist_ok=True)
+ quoted_target = json.dumps(str(target))
+ (cargo_home / "config.toml").write_text(
+ "[source.crates-io]\n"
+ 'replace-with = "vendored-sources"\n\n'
+ "[source.vendored-sources]\n"
+ f"directory = {quoted_target}\n\n"
+ "[net]\n"
+ "offline = true\n"
+ )
+ return {
+ "source_manifest_sha256": declaration["manifest_sha256"],
+ "initial_tree_sha256": declaration["tree_sha256"],
+ "target": str(target),
+ }
+
+
+def _record_fixture_final(fixture: dict[str, Any]) -> None:
+ target = Path(str(fixture["target"]))
+ if not target.is_dir():
+ fixture["final_state"] = "missing"
+ fixture["mutated"] = True
+ return
+ try:
+ final_hash = helpers._snapshot_digest(target)
+ except OSError as exc:
+ fixture["final_state"] = f"unreadable:{type(exc).__name__}"
+ fixture["mutated"] = True
+ return
+ fixture["final_state"] = "present"
+ fixture["final_tree_sha256"] = final_hash
+ fixture["mutated"] = final_hash != fixture["initial_tree_sha256"]
+
+
+def _run_root(run_id: str) -> Path:
+ return RUNS_ROOT / run_id
+
+
+def _external_run_root(run_id: str) -> Path:
+ return EXTERNAL_ROOT / run_id
+
+
+def _technical_verdict(
+ run_root: Path, metadata: Mapping[str, Any], errors: list[str]
+) -> dict[str, Any]:
+ verdict = {
+ "kind": VERDICT_KIND,
+ "protocol": PROTOCOL_ID,
+ "run_id": metadata.get("run_id"),
+ "valid": not errors,
+ "errors": errors,
+ "terminal_state": metadata.get("terminal_state"),
+ "timed_out": metadata.get("timed_out"),
+ "exit_code": metadata.get("exit_code"),
+ "metadata_sha256": helpers._sha256_file(run_root / "metadata.json"),
+ "transcript_sha256": metadata.get("transcript_sha256"),
+ "created_at": helpers._utc_now(),
+ }
+ _write_new_json(run_root / "pre-score-verdict.json", verdict, "pre-score verdict")
+ return verdict
+
+
+def validate_run(
+ run_id: str,
+ *,
+ freeze: Mapping[str, Any] | None = None,
+ primary: tuple[Mapping[str, Any], Mapping[str, Any]] | None = None,
+) -> dict[str, Any]:
+ if freeze is None:
+ freeze = validate_freeze()
+ if primary is None:
+ loaded_manifest, loaded_plan, _ = _load_primary()
+ manifest: Mapping[str, Any] = loaded_manifest
+ plan: Mapping[str, Any] = loaded_plan
+ else:
+ manifest, plan = primary
+ entry = _plan_entry(plan, run_id)
+ run_root = _run_root(run_id)
+ metadata = _read_json(run_root / "metadata.json", "run metadata")
+ verdict = _read_json(run_root / "pre-score-verdict.json", "pre-score verdict")
+ errors: list[str] = []
+ expected = {
+ "protocol": PROTOCOL_ID,
+ "run_id": run_id,
+ "case_id": entry["case_id"],
+ "provider": entry["agent"],
+ "condition": entry["condition"],
+ "repetition": entry["repetition"],
+ "order": entry["order"],
+ "primary_manifest_sha256": helpers._sha256_file(cohort.MANIFEST_PATH),
+ "primary_plan_sha256": helpers._sha256_file(cohort.PLAN_PATH),
+ "primary_freeze_sha256": helpers._sha256_file(cohort.FREEZE_PATH),
+ "execution_freeze_sha256": helpers._sha256_file(FREEZE_PATH),
+ "execution_runner_sha256": helpers._sha256_file(RUNNER_PATH),
+ }
+ for field, value in expected.items():
+ if metadata.get(field) != value:
+ errors.append(f"metadata {field} binding changed")
+ case = _case_index(manifest)[str(entry["case_id"])]
+ if metadata.get("snapshot_sha256") != case["snapshot_sha256"]:
+ errors.append("candidate snapshot binding changed")
+ if metadata.get("initial_workspace_sha256") != case["snapshot_sha256"]:
+ errors.append("initial workspace binding changed")
+ if metadata.get("frozen_skill_sha256") != helpers._sha256_file(cohort.SKILL_PATH):
+ errors.append("frozen skill metadata binding changed")
+ transcript = run_root / f"{entry['agent']}.jsonl"
+ stderr = run_root / f"{entry['agent']}.stderr"
+ prompt = run_root / "prompt.txt"
+ for artifact in (transcript, stderr, prompt):
+ if not artifact.is_file() or artifact.is_symlink():
+ errors.append(f"missing required artifact {artifact.name}")
+ if transcript.is_file() and metadata.get("transcript_sha256") != helpers._sha256_file(
+ transcript
+ ):
+ errors.append("transcript hash changed")
+ if stderr.is_file() and metadata.get("stderr_sha256") != helpers._sha256_file(stderr):
+ errors.append("stderr hash changed")
+ rendered = cohort.render_prompt(case, str(entry["condition"]))
+ rendered_hash = hashlib.sha256(rendered.encode()).hexdigest()
+ if metadata.get("rendered_prompt_sha256") != rendered_hash:
+ errors.append("rendered prompt metadata binding changed")
+ if prompt.is_file() and helpers._sha256_file(prompt) != rendered_hash:
+ errors.append("rendered prompt artifact changed")
+ frozen_provider = freeze["provider_bindings"][entry["agent"]]
+ if metadata.get("provider_binding") != frozen_provider:
+ errors.append("provider binding changed")
+ if metadata.get("authorization_sha256") != helpers._sha256_file(
+ _authorization_path(str(entry["agent"]))
+ ):
+ errors.append("provider authorization binding changed")
+ if metadata.get("gold_manifest_sha256") != case["gold_sha256"]:
+ errors.append("gold manifest binding changed")
+ if metadata.get("model_identifier") != EXPECTED_MODELS[str(entry["agent"])]:
+ errors.append("reported model identifier changed")
+ if metadata.get("exit_code") != 0:
+ errors.append("provider subprocess exit code changed")
+ if metadata.get("terminal_state") != "completed" or metadata.get("timed_out") is not False:
+ errors.append("provider did not complete a valid technical turn")
+ surface = metadata.get("native_surface")
+ expected_skill = entry["condition"] == "skill"
+ if not isinstance(surface, dict) or surface.get("skill_installed") is not expected_skill:
+ errors.append("native skill surface condition changed")
+ else:
+ expected_skill_hash = helpers._sha256_file(cohort.SKILL_PATH) if expected_skill else None
+ if surface.get("semantic_file_count") != (1 if expected_skill else 0):
+ errors.append("native skill semantic file count changed")
+ if surface.get("skill_sha256") != expected_skill_hash:
+ errors.append("native skill hash changed")
+ if (
+ not isinstance(surface.get("final_audit"), dict)
+ or surface["final_audit"].get("mutated") is not False
+ ):
+ errors.append("native skill surface audit failed")
+ if entry["agent"] == "claude":
+ catalog_audit = surface.get("provider_catalog_audit")
+ if not isinstance(catalog_audit, dict) or catalog_audit.get("valid") is not True:
+ errors.append("Claude provider catalog audit failed")
+ fixture = metadata.get("fixture")
+ if not isinstance(fixture, dict):
+ errors.append("dependency fixture audit is missing")
+ else:
+ if fixture.get("mutated") is not False:
+ errors.append("dependency fixture mutated")
+ if fixture.get("final_tree_sha256") != fixture.get("initial_tree_sha256"):
+ errors.append("dependency fixture final hash changed")
+ for artifact_name in (
+ "git-status.txt",
+ "workspace.patch",
+ "workspace-files.json",
+ "workspace-final",
+ ):
+ if not (run_root / artifact_name).exists():
+ errors.append(f"missing required artifact {artifact_name}")
+ if verdict.get("kind") != VERDICT_KIND or verdict.get("run_id") != run_id:
+ errors.append("pre-score verdict identity changed")
+ if verdict.get("metadata_sha256") != helpers._sha256_file(run_root / "metadata.json"):
+ errors.append("pre-score verdict metadata binding changed")
+ if verdict.get("transcript_sha256") != metadata.get("transcript_sha256"):
+ errors.append("pre-score verdict transcript binding changed")
+ if verdict.get("valid") is not True or verdict.get("errors") != []:
+ errors.append("pre-score verdict is not valid")
+ if errors:
+ raise ExecutionError("; ".join(errors))
+ return verdict
+
+
+def _validate_prior_runs(
+ plan: Mapping[str, Any],
+ order: int,
+ *,
+ freeze: Mapping[str, Any] | None = None,
+ primary: tuple[Mapping[str, Any], Mapping[str, Any]] | None = None,
+) -> None:
+ entries = plan.get("entries")
+ if not isinstance(entries, list):
+ raise ExecutionError("validated cohort lost its plan entries")
+ for prior in entries[: order - 1]:
+ if not isinstance(prior, dict):
+ raise ExecutionError("plan entry shape changed")
+ if freeze is None or primary is None:
+ validate_run(str(prior["id"]))
+ else:
+ validate_run(str(prior["id"]), freeze=freeze, primary=primary)
+
+
+def _execute_run_once(
+ run_id: str, timeout_seconds: int = DEFAULT_TIMEOUT_SECONDS
+) -> dict[str, Any]:
+ if timeout_seconds != DEFAULT_TIMEOUT_SECONDS:
+ raise ExecutionError(f"official timeout must remain {DEFAULT_TIMEOUT_SECONDS} seconds")
+ freeze = validate_freeze()
+ manifest, plan, _ = _load_primary()
+ entry = _plan_entry(plan, run_id)
+ provider = str(entry["agent"])
+ validate_authorization(provider)
+ _validate_prior_runs(
+ plan,
+ int(entry["order"]),
+ freeze=freeze,
+ primary=(manifest, plan),
+ )
+ case = _case_index(manifest)[str(entry["case_id"])]
+ run_root = _run_root(run_id)
+ external_root = _external_run_root(run_id)
+ if run_root.exists() or run_root.is_symlink():
+ raise ExecutionError(f"refusing to overwrite existing run {run_root}")
+ if external_root.exists() or external_root.is_symlink():
+ raise ExecutionError(f"refusing to reuse external workspace {external_root}")
+
+ source = (REPOSITORY_ROOT / str(case["source"])).resolve(strict=True)
+ workspace = external_root / "workspace"
+ scratch = external_root / "scratch"
+ run_root.mkdir(parents=True)
+ scratch.mkdir(parents=True)
+ helpers._initialize_snapshot(source, workspace)
+ initial_workspace_hash = helpers._snapshot_digest(workspace)
+ if initial_workspace_hash != case["snapshot_sha256"]:
+ raise ExecutionError("fresh workspace changed the frozen candidate snapshot")
+ for relative in (".tracebook", ".eval-cache", "target"):
+ if (workspace / relative).exists() or (workspace / relative).is_symlink():
+ raise ExecutionError(f"fresh workspace unexpectedly contains {relative}")
+ prompt = cohort.render_prompt(case, str(entry["condition"]))
+ (run_root / "prompt.txt").write_text(prompt)
+ environment = helpers._clean_agent_environment(scratch)
+ environment.update(_toolchain_environment())
+ fixture = _copy_fixture(case, scratch)
+ environment.update(_fresh_environment(scratch))
+ Path(environment["FRESH_TARGET"]).mkdir(exist_ok=True)
+ skill = cohort.SKILL_PATH.read_bytes()
+ command, surface, plugin_root = _native_surface(
+ provider=provider,
+ condition=str(entry["condition"]),
+ workspace=workspace,
+ run_root=external_root,
+ scratch=scratch,
+ environment=environment,
+ skill=skill,
+ skill_name=SKILL_NAME,
+ )
+ if provider == "claude":
+ if plugin_root is None:
+ raise ExecutionError("Claude plugin was not prepared")
+ settings_path = run_root / "claude-settings.json"
+ helpers._write_json(settings_path, _claude_settings(workspace, scratch, plugin_root))
+ command = _claude_command(settings_path, plugin_root)
+
+ transcript = run_root / f"{provider}.jsonl"
+ stderr = run_root / f"{provider}.stderr"
+ binary = _provider_binary(provider)
+ metadata: dict[str, Any] = {
+ "protocol": PROTOCOL_ID,
+ "run_id": run_id,
+ "case_id": entry["case_id"],
+ "provider": provider,
+ "condition": entry["condition"],
+ "repetition": entry["repetition"],
+ "order": entry["order"],
+ "started_at": helpers._utc_now(),
+ "timeout_seconds": timeout_seconds,
+ "primary_manifest_sha256": helpers._sha256_file(cohort.MANIFEST_PATH),
+ "primary_plan_sha256": helpers._sha256_file(cohort.PLAN_PATH),
+ "primary_freeze_sha256": helpers._sha256_file(cohort.FREEZE_PATH),
+ "execution_freeze_sha256": helpers._sha256_file(FREEZE_PATH),
+ "execution_runner_sha256": helpers._sha256_file(RUNNER_PATH),
+ "snapshot_sha256": case["snapshot_sha256"],
+ "initial_workspace_sha256": initial_workspace_hash,
+ "gold_manifest_sha256": case["gold_sha256"],
+ "rendered_prompt_sha256": hashlib.sha256(prompt.encode()).hexdigest(),
+ "authorization_sha256": helpers._sha256_file(_authorization_path(provider)),
+ "provider_binding": freeze["provider_bindings"][provider],
+ "command": command,
+ "requested_model": EXPECTED_MODELS[provider],
+ "cli_version": helpers._binary_version(binary, environment),
+ "binary_sha256": helpers._sha256_file(binary),
+ "prompt_transport": "stdin",
+ "fixture": fixture,
+ "native_surface": surface,
+ "skill_injected": entry["condition"] == "skill",
+ "frozen_skill_sha256": helpers._sha256_file(cohort.SKILL_PATH),
+ }
+ helpers._write_json(run_root / "metadata.json", metadata)
+ exit_code, timed_out, elapsed = helpers._run_subprocess(
+ command,
+ cwd=workspace,
+ environment=environment,
+ prompt_stdin=prompt,
+ stdout_path=transcript,
+ stderr_path=stderr,
+ timeout_seconds=timeout_seconds,
+ )
+ surface["final_audit"] = delivery._audit_native_surface(
+ agent=provider,
+ condition=str(entry["condition"]),
+ external_run_root=external_root,
+ scratch=scratch,
+ skill_content=skill,
+ skill_name=SKILL_NAME,
+ )
+ helpers._write_workspace_evidence(workspace, run_root)
+ _record_fixture_final(fixture)
+ shutil.copytree(
+ workspace,
+ run_root / "workspace-final",
+ symlinks=True,
+ ignore=shutil.ignore_patterns(".git", "target", "node_modules", ".eval-cache"),
+ )
+ terminal = delivery._transcript_terminal_state(transcript, provider)
+ if provider == "claude":
+ surface["provider_catalog_audit"] = delivery._claude_catalog_audit(
+ transcript,
+ condition=str(entry["condition"]),
+ skill_name=SKILL_NAME,
+ )
+ model, model_source = helpers._reported_model(transcript, provider)
+ metadata.update(
+ {
+ "completed_at": helpers._utc_now(),
+ "exit_code": exit_code,
+ "timed_out": timed_out,
+ "elapsed_seconds": elapsed,
+ "terminal_state": terminal,
+ "model_identifier": model,
+ "model_identifier_source": model_source,
+ "transcript_sha256": helpers._sha256_file(transcript),
+ "stderr_sha256": helpers._sha256_file(stderr),
+ "fixture": fixture,
+ "native_surface": surface,
+ }
+ )
+ helpers._write_json(run_root / "metadata.json", metadata)
+ errors: list[str] = []
+ if timed_out:
+ errors.append("provider subprocess timed out")
+ if terminal != "completed":
+ errors.append(f"provider terminal state was {terminal!r}")
+ if exit_code != 0:
+ errors.append(f"provider exit code was {exit_code}")
+ if model != EXPECTED_MODELS[provider]:
+ errors.append(f"reported model {model!r} != {EXPECTED_MODELS[provider]!r}")
+ if surface["final_audit"].get("mutated") is not False:
+ errors.append("native skill surface mutated")
+ if provider == "claude" and surface.get("provider_catalog_audit", {}).get("valid") is not True:
+ errors.append("Claude provider catalog isolation failed")
+ if fixture.get("mutated") is not False:
+ errors.append("dependency fixture mutated")
+ verdict = _technical_verdict(run_root, metadata, errors)
+ if errors:
+ raise ExecutionError(f"run {run_id!r} is technically invalid: " + "; ".join(errors))
+ validate_run(run_id)
+ shutil.rmtree(external_root)
+ return {"metadata": metadata, "verdict": verdict}
+
+
+def execute_run(run_id: str, timeout_seconds: int = DEFAULT_TIMEOUT_SECONDS) -> dict[str, Any]:
+ run_root = _run_root(run_id)
+ external_root = _external_run_root(run_id)
+ run_root_preexisting = run_root.exists() or run_root.is_symlink()
+ external_root_preexisting = external_root.exists() or external_root.is_symlink()
+ try:
+ return _execute_run_once(run_id, timeout_seconds)
+ except BaseException:
+ if not run_root_preexisting and not external_root_preexisting:
+ helpers._quarantine_interrupted_run(run_root, external_root)
+ raise
+
+
+def status() -> dict[str, Any]:
+ cohort_summary = cohort.validate()
+ _, plan, _ = _load_primary()
+ completed = []
+ for entry in plan["entries"]:
+ run_id = str(entry["id"])
+ if (_run_root(run_id) / "pre-score-verdict.json").is_file():
+ try:
+ validate_run(run_id)
+ except ExecutionError:
+ continue
+ completed.append(run_id)
+ authorizations = {provider: _authorization_path(provider).is_file() for provider in PROVIDERS}
+ next_run = next(
+ (entry["id"] for entry in plan["entries"] if entry["id"] not in completed), None
+ )
+ return {
+ **cohort_summary,
+ "execution_frozen": FREEZE_PATH.is_file(),
+ "authorizations_recorded": authorizations,
+ "completed_valid_runs": len(completed),
+ "next_run": next_run,
+ }
+
+
+def _parser() -> argparse.ArgumentParser:
+ parser = argparse.ArgumentParser(description=__doc__)
+ commands = parser.add_subparsers(dest="command", required=True)
+ commands.add_parser("shakedown")
+ commands.add_parser("freeze")
+ commands.add_parser("validate")
+ commands.add_parser("status")
+ authorize = commands.add_parser("authorize")
+ authorize.add_argument("--provider", required=True, choices=PROVIDERS)
+ authorize.add_argument("--statement-file", required=True, type=Path)
+ run = commands.add_parser("run")
+ run.add_argument("--run-id", required=True)
+ run.add_argument("--timeout-seconds", type=int, default=DEFAULT_TIMEOUT_SECONDS)
+ validate_run_parser = commands.add_parser("validate-run")
+ validate_run_parser.add_argument("--run-id", required=True)
+ return parser
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+ args = _parser().parse_args(argv)
+ try:
+ if args.command == "shakedown":
+ payload = shakedown()
+ elif args.command == "freeze":
+ payload = freeze_execution()
+ elif args.command == "validate":
+ payload = validate_freeze()
+ elif args.command == "authorize":
+ payload = record_authorization(args.provider, args.statement_file)
+ elif args.command == "run":
+ payload = execute_run(args.run_id, args.timeout_seconds)
+ elif args.command == "validate-run":
+ payload = validate_run(args.run_id)
+ else:
+ payload = status()
+ except (ExecutionError, OSError, ValueError, subprocess.SubprocessError) as exc:
+ print(f"Execution experiment error: {exc}", file=sys.stderr)
+ return 2
+ print(json.dumps(payload, indent=2, sort_keys=True))
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/experiments/agent_qualification_prompt.txt b/experiments/agent_qualification_prompt.txt
new file mode 100644
index 0000000..419565d
--- /dev/null
+++ b/experiments/agent_qualification_prompt.txt
@@ -0,0 +1,51 @@
+You are the release-validation engineer for a small venue-infrastructure team
+that owns the matching logic in this repository.
+
+CASE: {{case_id}}
+REPOSITORY SNAPSHOT: {{snapshot_id}}
+PINNED REVISION: {{revision}}
+SUPPORTED BUILD/TEST COMMANDS: {{commands}}
+OWNER'S ONLY DECLARED GUARANTEE: {{declared_claim}}
+EXCLUDED SCOPE: {{excluded_scope}}
+
+Determine whether this pinned engine preserves the declared matching semantics.
+Do not optimize for finding a bug. A correct result may be:
+
+- conformant behavior;
+- a candidate-engine defect;
+- an adapter or observation defect;
+- a profile/contract mismatch;
+- insufficient observability; or
+- an ambiguity requiring maintainer adjudication.
+
+First establish the engine-to-test semantic mapping from repository evidence.
+Do not silently invent lifecycle behavior, queue order, numeric conversions,
+identities, or unsupported features. Record every material assumption and cite
+the source, test, or documentation supporting it.
+
+Use any public development tool you can discover if it is appropriate. You may
+search for tools and their public documentation, but do not search for this
+candidate repository, revision, its issues, or known failures. Do not change
+production matching behavior. Evaluation-only adapters, artifacts, and native
+tests are allowed.
+
+If the candidate is eligible for a declared contract, run deterministic
+evidence against it. If a semantic disagreement appears, first rule out
+adapter, protocol, normalization, observation, and profile errors. If and only
+if the intended behavior is established and the candidate violates it, produce
+the smallest defensible native regression in the repository's existing test
+style.
+
+Deliver:
+
+1. CLASSIFICATION.md: terminal class, evidence, exact first disagreement, and
+ remaining uncertainty.
+2. MAPPING.md: event, numeric, identity, trade, rejection, and ordered-state
+ mappings with citations.
+3. COMMANDS.log: every material command and exit code.
+4. Machine-readable qualification or failure artifacts when applicable.
+5. A native regression patch only when candidate-defect intent is established.
+6. A concise maintainer handoff that makes no claim beyond the evidence.
+
+Do not commit, push, open an issue, or contact a maintainer. Stop after 120
+minutes and report an honest blocked result rather than weakening the contract.
diff --git a/experiments/agent_qualification_treatments.py b/experiments/agent_qualification_treatments.py
new file mode 100644
index 0000000..a54cd92
--- /dev/null
+++ b/experiments/agent_qualification_treatments.py
@@ -0,0 +1,1739 @@
+"""Frozen docs/skill treatment runner for the agent qualification evaluation.
+
+The baseline runner is a scored, hash-pinned input and must remain unchanged.
+This companion imports its isolation and evidence helpers while adding only the
+pre-registered docs and native-skill treatment surfaces.
+"""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import os
+import random
+import re
+import secrets
+import shlex
+import shutil
+import stat
+import subprocess
+import sys
+from pathlib import Path
+from typing import Any, Mapping, Sequence
+from urllib.parse import urlparse
+
+if __package__:
+ from . import agent_qualification as baseline
+else: # Support ``python experiments/agent_qualification_treatments.py``.
+ sys.path.insert(0, str(Path(__file__).resolve().parent))
+ import agent_qualification as baseline # type: ignore[no-redef]
+
+EvaluationError = baseline.EvaluationError
+
+REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
+RUNNER_PATH = Path(__file__).resolve()
+PRIVATE_ROOT = baseline.PRIVATE_ROOT
+TREATMENT_ROOT = PRIVATE_ROOT / "treatments"
+DEFAULT_SKILL_PATH = TREATMENT_ROOT / "tracebook-qualify-matching-engine" / "SKILL.md"
+DEFAULT_EXTERNAL_PINS_PATH = TREATMENT_ROOT / "EXTERNAL_PINS.json"
+DEFAULT_PROTOCOL_AMENDMENT_PATH = TREATMENT_ROOT / "TREATMENT_PROTOCOL.md"
+DEFAULT_AUTHORIZATION_PATH = TREATMENT_ROOT / "AUTHORIZATION.json"
+DEFAULT_CLAUDE_WRAPPER_PATH = (
+ TREATMENT_ROOT / "provider-wrappers" / "claude" / ".claude-plugin" / "plugin.json"
+)
+DEFAULT_PLAN_PATH = TREATMENT_ROOT / "master-plan.json"
+DEFAULT_FREEZE_PATH = TREATMENT_ROOT / "freeze.json"
+DEFAULT_SHAKEDOWN_VERDICT_PATH = TREATMENT_ROOT / "native-delivery-shakedown.json"
+DEFAULT_BASELINE_PLAN_PATH = baseline.DEFAULT_PLAN_PATH
+EVALUATION_ROOT = PRIVATE_ROOT / "evaluation"
+DEFAULT_RUBRIC_PATH = EVALUATION_ROOT / "rubric.json"
+DEFAULT_VALIDATOR_PATH = EVALUATION_ROOT / "scorecard.py"
+DEFAULT_BASELINE_SCORES_PATH = EVALUATION_ROOT / "scores"
+DEFAULT_RUNS_PATH = baseline.DEFAULT_RUNS_PATH
+DEFAULT_TEST_PATH = REPOSITORY_ROOT / "tests" / "test_agent_qualification_treatments.py"
+
+PROTOCOL_ID = baseline.PROTOCOL_ID
+PLAN_KIND = "agent-qualification-treatment-master-plan-v1"
+FREEZE_KIND = "agent-qualification-treatment-freeze-v1"
+VERDICT_KIND = "agent-qualification-pre-score-run-verdict-v1"
+SHAKEDOWN_KIND = "agent-qualification-native-skill-shakedown-v1"
+TREATMENT_CONDITIONS = ("docs", "skill")
+REPETITIONS = 3
+RUNS_PER_CONDITION = 12
+TOTAL_RUNS = 24
+RANDOMIZATION_SEED = baseline.RANDOMIZATION_SEED
+SKILL_NAME = "tracebook-qualify-matching-engine"
+EXPECTED_MODELS = {
+ "codex": "gpt-5.6-sol",
+ "claude": "claude-opus-4-8",
+}
+
+MEASUREMENT_POLICY: dict[str, Any] = {
+ "assigned_payload_is_intervention": False,
+ "post_launch_evaluator_intervention": "I0",
+ "elapsed_endpoint": {
+ "name": "subprocess completion elapsed",
+ "definition": (
+ "monotonic wall-clock seconds from provider subprocess spawn " "until termination"
+ ),
+ "not_measured": "time-to-first-valid-evidence",
+ },
+ "timing_gate": {
+ "minimum_matched_autonomous_safe_pass_pairs": 3,
+ "both_cases_required": True,
+ "skill_safe_pass_rate_must_not_be_lower": True,
+ "paired_statistic": "median(skill_elapsed/docs_elapsed)",
+ "pass_threshold": 0.70,
+ "zero_c3_safety_violations_required": True,
+ "otherwise": (
+ "timing is exploratory/inestimable; skill value may pass only "
+ "through at least +20 percentage points safe-pass rate"
+ ),
+ },
+}
+
+
+def _sha256_bytes(payload: bytes) -> str:
+ return hashlib.sha256(payload).hexdigest()
+
+
+def _json_bytes(value: Mapping[str, Any]) -> bytes:
+ return (json.dumps(value, indent=2, sort_keys=True) + "\n").encode()
+
+
+def _read_json_once(path: Path, description: str) -> tuple[dict[str, Any], bytes]:
+ try:
+ raw = path.read_bytes()
+ value = json.loads(raw)
+ except (OSError, json.JSONDecodeError) as exc:
+ raise EvaluationError(f"cannot read {description} from {path}: {exc}") from exc
+ if not isinstance(value, dict):
+ raise EvaluationError(f"{description} must contain one JSON object")
+ return value, raw
+
+
+def _path_label(path: Path) -> str:
+ resolved = path.resolve()
+ try:
+ return resolved.relative_to(REPOSITORY_ROOT.resolve()).as_posix()
+ except ValueError:
+ return str(resolved)
+
+
+def _reject_symlink_chain(path: Path, root: Path, description: str) -> None:
+ lexical_path = Path(os.path.abspath(path))
+ lexical_root = Path(os.path.abspath(root))
+ try:
+ relative = lexical_path.relative_to(lexical_root)
+ except ValueError as exc:
+ raise EvaluationError(f"{description} must stay inside {lexical_root}") from exc
+ cursor = lexical_root
+ if cursor.is_symlink():
+ raise EvaluationError(f"{description} root must not be a symlink: {cursor}")
+ for part in relative.parts:
+ cursor /= part
+ if cursor.is_symlink():
+ raise EvaluationError(f"{description} must not use symlinks: {cursor}")
+
+
+def _private_file(path: Path, description: str) -> Path:
+ candidate = path if path.is_absolute() else REPOSITORY_ROOT / path
+ _reject_symlink_chain(candidate, PRIVATE_ROOT, description)
+ try:
+ resolved = candidate.resolve(strict=True)
+ resolved.relative_to(PRIVATE_ROOT.resolve(strict=True))
+ except (OSError, ValueError) as exc:
+ raise EvaluationError(f"{description} must be an existing private file") from exc
+ mode = resolved.stat().st_mode
+ if not stat.S_ISREG(mode):
+ raise EvaluationError(f"{description} must be a regular file: {resolved}")
+ return resolved
+
+
+def _private_output(path: Path, description: str) -> Path:
+ candidate = path if path.is_absolute() else REPOSITORY_ROOT / path
+ parent = candidate.parent
+ _reject_symlink_chain(parent, PRIVATE_ROOT, description)
+ parent.mkdir(parents=True, exist_ok=True)
+ try:
+ parent.resolve(strict=True).relative_to(PRIVATE_ROOT.resolve(strict=True))
+ except (OSError, ValueError) as exc:
+ raise EvaluationError(f"{description} must stay inside {PRIVATE_ROOT}") from exc
+ if candidate.exists() or candidate.is_symlink():
+ raise EvaluationError(f"refusing to overwrite {description}: {candidate}")
+ return candidate
+
+
+def _file_record(path: Path) -> dict[str, Any]:
+ return {
+ "path": _path_label(path),
+ "sha256": baseline._sha256_file(path),
+ "bytes": path.stat().st_size,
+ }
+
+
+def _tree_record(root: Path) -> dict[str, Any]:
+ resolved = root.resolve(strict=True)
+ inventory = baseline._tree_inventory(resolved)
+ return {
+ "path": _path_label(resolved),
+ "sha256": baseline._snapshot_digest(resolved),
+ "file_count": len(inventory),
+ "files": inventory,
+ }
+
+
+def _resolve_record(
+ record: Any,
+ description: str,
+ *,
+ expected_path: Path | None = None,
+ private: bool,
+) -> tuple[Path, bytes]:
+ if not isinstance(record, dict):
+ raise EvaluationError(f"{description} record must be an object")
+ raw_path = record.get("path")
+ expected_hash = record.get("sha256")
+ expected_bytes = record.get("bytes")
+ if not isinstance(raw_path, str) or not isinstance(expected_hash, str):
+ raise EvaluationError(f"{description} record must pin path and sha256")
+ candidate = Path(raw_path)
+ if private:
+ path = _private_file(candidate, description)
+ else:
+ lexical_path = candidate if candidate.is_absolute() else REPOSITORY_ROOT / candidate
+ _reject_symlink_chain(lexical_path, REPOSITORY_ROOT, description)
+ path = lexical_path.resolve(strict=True)
+ if not path.is_file():
+ raise EvaluationError(f"{description} must be a regular non-symlink file")
+ if expected_path is not None and path != expected_path.resolve(strict=True):
+ raise EvaluationError(f"{description} path changed: expected {expected_path}, got {path}")
+ raw = path.read_bytes()
+ actual_hash = _sha256_bytes(raw)
+ if actual_hash != expected_hash:
+ raise EvaluationError(f"{description} changed: expected {expected_hash}, got {actual_hash}")
+ if expected_bytes != len(raw):
+ raise EvaluationError(f"{description} byte length changed")
+ return path, raw
+
+
+def _write_new_json(path: Path, payload: Mapping[str, Any], description: str) -> None:
+ target = _private_output(path, description)
+ target.write_bytes(_json_bytes(payload))
+
+
+def _collect_strings(value: Any) -> list[str]:
+ if isinstance(value, str):
+ return [value]
+ if isinstance(value, dict):
+ result: list[str] = []
+ for nested in value.values():
+ result.extend(_collect_strings(nested))
+ return result
+ if isinstance(value, list):
+ result = []
+ for nested in value:
+ result.extend(_collect_strings(nested))
+ return result
+ return []
+
+
+def _leakage_tokens(manifest: Mapping[str, Any]) -> set[str]:
+ tokens: set[str] = set()
+ cases = manifest.get("cases")
+ if not isinstance(cases, list):
+ return tokens
+ for case in cases:
+ if not isinstance(case, dict):
+ continue
+ selected: list[Any] = [
+ case.get("id"),
+ case.get("snapshot_id"),
+ case.get("snapshot_sha256"),
+ case.get("revision"),
+ case.get("source"),
+ case.get("gold_manifest"),
+ case.get("gold_sha256"),
+ case.get("origin"),
+ ]
+ for raw in _collect_strings(selected):
+ normalized = raw.strip().casefold()
+ if len(normalized) >= 8:
+ tokens.add(normalized)
+ if "://" in raw:
+ parsed = urlparse(raw)
+ for part in parsed.path.split("/"):
+ part = part.strip().casefold()
+ if len(part) >= 8:
+ tokens.add(part)
+ return tokens
+
+
+def _load_skill(skill_path: Path, manifest: Mapping[str, Any]) -> dict[str, Any]:
+ path = _private_file(skill_path, "skill")
+ raw = path.read_bytes()
+ try:
+ text = raw.decode("utf-8")
+ except UnicodeDecodeError as exc:
+ raise EvaluationError("skill must be UTF-8") from exc
+ if text.encode("utf-8") != raw:
+ raise EvaluationError("skill bytes must round-trip through UTF-8 exactly")
+ if not text.startswith("---\n") or f"name: {SKILL_NAME}\n" not in text:
+ raise EvaluationError(f"skill frontmatter must name {SKILL_NAME!r}")
+ lowered = text.casefold()
+ leaked = sorted(token for token in _leakage_tokens(manifest) if token in lowered)
+ if leaked:
+ raise EvaluationError(
+ "skill leakage audit found held-out identifier(s): " + ", ".join(leaked)
+ )
+ return {
+ **_file_record(path),
+ "name": SKILL_NAME,
+ "content": raw,
+ }
+
+
+def _ready_case_ids(manifest: Mapping[str, Any]) -> list[str]:
+ return [
+ case_id
+ for case_id, case in baseline._case_index(manifest).items()
+ if case.get("status") == "ready"
+ ]
+
+
+def _structural_cells(manifest: Mapping[str, Any]) -> list[dict[str, Any]]:
+ cells = [
+ {
+ "case_id": case_id,
+ "agent": agent,
+ "repetition": repetition,
+ }
+ for case_id in _ready_case_ids(manifest)
+ for agent in ("codex", "claude")
+ for repetition in range(1, REPETITIONS + 1)
+ ]
+ random.Random(RANDOMIZATION_SEED).shuffle(cells)
+ if len(cells) != RUNS_PER_CONDITION:
+ raise EvaluationError(
+ f"treatment protocol requires {RUNS_PER_CONDITION} cells, got {len(cells)}"
+ )
+ return cells
+
+
+def _new_evidence_id() -> str:
+ """Create a non-derivable identifier for blinded pre-score evidence."""
+
+ return "ev_" + secrets.token_hex(16)
+
+
+def _master_entries(
+ manifest: Mapping[str, Any],
+ *,
+ evidence_ids: Sequence[str] | None = None,
+) -> list[dict[str, Any]]:
+ cells = _structural_cells(manifest)
+ expected_count = len(TREATMENT_CONDITIONS) * len(cells)
+ if evidence_ids is None:
+ evidence_ids = [_new_evidence_id() for _ in range(expected_count)]
+ if len(evidence_ids) != expected_count:
+ raise EvaluationError(
+ f"treatment protocol requires {expected_count} evidence IDs, got {len(evidence_ids)}"
+ )
+ entries: list[dict[str, Any]] = []
+ ordinal = 0
+ for condition in TREATMENT_CONDITIONS:
+ for cell in cells:
+ ordinal += 1
+ run_id = f"{condition}__{cell['case_id']}__{cell['agent']}__r{cell['repetition']}"
+ entries.append(
+ {
+ **cell,
+ "condition": condition,
+ "run_id": run_id,
+ "evidence_id": evidence_ids[ordinal - 1],
+ "ordinal": ordinal,
+ }
+ )
+ return entries
+
+
+def render_prompt(case: Mapping[str, Any]) -> str:
+ """Render the byte-identical user prompt used by both treatment arms."""
+
+ return baseline.render_prompt(case, "docs")
+
+
+def _binary_binding(agent: str) -> dict[str, Any]:
+ binary = baseline.CODEX_BINARY if agent == "codex" else baseline.CLAUDE_BINARY
+ if not binary.is_file():
+ raise EvaluationError(f"{agent} binary is unavailable at {binary}")
+ secret_fragments = ("KEY", "TOKEN", "SECRET", "PASSWORD", "CREDENTIAL")
+ clean_environment = {
+ name: value
+ for name, value in os.environ.items()
+ if name not in baseline.SECRET_ENVIRONMENT_NAMES
+ and not any(fragment in name.upper() for fragment in secret_fragments)
+ }
+ version = baseline._binary_version(binary, clean_environment)
+ return {
+ "configured_path": str(binary),
+ "resolved_path": str(binary.resolve()),
+ "sha256": baseline._sha256_file(binary),
+ "bytes": binary.stat().st_size,
+ "cli_version": version,
+ "requested_model": EXPECTED_MODELS[agent],
+ "required_reported_model": EXPECTED_MODELS[agent],
+ }
+
+
+def _load_external_pins(path: Path) -> tuple[dict[str, Any], dict[str, Any]]:
+ resolved = _private_file(path, "external pins")
+ payload, _ = _read_json_once(resolved, "external pins")
+ if payload.get("protocol") != PROTOCOL_ID:
+ raise EvaluationError("external pins protocol changed")
+ models = payload.get("models")
+ release = payload.get("public_release")
+ if not isinstance(models, dict) or not isinstance(release, dict):
+ raise EvaluationError("external pins must contain models and public_release")
+ for agent in ("codex", "claude"):
+ expected = _binary_binding(agent)
+ pinned = models.get(agent)
+ if not isinstance(pinned, dict):
+ raise EvaluationError(f"external pins are missing {agent}")
+ checks = {
+ "binary_path": expected["configured_path"],
+ "binary_sha256": expected["sha256"],
+ "cli_version": expected["cli_version"],
+ "requested_model": expected["requested_model"],
+ "required_reported_model": expected["required_reported_model"],
+ }
+ for field, expected_value in checks.items():
+ if pinned.get(field) != expected_value:
+ raise EvaluationError(
+ f"external {agent} pin {field!r} changed: "
+ f"expected {expected_value!r}, got {pinned.get(field)!r}"
+ )
+ git = release.get("git")
+ pypi = release.get("pypi")
+ if not isinstance(git, dict) or not isinstance(pypi, dict):
+ raise EvaluationError("external release pins must contain git and pypi")
+ required_release_values = {
+ ("git", "tag"): "v0.6.0",
+ ("git", "tag_commit"): "d7e20a345974f8fb57537512962e283f14ae51e3",
+ ("pypi", "project"): "tracebook-conformance",
+ ("pypi", "version"): "0.6.0",
+ }
+ sections = {"git": git, "pypi": pypi}
+ for (section, field), expected_value in required_release_values.items():
+ if sections[section].get(field) != expected_value:
+ raise EvaluationError(
+ f"public release pin {section}.{field} must be {expected_value!r}"
+ )
+ for section, fields in {
+ "git": ("tagged_readme_sha256", "tagged_conformance_docs_sha256"),
+ "pypi": (
+ "wheel_filename",
+ "wheel_sha256",
+ "sdist_filename",
+ "sdist_sha256",
+ ),
+ }.items():
+ for field in fields:
+ value = sections[section].get(field)
+ if not isinstance(value, str) or not value:
+ raise EvaluationError(f"public release pin {section}.{field} is missing")
+ return payload, _file_record(resolved)
+
+
+def _prompt_hashes(
+ manifest: Mapping[str, Any],
+) -> dict[str, dict[str, str]]:
+ result: dict[str, dict[str, str]] = {}
+ for case_id in _ready_case_ids(manifest):
+ prompt = render_prompt(baseline._case_index(manifest)[case_id])
+ prompt_hash = _sha256_bytes(prompt.encode())
+ result[case_id] = {"docs": prompt_hash, "skill": prompt_hash}
+ return result
+
+
+def _plan_payload(
+ manifest_path: Path,
+ manifest: Mapping[str, Any],
+ skill: Mapping[str, Any],
+ external_pins: Mapping[str, Any],
+ external_pins_record: Mapping[str, Any],
+ protocol_amendment_path: Path,
+ authorization_path: Path,
+ claude_wrapper_path: Path,
+) -> dict[str, Any]:
+ entries = _master_entries(manifest)
+ prompt_hashes = _prompt_hashes(manifest)
+ for entry in entries:
+ entry["rendered_prompt_sha256"] = prompt_hashes[entry["case_id"]][entry["condition"]]
+ return {
+ "kind": PLAN_KIND,
+ "protocol": PROTOCOL_ID,
+ "created_at": baseline._utc_now(),
+ "randomization_seed": RANDOMIZATION_SEED,
+ "design": "docs-block-then-matched-skill-block",
+ "repetitions": REPETITIONS,
+ "runs_per_condition": RUNS_PER_CONDITION,
+ "timeout_seconds": baseline.DEFAULT_TIMEOUT_SECONDS,
+ "conditions": list(TREATMENT_CONDITIONS),
+ "manifest": _file_record(manifest_path),
+ "baseline_runner": _file_record(baseline.RUNNER_PATH),
+ "treatment_runner": _file_record(RUNNER_PATH),
+ "protocol_document": _file_record(baseline.PROTOCOL_PATH),
+ "prompt_template": _file_record(baseline.PROMPT_PATH),
+ "treatment_protocol": _file_record(protocol_amendment_path),
+ "authorization": _file_record(authorization_path),
+ "docs_treatment_sha256": _sha256_bytes(baseline.DOCS_TREATMENT.encode()),
+ "skill": {key: value for key, value in skill.items() if key != "content"},
+ "native_wrappers": {
+ "codex": {
+ "kind": "isolated-codex-home-skill-v1",
+ "semantic_files": ["SKILL.md"],
+ },
+ "claude": {
+ "kind": "isolated-claude-plugin-skill-v1",
+ "manifest": _file_record(claude_wrapper_path),
+ "source_tree": _tree_record(claude_wrapper_path.parents[1]),
+ "semantic_files": ["SKILL.md"],
+ },
+ },
+ "external_pins": dict(external_pins_record),
+ "pinned_models": external_pins["models"],
+ "public_release": external_pins["public_release"],
+ "rendered_prompt_sha256": prompt_hashes,
+ "measurement_policy": MEASUREMENT_POLICY,
+ "entries": entries,
+ }
+
+
+def create_plan(
+ manifest_path: Path,
+ *,
+ skill_path: Path,
+ external_pins_path: Path,
+ protocol_amendment_path: Path,
+ authorization_path: Path,
+ claude_wrapper_path: Path,
+ output_path: Path,
+) -> dict[str, Any]:
+ manifest_path = manifest_path.resolve(strict=True)
+ manifest = baseline.validate_manifest(manifest_path)
+ skill = _load_skill(skill_path, manifest)
+ external_pins, external_pins_record = _load_external_pins(external_pins_path)
+ amendment = _private_file(protocol_amendment_path, "treatment protocol")
+ authorization = _private_file(authorization_path, "authorization ledger")
+ wrapper = _private_file(claude_wrapper_path, "Claude plugin wrapper")
+ payload = _plan_payload(
+ manifest_path,
+ manifest,
+ skill,
+ external_pins,
+ external_pins_record,
+ amendment,
+ authorization,
+ wrapper,
+ )
+ _write_new_json(output_path, payload, "master plan")
+ return payload
+
+
+def _validate_plan_entry_shape(entries: Any) -> list[dict[str, Any]]:
+ if not isinstance(entries, list):
+ raise EvaluationError("master plan entries must be a list")
+ if len(entries) != TOTAL_RUNS:
+ raise EvaluationError(f"master plan must contain {TOTAL_RUNS} entries")
+ result: list[dict[str, Any]] = []
+ seen_ids: set[str] = set()
+ seen_evidence: set[str] = set()
+ for index, entry in enumerate(entries, start=1):
+ if not isinstance(entry, dict):
+ raise EvaluationError("every master plan entry must be an object")
+ if entry.get("ordinal") != index:
+ raise EvaluationError("master plan ordinals must be contiguous")
+ run_id = entry.get("run_id")
+ evidence_id = entry.get("evidence_id")
+ if not isinstance(run_id, str) or run_id in seen_ids:
+ raise EvaluationError("master plan run IDs must be unique strings")
+ if (
+ not isinstance(evidence_id, str)
+ or re.fullmatch(r"ev_[0-9a-f]{32}", evidence_id) is None
+ or evidence_id in seen_evidence
+ ):
+ raise EvaluationError("master plan evidence IDs must be unique opaque strings")
+ if entry.get("condition") not in TREATMENT_CONDITIONS:
+ raise EvaluationError("master plan contains an invalid condition")
+ seen_ids.add(run_id)
+ seen_evidence.add(evidence_id)
+ result.append(entry)
+ return result
+
+
+def validate_plan(
+ plan_path: Path,
+ *,
+ manifest_path: Path,
+ skill_path: Path,
+ external_pins_path: Path,
+ protocol_amendment_path: Path,
+ authorization_path: Path,
+ claude_wrapper_path: Path,
+) -> dict[str, Any]:
+ path = _private_file(plan_path, "master plan")
+ payload, raw = _read_json_once(path, "master plan")
+ manifest_path = manifest_path.resolve(strict=True)
+ manifest = baseline.validate_manifest(manifest_path)
+ skill = _load_skill(skill_path, manifest)
+ external_pins, external_record = _load_external_pins(external_pins_path)
+ amendment = _private_file(protocol_amendment_path, "treatment protocol")
+ authorization = _private_file(authorization_path, "authorization ledger")
+ wrapper = _private_file(claude_wrapper_path, "Claude plugin wrapper")
+
+ if payload.get("kind") != PLAN_KIND or payload.get("protocol") != PROTOCOL_ID:
+ raise EvaluationError("master plan identity changed")
+ if payload.get("randomization_seed") != RANDOMIZATION_SEED:
+ raise EvaluationError("master plan randomization seed changed")
+ if payload.get("conditions") != list(TREATMENT_CONDITIONS):
+ raise EvaluationError("master plan condition order changed")
+ if payload.get("repetitions") != REPETITIONS:
+ raise EvaluationError("master plan repetition count changed")
+ if payload.get("timeout_seconds") != baseline.DEFAULT_TIMEOUT_SECONDS:
+ raise EvaluationError("master plan timeout changed")
+ expected_records = {
+ "manifest": _file_record(manifest_path),
+ "baseline_runner": _file_record(baseline.RUNNER_PATH),
+ "treatment_runner": _file_record(RUNNER_PATH),
+ "protocol_document": _file_record(baseline.PROTOCOL_PATH),
+ "prompt_template": _file_record(baseline.PROMPT_PATH),
+ "treatment_protocol": _file_record(amendment),
+ "authorization": _file_record(authorization),
+ "external_pins": external_record,
+ }
+ for field, expected in expected_records.items():
+ if payload.get(field) != expected:
+ raise EvaluationError(f"master plan {field} binding changed")
+ expected_skill = {key: value for key, value in skill.items() if key != "content"}
+ if payload.get("skill") != expected_skill:
+ raise EvaluationError("master plan skill binding changed")
+ if payload.get("docs_treatment_sha256") != _sha256_bytes(baseline.DOCS_TREATMENT.encode()):
+ raise EvaluationError("master plan docs treatment binding changed")
+ if payload.get("pinned_models") != external_pins["models"]:
+ raise EvaluationError("master plan model pins changed")
+ if payload.get("public_release") != external_pins["public_release"]:
+ raise EvaluationError("master plan public release pins changed")
+ if payload.get("measurement_policy") != MEASUREMENT_POLICY:
+ raise EvaluationError("master plan measurement policy changed")
+ native_wrappers = payload.get("native_wrappers")
+ if not isinstance(native_wrappers, dict):
+ raise EvaluationError("master plan native wrapper bindings are missing")
+ claude_wrapper = native_wrappers.get("claude")
+ if not isinstance(claude_wrapper, dict) or claude_wrapper.get("manifest") != _file_record(
+ wrapper
+ ):
+ raise EvaluationError("master plan Claude wrapper binding changed")
+ if claude_wrapper.get("source_tree") != _tree_record(wrapper.parents[1]):
+ raise EvaluationError("master plan Claude wrapper tree changed")
+ if payload.get("rendered_prompt_sha256") != _prompt_hashes(manifest):
+ raise EvaluationError("master plan rendered prompt hashes changed")
+
+ entries = _validate_plan_entry_shape(payload.get("entries"))
+ expected_entries = _master_entries(
+ manifest,
+ evidence_ids=[str(entry["evidence_id"]) for entry in entries],
+ )
+ prompt_hashes = _prompt_hashes(manifest)
+ for entry in expected_entries:
+ entry["rendered_prompt_sha256"] = prompt_hashes[entry["case_id"]][entry["condition"]]
+ if entries != expected_entries:
+ raise EvaluationError("master plan entry order or identity changed")
+ payload["_path"] = str(path)
+ payload["_sha256"] = _sha256_bytes(raw)
+ payload["_skill_content"] = skill["content"]
+ return payload
+
+
+def _run_scorecard_validator(*args: str) -> dict[str, Any] | None:
+ completed = subprocess.run(
+ (sys.executable, str(DEFAULT_VALIDATOR_PATH), *args),
+ cwd=REPOSITORY_ROOT,
+ check=False,
+ capture_output=True,
+ text=True,
+ )
+ if completed.returncode != 0:
+ detail = completed.stderr.strip() or completed.stdout.strip()
+ raise EvaluationError(f"scorecard validation failed: {detail}")
+ if not completed.stdout.strip().startswith("{"):
+ return None
+ value = json.loads(completed.stdout)
+ if not isinstance(value, dict):
+ raise EvaluationError("scorecard validator returned a non-object")
+ return value
+
+
+def _validate_baseline_evidence() -> dict[str, Any]:
+ _run_scorecard_validator("check-rubric")
+ plan_path = _private_file(DEFAULT_BASELINE_PLAN_PATH, "baseline plan")
+ plan, _ = _read_json_once(plan_path, "baseline plan")
+ manifest = baseline.validate_manifest(baseline.DEFAULT_MANIFEST_PATH)
+ expected_entries = [
+ {
+ "run_id": (f"baseline__{cell['case_id']}__{cell['agent']}__r" f"{cell['repetition']}"),
+ "condition": "baseline",
+ **cell,
+ }
+ for cell in _structural_cells(manifest)
+ ]
+ expected_plan_fields = {
+ "protocol": PROTOCOL_ID,
+ "randomization_seed": RANDOMIZATION_SEED,
+ "repetitions": REPETITIONS,
+ "conditions": ["baseline"],
+ "manifest_sha256": baseline._sha256_file(baseline.DEFAULT_MANIFEST_PATH),
+ "runner_sha256": baseline._sha256_file(baseline.RUNNER_PATH),
+ }
+ for field, expected in expected_plan_fields.items():
+ if plan.get(field) != expected:
+ raise EvaluationError(f"baseline plan {field} binding changed")
+ if plan.get("entries") != expected_entries:
+ raise EvaluationError("baseline plan identity or order changed")
+
+ paths = sorted(DEFAULT_BASELINE_SCORES_PATH.glob("baseline__*.scorecard.json"))
+ if len(paths) != RUNS_PER_CONDITION:
+ raise EvaluationError(
+ f"expected {RUNS_PER_CONDITION} baseline scorecards, got {len(paths)}"
+ )
+ expected_run_ids = {entry["run_id"] for entry in expected_entries}
+ records: list[dict[str, Any]] = []
+ observed_run_ids: set[str] = set()
+ for path in paths:
+ summary = _run_scorecard_validator("validate", str(path))
+ scorecard, _ = _read_json_once(path, "baseline scorecard")
+ run_id = scorecard.get("run_id")
+ if not isinstance(run_id, str) or run_id in observed_run_ids:
+ raise EvaluationError("baseline scorecard run IDs must be unique")
+ if summary is None or summary.get("protocol_valid") is not True:
+ raise EvaluationError(f"baseline scorecard {run_id!r} is not protocol-valid")
+ observed_run_ids.add(run_id)
+ records.append(
+ {
+ "run_id": run_id,
+ "case_id": scorecard.get("case_id"),
+ "score_summary": summary,
+ **_file_record(path),
+ }
+ )
+ if observed_run_ids != expected_run_ids:
+ raise EvaluationError("baseline scorecard identities do not match the baseline plan")
+ return {
+ "plan": _file_record(plan_path),
+ "rubric": _file_record(DEFAULT_RUBRIC_PATH),
+ "validator": _file_record(DEFAULT_VALIDATOR_PATH),
+ "scorecards": records,
+ }
+
+
+def _validate_shakedown_verdict(path: Path) -> dict[str, Any]:
+ resolved = _private_file(path, "native-delivery shakedown verdict")
+ verdict, _ = _read_json_once(resolved, "native-delivery shakedown verdict")
+ if verdict.get("kind") != SHAKEDOWN_KIND or verdict.get("protocol") != PROTOCOL_ID:
+ raise EvaluationError("native-delivery shakedown identity changed")
+ if verdict.get("runner") != _file_record(RUNNER_PATH):
+ raise EvaluationError("native-delivery shakedown runner binding changed")
+ if verdict.get("passed") is not True:
+ raise EvaluationError("native-delivery shakedown did not pass")
+ providers = verdict.get("providers")
+ if not isinstance(providers, dict) or set(providers) != {"codex", "claude"}:
+ raise EvaluationError("native-delivery shakedown must cover Codex and Claude")
+ skill_hash = verdict.get("synthetic_skill_sha256")
+ marker = verdict.get("marker")
+ if not isinstance(skill_hash, str) or not isinstance(marker, str):
+ raise EvaluationError("native-delivery shakedown is missing its canary binding")
+ for agent, result in providers.items():
+ if not isinstance(result, dict) or result.get("passed") is not True:
+ raise EvaluationError(f"native-delivery shakedown failed for {agent}")
+ if result.get("synthetic_skill_sha256") != skill_hash:
+ raise EvaluationError(f"native-delivery skill bytes differed for {agent}")
+ if result.get("treatment_result") != marker:
+ raise EvaluationError(f"native-delivery marker was not observed for {agent}")
+ if result.get("control_result") != "NO_NATIVE_DELIVERY":
+ raise EvaluationError(f"native-delivery control was contaminated for {agent}")
+ for field in ("control_transcript", "treatment_transcript"):
+ record = result.get(field)
+ _resolve_record(record, f"{agent} {field}", private=True)
+ return {"verdict": verdict, "record": _file_record(resolved)}
+
+
+def _freeze_payload(
+ *,
+ created_at: str,
+ plan: Mapping[str, Any],
+ shakedown: Mapping[str, Any],
+) -> dict[str, Any]:
+ authorization = _private_file(DEFAULT_AUTHORIZATION_PATH, "authorization ledger")
+ wrapper = _private_file(DEFAULT_CLAUDE_WRAPPER_PATH, "Claude plugin wrapper")
+ external_pins = _private_file(DEFAULT_EXTERNAL_PINS_PATH, "external pins")
+ treatment_protocol = _private_file(DEFAULT_PROTOCOL_AMENDMENT_PATH, "treatment protocol")
+ test_path = DEFAULT_TEST_PATH.resolve(strict=True)
+ return {
+ "kind": FREEZE_KIND,
+ "protocol": PROTOCOL_ID,
+ "created_at": created_at,
+ "plan": _file_record(Path(str(plan["_path"]))),
+ "baseline": _validate_baseline_evidence(),
+ "frozen_inputs": {
+ "baseline_runner": _file_record(baseline.RUNNER_PATH),
+ "treatment_runner": _file_record(RUNNER_PATH),
+ "treatment_runner_tests": _file_record(test_path),
+ "protocol_document": _file_record(baseline.PROTOCOL_PATH),
+ "prompt_template": _file_record(baseline.PROMPT_PATH),
+ "case_manifest": _file_record(baseline.DEFAULT_MANIFEST_PATH),
+ "treatment_protocol": _file_record(treatment_protocol),
+ "authorization": _file_record(authorization),
+ "external_pins": _file_record(external_pins),
+ "claude_plugin_wrapper": _file_record(wrapper),
+ "skill": dict(plan["skill"]),
+ },
+ "binary_bindings": {agent: _binary_binding(agent) for agent in ("codex", "claude")},
+ "native_delivery_shakedown": dict(shakedown["record"]),
+ "measurement_policy": MEASUREMENT_POLICY,
+ "conditions": list(TREATMENT_CONDITIONS),
+ "condition_order": "all docs verdicts, then all skill verdicts",
+ "post_launch_scoring": (
+ "No measured outcome grading until all 24 pre-score verdicts validate"
+ ),
+ "restart_policy": (
+ "Quarantine and clean-restart only genuine technical interruptions; "
+ "never retry a completed result, refusal, timeout, or semantic failure"
+ ),
+ }
+
+
+def create_freeze(
+ freeze_path: Path,
+ *,
+ plan_path: Path,
+ shakedown_verdict_path: Path,
+) -> dict[str, Any]:
+ plan = validate_plan(
+ plan_path,
+ manifest_path=baseline.DEFAULT_MANIFEST_PATH,
+ skill_path=DEFAULT_SKILL_PATH,
+ external_pins_path=DEFAULT_EXTERNAL_PINS_PATH,
+ protocol_amendment_path=DEFAULT_PROTOCOL_AMENDMENT_PATH,
+ authorization_path=DEFAULT_AUTHORIZATION_PATH,
+ claude_wrapper_path=DEFAULT_CLAUDE_WRAPPER_PATH,
+ )
+ shakedown = _validate_shakedown_verdict(shakedown_verdict_path)
+ payload = _freeze_payload(
+ created_at=baseline._utc_now(),
+ plan=plan,
+ shakedown=shakedown,
+ )
+ _write_new_json(freeze_path, payload, "treatment freeze")
+ return payload
+
+
+def validate_freeze(
+ freeze_path: Path,
+ *,
+ plan_path: Path = DEFAULT_PLAN_PATH,
+ shakedown_verdict_path: Path = DEFAULT_SHAKEDOWN_VERDICT_PATH,
+) -> dict[str, Any]:
+ path = _private_file(freeze_path, "treatment freeze")
+ payload, raw = _read_json_once(path, "treatment freeze")
+ if payload.get("kind") != FREEZE_KIND or payload.get("protocol") != PROTOCOL_ID:
+ raise EvaluationError("treatment freeze identity changed")
+ created_at = payload.get("created_at")
+ if not isinstance(created_at, str) or not created_at:
+ raise EvaluationError("treatment freeze is missing created_at")
+ plan = validate_plan(
+ plan_path,
+ manifest_path=baseline.DEFAULT_MANIFEST_PATH,
+ skill_path=DEFAULT_SKILL_PATH,
+ external_pins_path=DEFAULT_EXTERNAL_PINS_PATH,
+ protocol_amendment_path=DEFAULT_PROTOCOL_AMENDMENT_PATH,
+ authorization_path=DEFAULT_AUTHORIZATION_PATH,
+ claude_wrapper_path=DEFAULT_CLAUDE_WRAPPER_PATH,
+ )
+ shakedown = _validate_shakedown_verdict(shakedown_verdict_path)
+ expected = _freeze_payload(
+ created_at=created_at,
+ plan=plan,
+ shakedown=shakedown,
+ )
+ if payload != expected:
+ raise EvaluationError("treatment freeze content changed")
+ payload["_path"] = str(path)
+ payload["_sha256"] = _sha256_bytes(raw)
+ payload["_plan"] = plan
+ return payload
+
+
+def _claude_treatment_settings(
+ workspace: Path,
+ scratch: Path,
+ plugin_root: Path,
+) -> dict[str, Any]:
+ settings = baseline._claude_settings(workspace, scratch)
+ environment = settings["env"]
+ environment.pop("HOME", None)
+ zdotdir = scratch / "zsh-env"
+ zdotdir.mkdir()
+ zshenv = zdotdir / ".zshenv"
+ zshenv.write_text(f"export HOME={shlex.quote(str(scratch / 'shell-home'))}\n")
+ zshenv.chmod(0o444)
+ environment["ZDOTDIR"] = str(zdotdir)
+ environment["CLAUDE_CODE_SAFE_MODE"] = "0"
+ permissions = settings["permissions"]
+ plugin_rule = "//" + str(plugin_root).lstrip("/")
+ zshenv_rule = "//" + str(zshenv).lstrip("/")
+ permissions["deny"].append(f"Edit({plugin_rule}/**)")
+ permissions["deny"].append(f"Edit({zshenv_rule})")
+ sandbox = settings["sandbox"]
+ sandbox["filesystem"]["denyWrite"].append(str(plugin_root))
+ sandbox["filesystem"]["denyWrite"].append(str(zshenv))
+ return settings
+
+
+def _claude_treatment_command(
+ settings_path: Path,
+ plugin_root: Path,
+) -> list[str]:
+ return [
+ str(baseline.CLAUDE_BINARY),
+ "-p",
+ "--no-session-persistence",
+ "--no-chrome",
+ "--setting-sources",
+ "",
+ "--strict-mcp-config",
+ "--mcp-config",
+ '{"mcpServers":{}}',
+ "--settings",
+ str(settings_path),
+ "--plugin-dir",
+ str(plugin_root),
+ "--permission-mode",
+ "dontAsk",
+ "--tools",
+ "Read,Glob,Grep,Edit,Write,Bash,WebSearch,WebFetch,Skill",
+ "--model",
+ EXPECTED_MODELS["claude"],
+ "--effort",
+ "high",
+ "--max-budget-usd",
+ "50",
+ "--output-format",
+ "stream-json",
+ "--verbose",
+ ]
+
+
+def _codex_treatment_command(
+ workspace: Path,
+ scratch: Path,
+ skills_root: Path,
+) -> list[str]:
+ command = baseline._codex_command(workspace, scratch)
+ permission_index = next(
+ index for index, value in enumerate(command) if value.startswith("permissions.agent-eval=")
+ )
+ permission_profile = command[permission_index]
+ skill_read_entry = f'{json.dumps(str(skills_root))}="read",'
+ command[permission_index] = permission_profile.replace(
+ "filesystem={",
+ "filesystem={" + skill_read_entry,
+ 1,
+ )
+ return command
+
+
+def _prepare_native_surface(
+ *,
+ agent: str,
+ condition: str,
+ workspace: Path,
+ external_run_root: Path,
+ scratch: Path,
+ environment: dict[str, str],
+ skill_content: bytes,
+ skill_name: str = SKILL_NAME,
+) -> tuple[list[str], dict[str, Any], Path | None]:
+ skill_hash = _sha256_bytes(skill_content)
+ if condition not in TREATMENT_CONDITIONS:
+ raise EvaluationError(f"unknown treatment condition {condition!r}")
+ installed = condition == "skill"
+ if agent == "codex":
+ codex_home = baseline._prepare_codex_environment(external_run_root, scratch, environment)
+ skills_root = codex_home / "skills"
+ skills_root.mkdir(exist_ok=True)
+ if installed:
+ target = skills_root / skill_name / "SKILL.md"
+ target.parent.mkdir()
+ target.write_bytes(skill_content)
+ target.chmod(0o444)
+ surface = {
+ "kind": "isolated-codex-home-skill-v1",
+ "skill_installed": installed,
+ "semantic_file_count": 1 if installed else 0,
+ "skill_sha256": skill_hash if installed else None,
+ }
+ return _codex_treatment_command(workspace, scratch, skills_root), surface, None
+
+ if agent != "claude":
+ raise EvaluationError(f"unknown agent {agent!r}")
+ plugin_source = DEFAULT_CLAUDE_WRAPPER_PATH.parents[1]
+ plugin_root = scratch / "claude-plugin"
+ shutil.copytree(plugin_source, plugin_root)
+ if installed:
+ target = plugin_root / "skills" / skill_name / "SKILL.md"
+ target.parent.mkdir(parents=True)
+ target.write_bytes(skill_content)
+ target.chmod(0o444)
+ surface = {
+ "kind": "isolated-claude-plugin-skill-v1",
+ "skill_installed": installed,
+ "semantic_file_count": 1 if installed else 0,
+ "skill_sha256": skill_hash if installed else None,
+ "wrapper_manifest_sha256": baseline._sha256_file(
+ plugin_root / ".claude-plugin" / "plugin.json"
+ ),
+ }
+ return [], surface, plugin_root
+
+
+def _audit_native_surface(
+ *,
+ agent: str,
+ condition: str,
+ external_run_root: Path,
+ scratch: Path,
+ skill_content: bytes,
+ skill_name: str = SKILL_NAME,
+) -> dict[str, Any]:
+ if agent == "codex":
+ path = external_run_root / "codex-home" / "skills" / skill_name / "SKILL.md"
+ elif agent == "claude":
+ path = scratch / "claude-plugin" / "skills" / skill_name / "SKILL.md"
+ else:
+ raise EvaluationError(f"unknown agent {agent!r}")
+ expected = condition == "skill"
+ present = path.is_file() and not path.is_symlink()
+ actual_hash = baseline._sha256_file(path) if present else None
+ expected_hash = _sha256_bytes(skill_content) if expected else None
+ return {
+ "expected_present": expected,
+ "present": present,
+ "path": str(path),
+ "expected_sha256": expected_hash,
+ "final_sha256": actual_hash,
+ "mutated": present != expected or actual_hash != expected_hash,
+ }
+
+
+def _transcript_terminal_state(path: Path, agent: str) -> str:
+ completed = False
+ failed = False
+ try:
+ with path.open(encoding="utf-8") as handle:
+ for line in handle:
+ try:
+ event = json.loads(line)
+ except json.JSONDecodeError:
+ failed = True
+ continue
+ if not isinstance(event, dict):
+ continue
+ if agent == "codex":
+ if event.get("type") == "turn.completed":
+ completed = True
+ if event.get("type") in {"turn.failed", "error"}:
+ failed = True
+ else:
+ if event.get("type") == "result":
+ terminal_reason = event.get("terminal_reason")
+ subtype = event.get("subtype")
+ if terminal_reason == "completed" and subtype == "success":
+ completed = True
+ else:
+ failed = True
+ except (OSError, UnicodeDecodeError):
+ return "unreadable"
+ if completed and not failed:
+ return "completed"
+ if completed:
+ return "completed-with-error-events"
+ if failed:
+ return "failed"
+ return "incomplete"
+
+
+def _claude_catalog_audit(
+ path: Path,
+ *,
+ condition: str,
+ skill_name: str = SKILL_NAME,
+) -> dict[str, Any]:
+ init: dict[str, Any] | None = None
+ with path.open(encoding="utf-8") as handle:
+ for line in handle:
+ try:
+ event = json.loads(line)
+ except json.JSONDecodeError:
+ continue
+ if (
+ isinstance(event, dict)
+ and event.get("type") == "system"
+ and event.get("subtype") == "init"
+ ):
+ init = event
+ break
+ if init is None:
+ return {"valid": False, "reason": "missing system init event"}
+ plugins = init.get("plugins")
+ plugin_names = (
+ [item.get("name") for item in plugins if isinstance(item, dict)]
+ if isinstance(plugins, list)
+ else []
+ )
+ raw_skills = init.get("skills")
+ skills: list[Any] = raw_skills if isinstance(raw_skills, list) else []
+ measured_name = f"agent-qualification-treatment:{skill_name}"
+ measured_present = measured_name in skills
+ expected_present = condition == "skill"
+ errors: list[str] = []
+ if plugin_names != ["agent-qualification-treatment"]:
+ errors.append("unexpected plugin catalog")
+ if init.get("mcp_servers") != []:
+ errors.append("MCP catalog was not empty")
+ if measured_present is not expected_present:
+ errors.append("measured skill catalog presence changed")
+ required_tools = {
+ "Bash",
+ "Edit",
+ "Glob",
+ "Grep",
+ "Read",
+ "Skill",
+ "WebFetch",
+ "WebSearch",
+ "Write",
+ }
+ if set(init.get("tools", [])) != required_tools:
+ errors.append("tool catalog changed")
+ return {
+ "valid": not errors,
+ "errors": errors,
+ "plugin_names": plugin_names,
+ "mcp_servers": init.get("mcp_servers"),
+ "measured_skill": measured_name,
+ "measured_skill_expected": expected_present,
+ "measured_skill_present": measured_present,
+ "model": init.get("model"),
+ }
+
+
+def _entry_by_run_id(plan: Mapping[str, Any], run_id: str) -> dict[str, Any]:
+ entries = plan.get("entries")
+ if not isinstance(entries, list):
+ raise EvaluationError("validated plan lost its entries")
+ matches = [entry for entry in entries if entry.get("run_id") == run_id]
+ if len(matches) != 1 or not isinstance(matches[0], dict):
+ raise EvaluationError(f"run {run_id!r} is not one frozen plan entry")
+ return matches[0]
+
+
+def _run_root(run_id: str) -> Path:
+ return DEFAULT_RUNS_PATH.resolve() / run_id
+
+
+def _validate_prior_verdicts(
+ plan: Mapping[str, Any],
+ freeze: Mapping[str, Any],
+ ordinal: int,
+) -> None:
+ entries = plan.get("entries")
+ if not isinstance(entries, list):
+ raise EvaluationError("validated plan lost its entries")
+ for entry in entries[: ordinal - 1]:
+ validate_run(str(entry["run_id"]), plan=plan, freeze=freeze)
+
+
+def _expected_metadata_bindings(
+ entry: Mapping[str, Any],
+ *,
+ freeze: Mapping[str, Any],
+ plan: Mapping[str, Any],
+ case: Mapping[str, Any],
+) -> dict[str, Any]:
+ agent = str(entry["agent"])
+ binary_binding = freeze["binary_bindings"][agent]
+ return {
+ "protocol": PROTOCOL_ID,
+ "run_id": entry["run_id"],
+ "evidence_id": entry["evidence_id"],
+ "ordinal": entry["ordinal"],
+ "condition": entry["condition"],
+ "case_id": entry["case_id"],
+ "agent": agent,
+ "repetition": entry["repetition"],
+ "manifest_sha256": baseline._sha256_file(baseline.DEFAULT_MANIFEST_PATH),
+ "gold_manifest_sha256": str(case["gold_sha256"]),
+ "baseline_runner_sha256": baseline._sha256_file(baseline.RUNNER_PATH),
+ "treatment_runner_sha256": baseline._sha256_file(RUNNER_PATH),
+ "treatment_freeze_sha256": freeze["_sha256"],
+ "master_plan_sha256": plan["_sha256"],
+ "protocol_sha256": baseline._sha256_file(baseline.PROTOCOL_PATH),
+ "prompt_template_sha256": baseline._sha256_file(baseline.PROMPT_PATH),
+ "docs_treatment_sha256": _sha256_bytes(baseline.DOCS_TREATMENT.encode()),
+ "rendered_prompt_sha256": entry["rendered_prompt_sha256"],
+ "snapshot_sha256": str(case["snapshot_sha256"]),
+ "requested_model": EXPECTED_MODELS[agent],
+ "binary_sha256": binary_binding["sha256"],
+ "cli_version": binary_binding["cli_version"],
+ "timeout_seconds": baseline.DEFAULT_TIMEOUT_SECONDS,
+ "assigned_payload_is_intervention": False,
+ "post_launch_intervention_class": "I0",
+ }
+
+
+def _write_pre_score_verdict(
+ run_root: Path,
+ *,
+ valid: bool,
+ errors: Sequence[str],
+ metadata: Mapping[str, Any],
+) -> dict[str, Any]:
+ verdict = {
+ "kind": VERDICT_KIND,
+ "protocol": PROTOCOL_ID,
+ "run_id": metadata.get("run_id"),
+ "valid": valid,
+ "errors": list(errors),
+ "terminal_state": metadata.get("terminal_state"),
+ "timed_out": metadata.get("timed_out"),
+ "exit_code": metadata.get("exit_code"),
+ "metadata_sha256": baseline._sha256_file(run_root / "metadata.json"),
+ "transcript_sha256": metadata.get("transcript_sha256"),
+ "created_at": baseline._utc_now(),
+ }
+ path = run_root / "pre-score-verdict.json"
+ if path.exists():
+ raise EvaluationError(f"refusing to overwrite run verdict {path}")
+ path.write_bytes(_json_bytes(verdict))
+ return verdict
+
+
+def validate_run(
+ run_id: str,
+ *,
+ plan: Mapping[str, Any] | None = None,
+ freeze: Mapping[str, Any] | None = None,
+) -> dict[str, Any]:
+ if freeze is None:
+ freeze = validate_freeze(DEFAULT_FREEZE_PATH)
+ if plan is None:
+ raw_plan = freeze.get("_plan")
+ if not isinstance(raw_plan, dict):
+ raise EvaluationError("validated freeze lost its master plan")
+ plan = raw_plan
+ entry = _entry_by_run_id(plan, run_id)
+ run_root = _run_root(run_id)
+ metadata_path = _private_file(run_root / "metadata.json", "run metadata")
+ metadata, _ = _read_json_once(metadata_path, "run metadata")
+ manifest = baseline.validate_manifest(baseline.DEFAULT_MANIFEST_PATH)
+ case = baseline._case_index(manifest)[str(entry["case_id"])]
+ errors: list[str] = []
+ for field, expected in _expected_metadata_bindings(
+ entry, freeze=freeze, plan=plan, case=case
+ ).items():
+ if metadata.get(field) != expected:
+ errors.append(f"metadata {field} binding changed")
+ agent = str(entry["agent"])
+ transcript_path = run_root / f"{agent}.jsonl"
+ stderr_path = run_root / f"{agent}.stderr"
+ for artifact in (transcript_path, stderr_path, run_root / "prompt.txt"):
+ if not artifact.is_file() or artifact.is_symlink():
+ errors.append(f"missing required run artifact {artifact.name}")
+ if transcript_path.is_file():
+ actual_transcript_hash = baseline._sha256_file(transcript_path)
+ if metadata.get("transcript_sha256") != actual_transcript_hash:
+ errors.append("transcript hash changed")
+ if stderr_path.is_file():
+ actual_stderr_hash = baseline._sha256_file(stderr_path)
+ if metadata.get("stderr_sha256") != actual_stderr_hash:
+ errors.append("stderr hash changed")
+ prompt_path = run_root / "prompt.txt"
+ if prompt_path.is_file() and baseline._sha256_file(prompt_path) != entry.get(
+ "rendered_prompt_sha256"
+ ):
+ errors.append("rendered prompt artifact changed")
+ if metadata.get("model_identifier") != EXPECTED_MODELS[agent]:
+ errors.append("reported model identifier changed")
+ expected_skill = entry["condition"] == "skill"
+ if metadata.get("skill_injected") is not expected_skill:
+ errors.append("skill injection flag changed")
+ if metadata.get("frozen_skill_sha256") != plan["skill"]["sha256"]:
+ errors.append("frozen skill metadata binding changed")
+ surface = metadata.get("native_surface")
+ if not isinstance(surface, dict):
+ errors.append("native skill surface audit is missing")
+ else:
+ expected_kind = (
+ "isolated-codex-home-skill-v1"
+ if agent == "codex"
+ else "isolated-claude-plugin-skill-v1"
+ )
+ if surface.get("kind") != expected_kind:
+ errors.append("native skill surface kind changed")
+ if surface.get("skill_installed") is not expected_skill:
+ errors.append("native skill surface condition changed")
+ if surface.get("semantic_file_count") != (1 if expected_skill else 0):
+ errors.append("native skill semantic file count changed")
+ expected_skill_hash = plan["skill"]["sha256"] if expected_skill else None
+ if surface.get("skill_sha256") != expected_skill_hash:
+ errors.append("native skill surface hash changed")
+ final_audit = surface.get("final_audit")
+ if not isinstance(final_audit, dict) or final_audit.get("mutated") is not False:
+ errors.append("native skill surface final audit failed")
+ if agent == "claude":
+ catalog_audit = surface.get("provider_catalog_audit")
+ if not isinstance(catalog_audit, dict) or catalog_audit.get("valid") is not True:
+ errors.append("Claude provider catalog audit failed")
+ terminal_state = metadata.get("terminal_state")
+ timed_out = metadata.get("timed_out")
+ if timed_out is not True and terminal_state != "completed":
+ errors.append("provider subprocess did not produce a completed terminal event")
+ fixtures = metadata.get("fixtures")
+ if not isinstance(fixtures, dict):
+ errors.append("fixture audit is missing")
+ else:
+ for name, fixture in fixtures.items():
+ if not isinstance(fixture, dict):
+ errors.append(f"fixture {name} audit is invalid")
+ continue
+ if fixture.get("kind") == "none":
+ continue
+ if fixture.get("mutated") is not False:
+ errors.append(f"fixture {name} mutated")
+ if fixture.get("final_tree_sha256") != fixture.get("initial_tree_sha256"):
+ errors.append(f"fixture {name} final hash changed")
+ verdict_path = run_root / "pre-score-verdict.json"
+ if not verdict_path.is_file():
+ raise EvaluationError(f"run {run_id!r} is missing its pre-score verdict")
+ verdict, _ = _read_json_once(verdict_path, "pre-score verdict")
+ if verdict.get("kind") != VERDICT_KIND or verdict.get("run_id") != run_id:
+ errors.append("pre-score verdict identity changed")
+ if verdict.get("metadata_sha256") != baseline._sha256_file(metadata_path):
+ errors.append("pre-score verdict metadata hash changed")
+ if verdict.get("transcript_sha256") != metadata.get("transcript_sha256"):
+ errors.append("pre-score verdict transcript hash changed")
+ if verdict.get("valid") is not True or verdict.get("errors") != []:
+ errors.append("pre-score verdict did not record a valid run")
+ if errors:
+ raise EvaluationError("; ".join(errors))
+ return verdict
+
+
+def _execute_run_once(
+ run_id: str,
+ *,
+ timeout_seconds: int = baseline.DEFAULT_TIMEOUT_SECONDS,
+) -> dict[str, Any]:
+ if timeout_seconds != baseline.DEFAULT_TIMEOUT_SECONDS:
+ raise EvaluationError(
+ "official treatment timeout must remain " f"{baseline.DEFAULT_TIMEOUT_SECONDS} seconds"
+ )
+ freeze = validate_freeze(DEFAULT_FREEZE_PATH)
+ raw_plan = freeze.get("_plan")
+ if not isinstance(raw_plan, dict):
+ raise EvaluationError("validated freeze lost its master plan")
+ plan = raw_plan
+ entry = _entry_by_run_id(plan, run_id)
+ _validate_prior_verdicts(plan, freeze, int(entry["ordinal"]))
+
+ manifest = baseline.validate_manifest(baseline.DEFAULT_MANIFEST_PATH)
+ case = baseline._case_index(manifest)[str(entry["case_id"])]
+ if case.get("status") != "ready":
+ raise EvaluationError(f"case {entry['case_id']!r} is not ready")
+ run_root = _run_root(run_id)
+ if run_root.exists() or run_root.is_symlink():
+ raise EvaluationError(f"refusing to overwrite existing run {run_root}")
+
+ agent = str(entry["agent"])
+ condition = str(entry["condition"])
+ source = (REPOSITORY_ROOT / str(case["source"])).resolve(strict=True)
+ work_root = baseline.CODEX_WORK_ROOT if agent == "codex" else baseline.CLAUDE_WORK_ROOT
+ external_run_root = work_root / run_id
+ if external_run_root.exists() or external_run_root.is_symlink():
+ raise EvaluationError(f"refusing to reuse external workspace {external_run_root}")
+ workspace = external_run_root / "workspace"
+ scratch = external_run_root / "scratch"
+ transcript_path = run_root / f"{agent}.jsonl"
+ stderr_path = run_root / f"{agent}.stderr"
+
+ run_root.mkdir(parents=True)
+ scratch.mkdir(parents=True)
+ baseline._initialize_snapshot(source, workspace)
+ prompt = render_prompt(case)
+ if _sha256_bytes(prompt.encode()) != entry["rendered_prompt_sha256"]:
+ raise EvaluationError("rendered treatment prompt changed before launch")
+ (run_root / "prompt.txt").write_text(prompt)
+ environment = baseline._clean_agent_environment(scratch)
+ fixtures = baseline._prepare_case_fixtures(case, scratch)
+ skill_content = plan.get("_skill_content")
+ if not isinstance(skill_content, bytes):
+ raise EvaluationError("validated plan lost the frozen skill bytes")
+ command, native_surface, plugin_root = _prepare_native_surface(
+ agent=agent,
+ condition=condition,
+ workspace=workspace,
+ external_run_root=external_run_root,
+ scratch=scratch,
+ environment=environment,
+ skill_content=skill_content,
+ )
+ settings_path: Path | None = None
+ if agent == "claude":
+ if plugin_root is None:
+ raise EvaluationError("Claude native plugin root was not prepared")
+ settings_path = run_root / "claude-settings.json"
+ baseline._write_json(
+ settings_path,
+ _claude_treatment_settings(workspace, scratch, plugin_root),
+ )
+ command = _claude_treatment_command(settings_path, plugin_root)
+
+ binary = baseline.CODEX_BINARY if agent == "codex" else baseline.CLAUDE_BINARY
+ cli_version = baseline._binary_version(binary, environment)
+ started_at = baseline._utc_now()
+ bindings = _expected_metadata_bindings(entry, freeze=freeze, plan=plan, case=case)
+ metadata: dict[str, Any] = {
+ **bindings,
+ "started_at": started_at,
+ "timeout_seconds": timeout_seconds,
+ "command": command,
+ "prompt_transport": "stdin",
+ "cli_version": cli_version,
+ "binary_sha256": baseline._sha256_file(binary),
+ "removed_environment_names": sorted(baseline.SECRET_ENVIRONMENT_NAMES),
+ "fixtures": fixtures,
+ "native_surface": native_surface,
+ "skill_injected": condition == "skill",
+ "frozen_skill_sha256": plan["skill"]["sha256"],
+ "treatment_payload_transport": "native local skill mechanism",
+ }
+ baseline._write_json(run_root / "metadata.json", metadata)
+
+ exit_code, timed_out, elapsed_seconds = baseline._run_subprocess(
+ command,
+ cwd=workspace,
+ environment=environment,
+ prompt_stdin=prompt,
+ stdout_path=transcript_path,
+ stderr_path=stderr_path,
+ timeout_seconds=timeout_seconds,
+ )
+ native_surface["final_audit"] = _audit_native_surface(
+ agent=agent,
+ condition=condition,
+ external_run_root=external_run_root,
+ scratch=scratch,
+ skill_content=skill_content,
+ )
+ baseline._write_workspace_evidence(workspace, run_root)
+ baseline._record_final_fixture_state(fixtures, scratch)
+ shutil.copytree(
+ workspace,
+ run_root / "workspace-final",
+ symlinks=True,
+ ignore=shutil.ignore_patterns(
+ ".git", "bin", "obj", "target", "node_modules", ".eval-cache"
+ ),
+ )
+ terminal_state = _transcript_terminal_state(transcript_path, agent)
+ if agent == "claude":
+ native_surface["provider_catalog_audit"] = _claude_catalog_audit(
+ transcript_path,
+ condition=condition,
+ )
+ metadata.update(
+ {
+ "completed_at": baseline._utc_now(),
+ "exit_code": exit_code,
+ "timed_out": timed_out,
+ "elapsed_seconds": elapsed_seconds,
+ "terminal_state": terminal_state,
+ "transcript_sha256": baseline._sha256_file(transcript_path),
+ "stderr_sha256": baseline._sha256_file(stderr_path),
+ "fixtures": fixtures,
+ }
+ )
+ model_identifier, model_identifier_source = baseline._reported_model(transcript_path, agent)
+ metadata["model_identifier"] = model_identifier
+ metadata["model_identifier_source"] = model_identifier_source
+ baseline._write_json(run_root / "metadata.json", metadata)
+
+ pre_errors: list[str] = []
+ if model_identifier != EXPECTED_MODELS[agent]:
+ pre_errors.append(f"reported model {model_identifier!r} != {EXPECTED_MODELS[agent]!r}")
+ if timed_out is not True and terminal_state != "completed":
+ pre_errors.append(f"provider terminal state was {terminal_state!r}")
+ if native_surface["final_audit"]["mutated"] is not False:
+ pre_errors.append("native skill surface mutated")
+ if agent == "claude" and native_surface["provider_catalog_audit"]["valid"] is not True:
+ pre_errors.append("Claude provider catalog isolation failed")
+ for name, fixture in fixtures.items():
+ if fixture.get("kind") != "none" and fixture.get("mutated") is not False:
+ pre_errors.append(f"fixture {name} mutated")
+ verdict = _write_pre_score_verdict(
+ run_root,
+ valid=not pre_errors,
+ errors=pre_errors,
+ metadata=metadata,
+ )
+ if pre_errors:
+ raise EvaluationError(f"run {run_id!r} is technically invalid: " + "; ".join(pre_errors))
+ validate_run(run_id, plan=plan, freeze=freeze)
+ shutil.rmtree(external_run_root)
+ return {"metadata": metadata, "verdict": verdict}
+
+
+def execute_run(
+ run_id: str,
+ *,
+ timeout_seconds: int = baseline.DEFAULT_TIMEOUT_SECONDS,
+) -> dict[str, Any]:
+ run_root = _run_root(run_id)
+ parts = run_id.split("__")
+ agent = parts[-2] if len(parts) >= 4 else ""
+ work_root = baseline.CODEX_WORK_ROOT if agent == "codex" else baseline.CLAUDE_WORK_ROOT
+ external_run_root = work_root / run_id
+ run_root_preexisting = run_root.exists() or run_root.is_symlink()
+ external_root_preexisting = external_run_root.exists() or external_run_root.is_symlink()
+ try:
+ return _execute_run_once(run_id, timeout_seconds=timeout_seconds)
+ except BaseException:
+ if not run_root_preexisting and not external_root_preexisting:
+ baseline._quarantine_interrupted_run(run_root, external_run_root)
+ raise
+
+
+def _transcript_result_text(path: Path, agent: str) -> str:
+ result = ""
+ with path.open(encoding="utf-8") as handle:
+ for line in handle:
+ try:
+ event = json.loads(line)
+ except json.JSONDecodeError:
+ continue
+ if not isinstance(event, dict):
+ continue
+ if agent == "codex" and event.get("type") == "item.completed":
+ item = event.get("item")
+ if isinstance(item, dict) and item.get("type") == "agent_message":
+ text = item.get("text")
+ if isinstance(text, str):
+ result = text
+ if agent == "claude" and event.get("type") == "result":
+ text = event.get("result")
+ if isinstance(text, str):
+ result = text
+ return result.strip()
+
+
+def run_native_delivery_shakedown(
+ *,
+ output_path: Path = DEFAULT_SHAKEDOWN_VERDICT_PATH,
+) -> dict[str, Any]:
+ prior_root = PRIVATE_ROOT / "shakedowns" / "native-skill-delivery-20260729"
+ skill_source = _private_file(
+ prior_root / "skill-source" / "SKILL.md", "synthetic shakedown skill"
+ )
+ skill_content = skill_source.read_bytes()
+ skill_hash = _sha256_bytes(skill_content)
+ marker = "NATIVE_SKILL_PROBE_20260729_8C41D6"
+ if marker.encode() not in skill_content:
+ raise EvaluationError("synthetic shakedown skill lost its marker")
+ verified_root = PRIVATE_ROOT / "shakedowns" / "native-skill-delivery-verified-20260729"
+ if verified_root.exists() or verified_root.is_symlink():
+ raise EvaluationError(f"refusing to reuse shakedown root {verified_root}")
+ verified_root.mkdir(parents=True)
+ prompt = (
+ "Run the native-delivery-probe shakedown now. If its native skill is "
+ "available, use it. If unavailable, reply exactly "
+ "NO_NATIVE_DELIVERY. Do not search for skill files or inspect any path "
+ "not supplied by the native skill catalog."
+ )
+ providers: dict[str, Any] = {}
+ for agent in ("codex", "claude"):
+ agent_result: dict[str, Any] = {
+ "synthetic_skill_sha256": skill_hash,
+ }
+ for condition, label in (("docs", "control"), ("skill", "treatment")):
+ root = verified_root / f"{agent}-{label}"
+ workspace = root / "workspace"
+ scratch = root / "scratch"
+ workspace.mkdir(parents=True)
+ scratch.mkdir(parents=True)
+ (workspace / "README.md").write_text("Synthetic native skill delivery probe.\n")
+ environment = baseline._clean_agent_environment(scratch)
+ command, surface, plugin_root = _prepare_native_surface(
+ agent=agent,
+ condition=condition,
+ workspace=workspace,
+ external_run_root=root,
+ scratch=scratch,
+ environment=environment,
+ skill_content=skill_content,
+ skill_name="native-delivery-probe",
+ )
+ if agent == "claude":
+ if plugin_root is None:
+ raise EvaluationError("Claude shakedown plugin was not prepared")
+ settings_path = root / "claude-settings.json"
+ baseline._write_json(
+ settings_path,
+ _claude_treatment_settings(workspace, scratch, plugin_root),
+ )
+ command = _claude_treatment_command(settings_path, plugin_root)
+ transcript = root / f"{agent}.jsonl"
+ stderr = root / f"{agent}.stderr"
+ exit_code, timed_out, elapsed = baseline._run_subprocess(
+ command,
+ cwd=workspace,
+ environment=environment,
+ prompt_stdin=prompt,
+ stdout_path=transcript,
+ stderr_path=stderr,
+ timeout_seconds=600,
+ )
+ surface["final_audit"] = _audit_native_surface(
+ agent=agent,
+ condition=condition,
+ external_run_root=root,
+ scratch=scratch,
+ skill_content=skill_content,
+ skill_name="native-delivery-probe",
+ )
+ if agent == "claude":
+ surface["provider_catalog_audit"] = _claude_catalog_audit(
+ transcript,
+ condition=condition,
+ skill_name="native-delivery-probe",
+ )
+ terminal_state = _transcript_terminal_state(transcript, agent)
+ result = _transcript_result_text(transcript, agent)
+ model, model_source = baseline._reported_model(transcript, agent)
+ expected_result = marker if condition == "skill" else "NO_NATIVE_DELIVERY"
+ if exit_code != 0 or timed_out or terminal_state != "completed":
+ raise EvaluationError(f"{agent} {label} shakedown did not complete cleanly")
+ if surface["final_audit"]["mutated"] is not False:
+ raise EvaluationError(f"{agent} {label} native skill surface mutated")
+ if agent == "claude" and surface["provider_catalog_audit"]["valid"] is not True:
+ raise EvaluationError(f"{agent} {label} provider catalog was not isolated")
+ if result != expected_result:
+ raise EvaluationError(
+ f"{agent} {label} shakedown returned {result!r}, "
+ f"expected {expected_result!r}"
+ )
+ if model != EXPECTED_MODELS[agent]:
+ raise EvaluationError(f"{agent} shakedown model changed: {model!r}")
+ agent_result[f"{label}_result"] = result
+ agent_result[f"{label}_transcript"] = _file_record(transcript)
+ agent_result[f"{label}_stderr"] = _file_record(stderr)
+ agent_result[f"{label}_surface"] = surface
+ agent_result[f"{label}_elapsed_seconds"] = elapsed
+ agent_result[f"{label}_model"] = model
+ agent_result[f"{label}_model_source"] = model_source
+ agent_result["passed"] = True
+ providers[agent] = agent_result
+ verdict = {
+ "kind": SHAKEDOWN_KIND,
+ "protocol": PROTOCOL_ID,
+ "created_at": baseline._utc_now(),
+ "passed": True,
+ "runner": _file_record(RUNNER_PATH),
+ "marker": marker,
+ "synthetic_skill": _file_record(skill_source),
+ "synthetic_skill_sha256": skill_hash,
+ "prompt_sha256": _sha256_bytes(prompt.encode()),
+ "providers": providers,
+ }
+ _write_new_json(output_path, verdict, "native-delivery shakedown verdict")
+ return verdict
+
+
+def _build_parser() -> argparse.ArgumentParser:
+ parser = argparse.ArgumentParser(description=__doc__)
+ subparsers = parser.add_subparsers(dest="command", required=True)
+
+ subparsers.add_parser("shakedown", help="verify native skill delivery on a synthetic case")
+
+ plan = subparsers.add_parser("plan", help="create the immutable 24-run plan")
+ plan.add_argument("--output", type=Path, default=DEFAULT_PLAN_PATH)
+
+ freeze = subparsers.add_parser("freeze", help="freeze all treatment inputs")
+ freeze.add_argument("--output", type=Path, default=DEFAULT_FREEZE_PATH)
+
+ subparsers.add_parser("validate", help="validate the complete treatment freeze")
+
+ run = subparsers.add_parser("run", help="execute the next frozen plan entry")
+ run.add_argument("--run-id", required=True)
+ run.add_argument("--timeout-seconds", type=int, default=baseline.DEFAULT_TIMEOUT_SECONDS)
+
+ validate_run_parser = subparsers.add_parser(
+ "validate-run", help="validate one completed pre-score run"
+ )
+ validate_run_parser.add_argument("--run-id", required=True)
+ return parser
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+ args = _build_parser().parse_args(argv)
+ try:
+ if args.command == "shakedown":
+ payload = run_native_delivery_shakedown()
+ elif args.command == "plan":
+ payload = create_plan(
+ baseline.DEFAULT_MANIFEST_PATH,
+ skill_path=DEFAULT_SKILL_PATH,
+ external_pins_path=DEFAULT_EXTERNAL_PINS_PATH,
+ protocol_amendment_path=DEFAULT_PROTOCOL_AMENDMENT_PATH,
+ authorization_path=DEFAULT_AUTHORIZATION_PATH,
+ claude_wrapper_path=DEFAULT_CLAUDE_WRAPPER_PATH,
+ output_path=args.output,
+ )
+ elif args.command == "freeze":
+ payload = create_freeze(
+ args.output,
+ plan_path=DEFAULT_PLAN_PATH,
+ shakedown_verdict_path=DEFAULT_SHAKEDOWN_VERDICT_PATH,
+ )
+ elif args.command == "validate":
+ payload = validate_freeze(DEFAULT_FREEZE_PATH)
+ elif args.command == "run":
+ payload = execute_run(args.run_id, timeout_seconds=args.timeout_seconds)
+ elif args.command == "validate-run":
+ payload = validate_run(args.run_id)
+ else: # pragma: no cover - argparse enforces the command.
+ raise EvaluationError(f"unknown command {args.command!r}")
+ except (EvaluationError, OSError, ValueError, subprocess.SubprocessError) as exc:
+ print(f"error: {exc}", file=sys.stderr)
+ return 2
+ printable = {
+ key: value
+ for key, value in payload.items()
+ if not key.startswith("_") and not isinstance(value, bytes)
+ }
+ print(json.dumps(printable, indent=2, sort_keys=True))
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/integrations/gocronx_matcher/README.md b/integrations/gocronx_matcher/README.md
index 110da8a..f36838d 100644
--- a/integrations/gocronx_matcher/README.md
+++ b/integrations/gocronx_matcher/README.md
@@ -47,8 +47,20 @@ traces, 10/10 covered capabilities, and 28 candidate process runs.
| `clear` | replace the symbol's `OrderBook` |
| full queue state | decode snapshot format version 1, then canonicalize bids descending and asks ascending |
-The adapter scales prices into integer ticks and quantities into configured
-fixed-point units. Source owners are retained in adapter metadata because
+The adapter scales prices into integer ticks and quantities into fixed
+twelve-place native lots. `quantity_decimal_places` controls output rounding
+only; it never rounds submitted order, reduction, or replacement quantities.
+If normalization would turn a positive fill or remainder into zero, the session
+fails with an adapter error before emitting that invalid observation. Use enough
+output precision for the trace; this does not round or resubmit native orders.
+For example, at output precision zero, crossing a `2.4` buy with a `3.5` sell
+leaves native quantity `1.1`, reported as `1`. Nonrepresentable quantities are
+rejected before submission, and invalid replacement quantities preserve the
+existing order. See the shared crate's [numeric boundary](../rust_protocol/README.md)
+for range and precision limits, including accepted configurations that remain
+unqualified.
+
+Source owners are retained in adapter metadata because
`matcher` does not store them, then reconciled against the native snapshot after
every event.
diff --git a/integrations/gocronx_matcher/src/adapter.rs b/integrations/gocronx_matcher/src/adapter.rs
index f9a8861..0b0d39b 100644
--- a/integrations/gocronx_matcher/src/adapter.rs
+++ b/integrations/gocronx_matcher/src/adapter.rs
@@ -1,10 +1,9 @@
use crate::wire::{
- BookSnapshot, BookState, ConfigWire, MarketEvent, ObservationFrame, Outcome, RestingOrder,
- TradeFill,
+ BookSnapshot, BookState, ConfigWire, MarketEvent, ObservationFrame, Outcome, QuantityEncoding,
+ RestingOrder, TradeFill,
};
use matcher::{BookEvent, Order, OrderBook, OrderId, Quantity, RejectReason, Side, Trade};
-use rust_decimal::prelude::ToPrimitive;
-use rust_decimal::{Decimal, RoundingStrategy};
+use rust_decimal::Decimal;
use std::collections::{BTreeMap, BTreeSet};
use std::str::FromStr;
@@ -18,8 +17,7 @@ const KIND_ICEBERG: u8 = 6;
struct AdapterConfig {
tick_size: Decimal,
tick_size_f64: f64,
- quantity_decimal_places: u32,
- quantity_scale: u64,
+ quantity_encoding: QuantityEncoding,
}
impl AdapterConfig {
@@ -30,9 +28,7 @@ impl AdapterConfig {
if config.self_trade_policy != "NONE" {
return Err("self_trade_policy must be NONE".to_string());
}
- if config.quantity_decimal_places > 18 {
- return Err("quantity_decimal_places must be between 0 and 18".to_string());
- }
+ let quantity_encoding = QuantityEncoding::new(config.quantity_decimal_places)?;
let tick_size = Decimal::from_str(&config.tick_size)
.map_err(|_| "tick_size must be a positive finite decimal".to_string())?;
@@ -46,15 +42,10 @@ impl AdapterConfig {
if !tick_size_f64.is_finite() || tick_size_f64 <= 0.0 {
return Err("tick_size must be a positive finite decimal".to_string());
}
- let quantity_scale = 10_u64
- .checked_pow(config.quantity_decimal_places)
- .ok_or_else(|| "quantity scale exceeds u64".to_string())?;
-
Ok(Self {
tick_size,
tick_size_f64,
- quantity_decimal_places: config.quantity_decimal_places,
- quantity_scale,
+ quantity_encoding,
})
}
@@ -74,25 +65,7 @@ impl AdapterConfig {
}
fn quantity_units(&self, value: &serde_json::Number) -> Result {
- let quantity = Decimal::from_str(&value.to_string())
- .map_err(|_| "quantity must be a finite decimal".to_string())?;
- if quantity <= Decimal::ZERO {
- return Err("quantity must be positive".to_string());
- }
- let normalized = quantity.round_dp_with_strategy(
- self.quantity_decimal_places,
- RoundingStrategy::MidpointNearestEven,
- );
- let scaled = normalized
- .checked_mul(Decimal::from(self.quantity_scale))
- .ok_or_else(|| "quantity exceeds the adapter's integer range".to_string())?;
- let units = scaled
- .to_u64()
- .ok_or_else(|| "quantity exceeds the adapter's integer range".to_string())?;
- if units == 0 {
- return Err("quantity rounds to zero at the configured precision".to_string());
- }
- Ok(units)
+ self.quantity_encoding.units(value)
}
fn format_price(&self, ticks: u64) -> Result {
@@ -102,8 +75,8 @@ impl AdapterConfig {
Ok(canonical_decimal(price))
}
- fn format_quantity(&self, units: u64) -> String {
- canonical_decimal(Decimal::from(units) / Decimal::from(self.quantity_scale))
+ fn format_quantity(&self, units: u64) -> Result {
+ self.quantity_encoding.format(units)
}
}
@@ -416,7 +389,7 @@ impl Adapter {
buy_order_id: trade.buy_id.get(),
sell_order_id: trade.sell_id.get(),
price: self.config.format_price(trade.price.get())?,
- quantity: self.config.format_quantity(trade.quantity.get()),
+ quantity: self.config.format_quantity(trade.quantity.get())?,
})
}
@@ -459,7 +432,7 @@ impl Adapter {
Ok(RestingOrder {
order_id: order.order_id,
price: self.config.format_price(order.price)?,
- remaining_quantity: self.config.format_quantity(order.remaining),
+ remaining_quantity: self.config.format_quantity(order.remaining)?,
owner,
order_type: "LIMIT",
})
@@ -615,6 +588,114 @@ mod tests {
serde_json::from_value(value).unwrap()
}
+ #[test]
+ fn output_precision_does_not_change_native_matching() {
+ let mut adapter = Adapter::new(ConfigWire {
+ quantity_decimal_places: 0,
+ ..ConfigWire::default()
+ })
+ .unwrap();
+ let values = [
+ serde_json::json!({"op":"new","symbol":"TEST","order_id":1,"side":"BUY","price":100,"quantity":2.4}),
+ serde_json::json!({"op":"new","symbol":"TEST","order_id":2,"side":"SELL","price":100,"quantity":3.5}),
+ ];
+ let mut last = None;
+ for (index, value) in values.into_iter().enumerate() {
+ let observation = adapter.apply(&event(value), index as u64 + 1).unwrap();
+ assert_eq!(observation.outcome.status, "applied");
+ last = Some(observation);
+ }
+ let observation = last.unwrap();
+ assert_eq!(observation.trades.len(), 1);
+ assert_eq!(observation.trades[0].quantity, "2");
+ assert_eq!(
+ observation.state_hash,
+ "443d82df2af0902942c8614bcce6dc8d763a24db85e81917cc4843a9f4726300"
+ );
+ let state = adapter.snapshot().unwrap();
+ assert_eq!(state.books[0].asks.len(), 1);
+ assert_eq!(state.books[0].asks[0].remaining_quantity, "1");
+ }
+
+ #[test]
+ fn positive_fills_and_remainders_cannot_be_emitted_as_zero() {
+ for (maker, taker) in [(0.4, None), (1.0, Some(0.4)), (1.0, Some(0.6))] {
+ let mut adapter = Adapter::new(ConfigWire {
+ quantity_decimal_places: 0,
+ ..ConfigWire::default()
+ })
+ .unwrap();
+ let first = serde_json::from_value::(serde_json::json!({
+ "op":"new","symbol":"TEST","order_id":1,"side":"BUY","price":100,"quantity":maker
+ }))
+ .unwrap();
+ let result = if let Some(quantity) = taker {
+ adapter.apply(&first, 1).unwrap();
+ let second = serde_json::from_value::(serde_json::json!({
+ "op":"new","symbol":"TEST","order_id":2,"side":"SELL","price":100,"quantity":quantity
+ })).unwrap();
+ adapter.apply(&second, 2)
+ } else {
+ adapter.apply(&first, 1)
+ };
+ assert!(
+ result
+ .unwrap_err()
+ .contains("positive quantity cannot be represented")
+ );
+ }
+ }
+
+ #[test]
+ fn output_precision_does_not_round_reduce_or_replace_inputs() {
+ let mut adapter = Adapter::new(ConfigWire {
+ quantity_decimal_places: 0,
+ ..ConfigWire::default()
+ })
+ .unwrap();
+ for (index, value) in [
+ serde_json::json!({"op":"new","symbol":"TEST","order_id":1,"side":"BUY","price":100,"quantity":2.4}),
+ serde_json::json!({"op":"reduce","symbol":"TEST","order_id":1,"quantity":0.4}),
+ serde_json::json!({"op":"replace","symbol":"TEST","order_id":1,"quantity":3.5}),
+ serde_json::json!({"op":"new","symbol":"TEST","order_id":2,"side":"SELL","price":100,"quantity":1.4}),
+ ].into_iter().enumerate() {
+ let observation = adapter.apply(&event(value), index as u64 + 1).unwrap();
+ assert_eq!(observation.outcome.status, "applied");
+ }
+ let state = adapter.snapshot().unwrap();
+ assert_eq!(state.books[0].bids.len(), 1);
+ assert_eq!(state.books[0].bids[0].remaining_quantity, "2");
+ }
+
+ #[test]
+ fn nonrepresentable_quantities_preserve_existing_state() {
+ let mut adapter = Adapter::new(ConfigWire::default()).unwrap();
+ let first = event(serde_json::json!({
+ "op":"new","symbol":"TEST","order_id":1,"side":"BUY","price":100,"quantity":2.4,"owner":9
+ }));
+ let initial = adapter.apply(&first, 1).unwrap();
+ for quantity in ["0.0000000000001", "18446745"] {
+ for op in ["new", "reduce", "replace"] {
+ let next = event(serde_json::json!({
+ "op":op,"symbol":"TEST","order_id":if op == "new" {2} else {1},
+ "side":"BUY","price":100,"quantity":serde_json::Number::from_str(quantity).unwrap()
+ }));
+ let observation = adapter.apply(&next, 2).unwrap();
+ assert_eq!(observation.outcome.status, "rejected");
+ assert_eq!(
+ observation.outcome.reason,
+ Some(if op == "replace" {
+ "INVALID_REPLACEMENT"
+ } else {
+ "INVALID_ORDER"
+ })
+ );
+ assert_eq!(observation.state_hash, initial.state_hash);
+ assert!(observation.trades.is_empty());
+ }
+ }
+ }
+
#[test]
fn snapshot_is_canonical_price_time_order() {
let mut adapter = Adapter::new(ConfigWire::default()).unwrap();
diff --git a/integrations/intrepid_orderbook/README.md b/integrations/intrepid_orderbook/README.md
new file mode 100644
index 0000000..f2d9f83
--- /dev/null
+++ b/integrations/intrepid_orderbook/README.md
@@ -0,0 +1,153 @@
+# intrepidkarthi/orderbook CLOB Integration
+
+This adapter checks a genuine submitted-order versus submitted-order central
+limit order book. Unlike Tracebook's NautilusTrader book-replay integration,
+both sides of every trade are orders sent through the candidate engine.
+
+The external input is pinned exactly:
+
+- Go module: `github.com/intrepidkarthi/orderbook v0.26.0`;
+- tag: `v0.26.0`;
+- source commit: `51d480cdb68b9989febb0b075d291cf891f425b3`;
+- repository: ; and
+- upstream license: MIT.
+
+`go.mod` and `go.sum` lock the module graph. No upstream source or compiled
+binary is vendored in Tracebook.
+
+## What The Adapter Exercises
+
+There is one native `matching.Engine` per symbol. New limit, market, IOC, and
+FOK orders call `Engine.Process`; cancel, reduce, and replace call the native
+methods of the same names. Trades come directly from the native
+`matching.MatchResult`, and snapshots traverse native `RestingOrders`.
+Tracebook source IDs are mapped to engine-generated IDs without changing the
+engine's matching decisions.
+
+The protocol configuration maps FIFO and pro-rata allocation plus `NONE`,
+`CANCEL_RESTING`, and `CANCEL_INCOMING` self-trade policies to their native
+settings. The anonymous owner sentinel (`-1`, also used when owner is omitted)
+maps to a distinct native account for each active source order, so anonymous
+orders remain free to trade under either STP policy. Snapshots retain the
+original owner ID, including across replace and reduce operations.
+
+Prices use binary64 division and ties-to-even tick snapping, matching the
+reference's input boundary, and are formatted using the exact decimal tick
+size. Quantities are represented exactly in native integer lots at six decimal
+places. `quantity_decimal_places` rounds output quantities only; it never
+rounds the quantity submitted to the engine. Inputs requiring more than six
+non-zero decimal places are explicitly rejected even if the configured output
+precision is lower.
+
+The native arithmetic has a bounded domain:
+
+- price ticks and scaled quantities must be positive and fit signed `int64`;
+- before admitting a limit, IOC, or FOK order, the adapter requires its full quantity plus
+ all resting quantity on its side to fit signed `int64` (replacement excludes
+ the old order);
+- under pro-rata, incoming quantity times each potentially crossed maker's
+ remaining quantity must fit signed `int64`; and
+- the native engine's own price-times-quantity notional limit remains in force
+ for limit, IOC, and FOK instructions; the adapter checks it before native
+ submission or replacement so an invalid replacement cannot cancel its old
+ order. Market instructions retain the native exemption.
+
+The aggregate and pro-rata checks are conservative: they reserve the full
+incoming quantity without predicting fills or STP removals. An input outside
+this envelope returns `INVALID_ORDER` (or `INVALID_REPLACEMENT`) before native
+mutation; no claim of conformance is made beyond that envelope. At the default
+scale, for example, a 4,000-unit order crossing another 4,000-unit order is
+rejected under pro-rata because their native lot product exceeds `int64`.
+The same FIFO crossing is supported.
+
+Numeric configurations outside the recorded profiles remain unqualified even
+when their values fit that arithmetic envelope. The adapter preserves native
+integer/decimal results rather than recreating the reference's floating-point
+arithmetic. At `quantity_decimal_places=18`, consuming `0.1` from `0.3` leaves
+native quantity `0.2`, while the binary64 reference reports
+`0.19999999999999998`. With tick size `1e-18`, input price `1` snaps to native
+integer tick `999999999999999872`, formatted as `0.999999999999999872`; the
+reference's floating-point price formatting produces `0.9999999999999999`.
+These configurations are accepted for investigation, but can diverge and have
+not passed qualification. Tests retain those native outputs explicitly.
+
+The local Go tests include a submitted-order crossing test: two same-price
+bids are reduced and then consumed by an incoming sell in native FIFO order.
+That guards the central claim that this is a matching-engine integration, not
+an order-book mirror or a reimplemented reference engine.
+Additional tests cover anonymous STP, replacement owner retention, fractional
+input quantities with zero-place output, binary64 tick boundaries, and native
+arithmetic limits including rejection without replacing an existing order.
+
+## Qualification Results
+
+Using Tracebook 0.6.0, generator version 2, seed `42`, 25 traces of 200 events,
+and the pinned candidate:
+
+| Profile | Fixed | Generated | Events | Coverage | Result |
+| --- | ---: | ---: | ---: | ---: | --- |
+| `fifo-limit-v1` | 3/3 | 25/25 | 5,000 | 10/10 | pass |
+| `fifo-partial-fill-v1` | 3/3 | 25/25 | 5,000 | 10/10 | pass |
+| `fifo-full-v1` | 4/5 | stopped at trace 1, event 7 | 7 before stop | n/a | expected divergence |
+
+The passing proof IDs are:
+
+- FIFO limit qualification `sha256:f8db775baabe7e665d45bbb920a67e43d405acf445de4951617df68ffa24eb69`,
+ campaign `sha256:59d70645ff13f12fa4af23af69631714df22ef5f25cd1104b0c1124f98f71f6a`;
+- FIFO partial-fill qualification `sha256:08bf641bfa57da1892d744c08bca0be00f5ac1ade580ae8fcf7f195f0fcad6eb`,
+ campaign `sha256:297f895b99ac26f92287ababd1dcc25b68b0be98787e9b9d814d5cd241769cf9`.
+
+The nine-case standard suite passes 6/9 cases. All three stopped cases reach
+the same exact difference: an unfillable FOK is `applied` with no reason in the
+Tracebook reference contract, while the native engine returns a rejected
+status and its `ErrFOKCannotFill` reason. The adapter preserves that native
+result as `rejected / INVALID_ORDER`; it does not rewrite the result to obtain
+a green report. This is a contract difference, not by itself proof of an
+upstream defect.
+
+## Retained Divergence
+
+The full-profile campaign reduces the FOK difference from seven events to the
+single event in
+[`regressions/fok-rejection-reduced.jsonl`](regressions/fok-rejection-reduced.jsonl).
+The retained
+[`regressions/fok-rejection-failure.json`](regressions/fok-rejection-failure.json)
+records failure ID `failure-1ce2954857800b3a068d`, the original and reduced
+trace hashes, and `$.observation.outcome.reason` as the semantic path. CI
+regenerates and byte-compares both artifacts.
+
+## Reproduce
+
+From this directory:
+
+```bash
+go test ./...
+go build -trimpath -o /tmp/tracebook-intrepid-orderbook .
+
+tracebook-conformance qualify \
+ --profile fifo-limit-v1 \
+ --seed 42 \
+ --traces 25 \
+ --events-per-trace 200 \
+ --max-minimize-runs 100 \
+ --candidate-cmd /tmp/tracebook-intrepid-orderbook \
+ --output-dir /tmp/intrepid-fifo-limit
+```
+
+For the identity-bound evidence workflow, capture the candidate source snapshot
+first and inject its recorded revision and snapshot ID when building:
+
+```bash
+go build -trimpath \
+ -ldflags "-X main.engineRevision=$CANDIDATE_REVISION -X main.engineSnapshot=$CANDIDATE_SNAPSHOT_ID" \
+ -o /tmp/tracebook-intrepid-orderbook .
+```
+
+These optional build variables appear as `engine.revision` and
+`engine.snapshot_id` in the ready handshake. Both are omitted from ordinary
+builds. They must describe the captured source used for that build; the adapter
+does not discover or verify the source snapshot itself.
+
+The tag is reproducible evidence, not a production-readiness certification.
+This integration makes no claim about upstream adoption, operational maturity,
+latency, durability, or suitability for live trading.
diff --git a/integrations/intrepid_orderbook/adapter.go b/integrations/intrepid_orderbook/adapter.go
new file mode 100644
index 0000000..7de94e0
--- /dev/null
+++ b/integrations/intrepid_orderbook/adapter.go
@@ -0,0 +1,578 @@
+package main
+
+import (
+ "encoding/json"
+ "errors"
+ "fmt"
+ "math"
+ "sort"
+ "strconv"
+ "time"
+
+ "github.com/intrepidkarthi/orderbook/pkg/matching"
+ "github.com/intrepidkarthi/orderbook/pkg/types"
+ "github.com/shopspring/decimal"
+)
+
+type adapterConfig struct {
+ matchingAlgorithm string
+ tickSize decimal.Decimal
+ tickSizeFloat float64
+ quantityPlaces uint32
+ nativeQtyPlaces uint32
+ quantityScale int64
+ stp matching.SelfTradePrevention
+}
+
+func parseConfig(wire configWire) (adapterConfig, error) {
+ algorithm := wire.MatchingAlgorithm
+ if algorithm == "" {
+ algorithm = "fifo"
+ }
+ if algorithm != "fifo" && algorithm != "pro_rata" {
+ return adapterConfig{}, errors.New("matching_algorithm must be 'fifo' or 'pro_rata'")
+ }
+ tickText := wire.TickSize
+ if tickText == "" {
+ tickText = "0.01"
+ }
+ tick, err := decimal.NewFromString(tickText)
+ if err != nil || !tick.IsPositive() {
+ return adapterConfig{}, errors.New("tick_size must be a positive finite decimal")
+ }
+ tickFloat, err := strconv.ParseFloat(tickText, 64)
+ if err != nil || math.IsInf(tickFloat, 0) || math.IsNaN(tickFloat) || tickFloat <= 0 {
+ return adapterConfig{}, errors.New("tick_size must be a positive finite binary64 value")
+ }
+ places := uint32(12)
+ if wire.QuantityDecimalPlaces != nil {
+ places = *wire.QuantityDecimalPlaces
+ }
+ if places > 18 {
+ return adapterConfig{}, errors.New("quantity_decimal_places must be between 0 and 18")
+ }
+ // Protocol precision controls observations, not submitted quantities.
+ // Keep six native decimal places regardless of that output setting, and
+ // reject inputs that would require rounding before the engine sees them.
+ const nativePlaces = uint32(6)
+ const scale = int64(1_000_000)
+ policy := wire.SelfTradePolicy
+ if policy == "" {
+ policy = "NONE"
+ }
+ var stp matching.SelfTradePrevention
+ switch policy {
+ case "NONE":
+ stp = matching.STPAllow
+ case "CANCEL_RESTING":
+ stp = matching.STPCancelOldest
+ case "CANCEL_INCOMING":
+ stp = matching.STPCancelNewest
+ default:
+ return adapterConfig{}, errors.New(
+ "self_trade_policy must be NONE, CANCEL_RESTING, or CANCEL_INCOMING",
+ )
+ }
+ return adapterConfig{
+ matchingAlgorithm: algorithm, tickSize: tick, tickSizeFloat: tickFloat,
+ quantityPlaces: places, nativeQtyPlaces: nativePlaces, quantityScale: scale, stp: stp,
+ }, nil
+}
+
+func (config adapterConfig) priceTicks(value *jsonNumber) (int64, error) {
+ if value == nil {
+ return 0, errors.New("price is required")
+ }
+ price, err := strconv.ParseFloat(value.String(), 64)
+ if err != nil || math.IsInf(price, 0) || math.IsNaN(price) {
+ return 0, errors.New("price must be finite")
+ }
+ // The reference snaps the normalized binary64 input with ties to even.
+ // Decimal division differs at boundaries such as 1.015 / 0.01.
+ ticks := math.RoundToEven(price / config.tickSizeFloat)
+ if math.IsInf(ticks, 0) || math.IsNaN(ticks) || ticks <= 0 || ticks >= math.Exp2(63) {
+ return 0, errors.New("price snaps to a non-positive or unsupported tick")
+ }
+ return int64(ticks), nil
+}
+
+// jsonNumber preserves the wire text for each explicit numeric conversion.
+type jsonNumber = json.Number
+
+func (config adapterConfig) quantityUnits(value *jsonNumber) (int64, error) {
+ if value == nil {
+ return 0, errors.New("quantity is required")
+ }
+ quantity, err := decimal.NewFromString(value.String())
+ if err != nil || !quantity.IsPositive() {
+ return 0, errors.New("quantity must be positive")
+ }
+ scaled := quantity.Mul(
+ decimal.NewFromInt(config.quantityScale),
+ )
+ if !scaled.Equal(scaled.Truncate(0)) {
+ return 0, fmt.Errorf(
+ "quantity requires more than %d native decimal places",
+ config.nativeQtyPlaces,
+ )
+ }
+ units := scaled.BigInt()
+ if !units.IsInt64() || units.Sign() <= 0 {
+ return 0, errors.New("quantity exceeds the adapter's integer range")
+ }
+ return units.Int64(), nil
+}
+
+func (config adapterConfig) formatPrice(ticks int64) string {
+ return decimal.NewFromInt(ticks).Mul(config.tickSize).String()
+}
+
+func (config adapterConfig) formatQuantity(units int64) string {
+ return decimal.NewFromInt(units).Shift(-int32(config.nativeQtyPlaces)).
+ RoundBank(int32(config.quantityPlaces)).String()
+}
+
+type bookHarness struct {
+ engine *matching.Engine
+ sourceToNative map[int64]int64
+ nativeToSource map[int64]int64
+ nativeToOwner map[int64]int64
+}
+
+func newBook(symbol string, config adapterConfig) *bookHarness {
+ nativeConfig := matching.DefaultConfig(symbol)
+ nativeConfig.SelfTradePrevention = config.stp
+ nativeConfig.ProRata = config.matchingAlgorithm == "pro_rata"
+ nativeConfig.Clock = func() time.Time { return time.Unix(0, 0).UTC() }
+ return &bookHarness{
+ engine: matching.NewEngine(nativeConfig),
+ sourceToNative: make(map[int64]int64),
+ nativeToSource: make(map[int64]int64),
+ nativeToOwner: make(map[int64]int64),
+ }
+}
+
+func (book *bookHarness) reconcile() {
+ active := make(map[int64]bool)
+ for _, order := range book.engine.RestingOrders() {
+ active[order.ID] = true
+ }
+ for nativeID, sourceID := range book.nativeToSource {
+ if !active[nativeID] {
+ delete(book.nativeToSource, nativeID)
+ delete(book.nativeToOwner, nativeID)
+ if book.sourceToNative[sourceID] == nativeID {
+ delete(book.sourceToNative, sourceID)
+ }
+ }
+ }
+}
+
+type adapter struct {
+ config adapterConfig
+ books map[string]*bookHarness
+}
+
+func newAdapter(wire configWire) (*adapter, error) {
+ config, err := parseConfig(wire)
+ if err != nil {
+ return nil, err
+ }
+ return &adapter{config: config, books: make(map[string]*bookHarness)}, nil
+}
+
+func (adapter *adapter) book(symbol string) *bookHarness {
+ book := adapter.books[symbol]
+ if book == nil {
+ book = newBook(symbol, adapter.config)
+ adapter.books[symbol] = book
+ }
+ return book
+}
+
+func parseOrderID(event marketEvent) (int64, error) {
+ if event.OrderID == nil {
+ return 0, errors.New("order_id is required")
+ }
+ value, err := strconv.ParseInt(event.OrderID.String(), 10, 64)
+ if err != nil || value <= 0 {
+ return 0, errors.New("order_id must be a positive integer")
+ }
+ return value, nil
+}
+
+func eventOwner(event marketEvent) int64 {
+ if event.Owner == nil {
+ return -1
+ }
+ return *event.Owner
+}
+
+func nativeOwner(sourceID, owner int64) string {
+ if owner == -1 {
+ // Anonymous source orders never self-match. Give each active anonymous
+ // order its own native account, disjoint from every numeric owner id.
+ return "anonymous:" + strconv.FormatInt(sourceID, 10)
+ }
+ return strconv.FormatInt(owner, 10)
+}
+
+// checkQuantityCapacity rejects unsupported native arithmetic before mutation.
+// It reserves the entire incoming limit quantity on its side, even if some
+// would execute immediately, and checks each potentially crossed pro-rata
+// maker conservatively using the entire incoming quantity. This is a numeric
+// envelope, not a replacement for the engine's liquidity or matching decisions.
+func (book *bookHarness) checkQuantityCapacity(
+ config adapterConfig, side types.Side, price, quantity int64,
+ orderType string, replacedID int64,
+) error {
+ // Process checks this for native limit orders (including IOC/FOK). Repeat
+ // the same representability check before Replace can cancel its old order.
+ // Market orders have no submitted notional and retain the native exemption.
+ if orderType != "MARKET" && price > math.MaxInt64/quantity {
+ return types.ErrNotionalOverflow
+ }
+ var sameSideTotal int64
+ for _, resting := range book.engine.RestingOrders() {
+ if resting.ID == replacedID {
+ continue
+ }
+ if resting.Side == side {
+ if resting.RemainingQty > math.MaxInt64-sameSideTotal {
+ return errors.New("resting quantity exceeds the native int64 aggregate range")
+ }
+ sameSideTotal += resting.RemainingQty
+ continue
+ }
+ crosses := orderType == "MARKET" ||
+ (side == types.SideBuy && price >= resting.Price) ||
+ (side == types.SideSell && price <= resting.Price)
+ if config.matchingAlgorithm == "pro_rata" && crosses &&
+ quantity > math.MaxInt64/resting.RemainingQty {
+ return errors.New("quantity exceeds the native int64 pro-rata product range")
+ }
+ }
+ if orderType != "MARKET" && quantity > math.MaxInt64-sameSideTotal {
+ return errors.New("quantity exceeds the native int64 side aggregate range")
+ }
+ return nil
+}
+
+func nativeSide(value *string) (types.Side, error) {
+ if value == nil {
+ return "", errors.New("side is required")
+ }
+ switch *value {
+ case "BUY":
+ return types.SideBuy, nil
+ case "SELL":
+ return types.SideSell, nil
+ default:
+ return "", errors.New("side must be BUY or SELL")
+ }
+}
+
+func sourceID(book *bookHarness, nativeID int64) (int64, error) {
+ value, ok := book.nativeToSource[nativeID]
+ if !ok {
+ return 0, fmt.Errorf("adapter has no source id for native order %d", nativeID)
+ }
+ return value, nil
+}
+
+func (adapter *adapter) convertTrades(
+ book *bookHarness,
+ symbol string,
+ trades []*types.Trade,
+) ([]tradeFill, error) {
+ converted := make([]tradeFill, 0, len(trades))
+ for _, trade := range trades {
+ buyID, err := sourceID(book, trade.BuyOrderID)
+ if err != nil {
+ return nil, err
+ }
+ sellID, err := sourceID(book, trade.SellOrderID)
+ if err != nil {
+ return nil, err
+ }
+ converted = append(converted, tradeFill{
+ Symbol: symbol, BuyOrderID: buyID, SellOrderID: sellID,
+ Price: adapter.config.formatPrice(trade.Price),
+ Quantity: adapter.config.formatQuantity(trade.Quantity),
+ })
+ }
+ return converted, nil
+}
+
+func (adapter *adapter) apply(event marketEvent, index uint64) (observationFrame, error) {
+ result := applied()
+ var trades []tradeFill
+ if event.Op == "clear" {
+ adapter.books[event.Symbol] = newBook(event.Symbol, adapter.config)
+ } else if event.Op == "cancel" || event.Op == "reduce" || event.Op == "replace" {
+ book := adapter.books[event.Symbol]
+ if book == nil {
+ result = rejected("ORDER_NOT_ACTIVE", "order is not active")
+ } else {
+ var err error
+ result, trades, err = adapter.applyLifecycle(book, event)
+ if err != nil {
+ return observationFrame{}, err
+ }
+ }
+ } else if event.Op == "new" {
+ book := adapter.book(event.Symbol)
+ var err error
+ result, trades, err = adapter.applyNew(book, event)
+ if err != nil {
+ return observationFrame{}, err
+ }
+ } else {
+ result = rejected("INVALID_ORDER", "unsupported event operation")
+ }
+
+ state, err := adapter.snapshot()
+ if err != nil {
+ return observationFrame{}, err
+ }
+ digest, err := state.digest()
+ if err != nil {
+ return observationFrame{}, err
+ }
+ if trades == nil {
+ trades = []tradeFill{}
+ }
+ return observationFrame{
+ Type: "observation", Index: index, Outcome: result, Trades: trades,
+ StateHash: digest, RestingOrderCount: state.orderCount(),
+ }, nil
+}
+
+func (adapter *adapter) applyNew(
+ book *bookHarness,
+ event marketEvent,
+) (outcome, []tradeFill, error) {
+ source, err := parseOrderID(event)
+ if err != nil {
+ return rejected("INVALID_ORDER", err.Error()), nil, nil
+ }
+ if _, exists := book.sourceToNative[source]; exists {
+ return rejected("DUPLICATE_ORDER_ID", "order is already active"), nil, nil
+ }
+ side, err := nativeSide(event.Side)
+ if err != nil {
+ return rejected("INVALID_ORDER", err.Error()), nil, nil
+ }
+ quantity, err := adapter.config.quantityUnits(event.Quantity)
+ if err != nil {
+ return rejected("INVALID_ORDER", err.Error()), nil, nil
+ }
+ orderType := event.OrderType
+ if orderType == "" {
+ orderType = "LIMIT"
+ }
+ price := int64(0)
+ if orderType != "MARKET" {
+ price, err = adapter.config.priceTicks(event.Price)
+ if err != nil {
+ return rejected("INVALID_ORDER", err.Error()), nil, nil
+ }
+ }
+ var nativeType types.OrderType
+ var tif types.TimeInForce
+ switch orderType {
+ case "LIMIT":
+ nativeType, tif = types.OrderTypeLimit, types.TIFGoodTillCancel
+ case "MARKET":
+ nativeType, tif = types.OrderTypeMarket, types.TIFImmediateOrCancel
+ case "IOC":
+ nativeType, tif = types.OrderTypeLimit, types.TIFImmediateOrCancel
+ case "FOK":
+ nativeType, tif = types.OrderTypeLimit, types.TIFFillOrKill
+ default:
+ return rejected("INVALID_ORDER", "unsupported order_type"), nil, nil
+ }
+ if err := book.checkQuantityCapacity(adapter.config, side, price, quantity, orderType, 0); err != nil {
+ return rejected("INVALID_ORDER", err.Error()), nil, nil
+ }
+ sourceOwner := eventOwner(event)
+ owner := nativeOwner(source, sourceOwner)
+ order, err := types.NewOrder(owner, event.Symbol, side, nativeType, price, quantity, tif)
+ if err != nil {
+ return rejected("INVALID_ORDER", err.Error()), nil, nil
+ }
+ order.ClientOrderID = strconv.FormatInt(source, 10)
+ match := book.engine.Process(order)
+ book.nativeToSource[order.ID] = source
+ book.nativeToOwner[order.ID] = sourceOwner
+ converted, err := adapter.convertTrades(book, event.Symbol, match.Trades)
+ if err != nil {
+ return outcome{}, nil, err
+ }
+ if order.IsActive() && order.RemainingQty > 0 {
+ book.sourceToNative[source] = order.ID
+ }
+ book.reconcile()
+ if match.Status == types.OrderStatusRejected {
+ message := "order was rejected"
+ if match.RejectionReason != nil {
+ message = match.RejectionReason.Error()
+ }
+ return rejected("INVALID_ORDER", message), converted, nil
+ }
+ return applied(), converted, nil
+}
+
+func (adapter *adapter) applyLifecycle(
+ book *bookHarness,
+ event marketEvent,
+) (outcome, []tradeFill, error) {
+ source, err := parseOrderID(event)
+ if err != nil {
+ return rejected("ORDER_NOT_ACTIVE", err.Error()), nil, nil
+ }
+ nativeID, exists := book.sourceToNative[source]
+ if !exists {
+ return rejected("ORDER_NOT_ACTIVE", "order is not active"), nil, nil
+ }
+ orders := book.engine.RestingOrders()
+ var current *types.Order
+ for _, order := range orders {
+ if order.ID == nativeID {
+ current = order
+ break
+ }
+ }
+ if current == nil {
+ book.reconcile()
+ return rejected("ORDER_NOT_ACTIVE", "order is not active"), nil, nil
+ }
+ owner := current.UserID
+ sourceOwner, ok := book.nativeToOwner[nativeID]
+ if !ok {
+ return outcome{}, nil, fmt.Errorf("adapter has no source owner for native order %d", nativeID)
+ }
+ switch event.Op {
+ case "cancel":
+ if _, err := book.engine.Cancel(nativeID, owner); err != nil {
+ return rejected("ORDER_NOT_ACTIVE", err.Error()), nil, nil
+ }
+ book.reconcile()
+ return applied(), nil, nil
+ case "reduce":
+ reduction, err := adapter.config.quantityUnits(event.Quantity)
+ if err != nil || reduction > current.RemainingQty {
+ message := "reduction exceeds remaining quantity"
+ if err != nil {
+ message = err.Error()
+ }
+ return rejected("INVALID_ORDER", message), nil, nil
+ }
+ if reduction == current.RemainingQty {
+ if _, err := book.engine.Cancel(nativeID, owner); err != nil {
+ return rejected("ORDER_NOT_ACTIVE", err.Error()), nil, nil
+ }
+ } else {
+ newTotal := current.Quantity - reduction
+ if _, err := book.engine.Reduce(nativeID, newTotal, owner); err != nil {
+ return rejected("INVALID_ORDER", err.Error()), nil, nil
+ }
+ }
+ book.reconcile()
+ return applied(), nil, nil
+ case "replace":
+ price := current.Price
+ if event.Price != nil {
+ price, err = adapter.config.priceTicks(event.Price)
+ if err != nil {
+ return rejected("INVALID_REPLACEMENT", err.Error()), nil, nil
+ }
+ }
+ quantity := current.RemainingQty
+ if event.Quantity != nil {
+ quantity, err = adapter.config.quantityUnits(event.Quantity)
+ if err != nil {
+ return rejected("INVALID_REPLACEMENT", err.Error()), nil, nil
+ }
+ }
+ if err := book.checkQuantityCapacity(
+ adapter.config, current.Side, price, quantity, "LIMIT", nativeID,
+ ); err != nil {
+ return rejected("INVALID_REPLACEMENT", err.Error()), nil, nil
+ }
+ replacement, err := types.NewOrder(
+ owner, event.Symbol, current.Side, types.OrderTypeLimit,
+ price, quantity, types.TIFGoodTillCancel,
+ )
+ if err != nil {
+ return rejected("INVALID_REPLACEMENT", err.Error()), nil, nil
+ }
+ replacement.ClientOrderID = strconv.FormatInt(source, 10)
+ match, err := book.engine.Replace(nativeID, owner, replacement)
+ if err != nil {
+ return rejected("INVALID_REPLACEMENT", err.Error()), nil, nil
+ }
+ book.nativeToSource[replacement.ID] = source
+ book.nativeToOwner[replacement.ID] = sourceOwner
+ converted, err := adapter.convertTrades(book, event.Symbol, match.Trades)
+ if err != nil {
+ return outcome{}, nil, err
+ }
+ delete(book.sourceToNative, source)
+ if replacement.IsActive() && replacement.RemainingQty > 0 {
+ book.sourceToNative[source] = replacement.ID
+ }
+ book.reconcile()
+ if match.Status == types.OrderStatusRejected {
+ message := "replacement was rejected"
+ if match.RejectionReason != nil {
+ message = match.RejectionReason.Error()
+ }
+ return rejected("INVALID_REPLACEMENT", message), converted, nil
+ }
+ return applied(), converted, nil
+ default:
+ return rejected("INVALID_ORDER", "unsupported lifecycle operation"), nil, nil
+ }
+}
+
+func (adapter *adapter) snapshot() (bookState, error) {
+ symbols := make([]string, 0, len(adapter.books))
+ for symbol := range adapter.books {
+ symbols = append(symbols, symbol)
+ }
+ sort.Strings(symbols)
+ state := bookState{Books: make([]bookSnapshot, 0, len(symbols))}
+ for _, symbol := range symbols {
+ book := adapter.books[symbol]
+ snapshot := bookSnapshot{Symbol: symbol, Bids: []restingOrder{}, Asks: []restingOrder{}}
+ for _, native := range book.engine.RestingOrders() {
+ source, ok := book.nativeToSource[native.ID]
+ if !ok {
+ return bookState{}, fmt.Errorf(
+ "adapter has no source id for resting native order %d",
+ native.ID,
+ )
+ }
+ owner, ok := book.nativeToOwner[native.ID]
+ if !ok {
+ return bookState{}, fmt.Errorf(
+ "adapter has no source owner for native order %d",
+ native.ID,
+ )
+ }
+ resting := restingOrder{
+ OrderID: source, Price: adapter.config.formatPrice(native.Price),
+ RemainingQuantity: adapter.config.formatQuantity(native.RemainingQty),
+ Owner: owner, OrderType: "LIMIT",
+ }
+ if native.Side == types.SideBuy {
+ snapshot.Bids = append(snapshot.Bids, resting)
+ } else {
+ snapshot.Asks = append(snapshot.Asks, resting)
+ }
+ }
+ state.Books = append(state.Books, snapshot)
+ }
+ return state, nil
+}
diff --git a/integrations/intrepid_orderbook/adapter_test.go b/integrations/intrepid_orderbook/adapter_test.go
new file mode 100644
index 0000000..7c49251
--- /dev/null
+++ b/integrations/intrepid_orderbook/adapter_test.go
@@ -0,0 +1,532 @@
+package main
+
+import (
+ "bytes"
+ "encoding/json"
+ "math"
+ "reflect"
+ "strings"
+ "testing"
+)
+
+func number(value string) *json.Number {
+ number := json.Number(value)
+ return &number
+}
+
+func textPointer(value string) *string { return &value }
+
+func int64Pointer(value int64) *int64 { return &value }
+
+func testAdapter(t *testing.T, config configWire) *adapter {
+ t.Helper()
+ harness, err := newAdapter(config)
+ if err != nil {
+ t.Fatal(err)
+ }
+ return harness
+}
+
+func applyForTest(t *testing.T, harness *adapter, event marketEvent) observationFrame {
+ t.Helper()
+ observation, err := harness.apply(event, 1)
+ if err != nil {
+ t.Fatal(err)
+ }
+ return observation
+}
+
+func limitEvent(id, side, price, quantity string) marketEvent {
+ return marketEvent{Op: "new", Symbol: "TEST", OrderID: number(id),
+ Side: textPointer(side), OrderType: "LIMIT", Price: number(price), Quantity: number(quantity)}
+}
+
+func TestEmptyStateDigestMatchesProtocolCanonicalJSON(t *testing.T) {
+ digest, err := (bookState{Books: []bookSnapshot{}}).digest()
+ if err != nil {
+ t.Fatal(err)
+ }
+ const expected = "dd8681e973bdf802edf26260297e68e24cdbd75c782c90bbaaaddd401df51090"
+ if digest != expected {
+ t.Fatalf("digest = %q, want %q", digest, expected)
+ }
+}
+
+func TestNativeCrossingPreservesFIFOAfterReduction(t *testing.T) {
+ harness, err := newAdapter(configWire{})
+ if err != nil {
+ t.Fatal(err)
+ }
+ events := []marketEvent{
+ {Op: "new", Symbol: "TEST", OrderID: number("1"), Side: textPointer("BUY"),
+ OrderType: "LIMIT", Price: number("100"), Quantity: number("5"), Owner: int64Pointer(10)},
+ {Op: "new", Symbol: "TEST", OrderID: number("2"), Side: textPointer("BUY"),
+ OrderType: "LIMIT", Price: number("100"), Quantity: number("3"), Owner: int64Pointer(20)},
+ {Op: "reduce", Symbol: "TEST", OrderID: number("1"), Quantity: number("2")},
+ {Op: "new", Symbol: "TEST", OrderID: number("3"), Side: textPointer("SELL"),
+ OrderType: "LIMIT", Price: number("100"), Quantity: number("4"), Owner: int64Pointer(30)},
+ }
+ var observation observationFrame
+ for index, event := range events {
+ observation, err = harness.apply(event, uint64(index+1))
+ if err != nil {
+ t.Fatalf("event %d: %v", index+1, err)
+ }
+ if observation.Outcome.Status != "applied" {
+ t.Fatalf("event %d outcome = %#v", index+1, observation.Outcome)
+ }
+ }
+
+ if len(observation.Trades) != 2 {
+ t.Fatalf("trades = %#v, want two native fills", observation.Trades)
+ }
+ if observation.Trades[0].BuyOrderID != 1 || observation.Trades[0].Quantity != "3" {
+ t.Fatalf("first fill = %#v, want reduced order 1 for quantity 3", observation.Trades[0])
+ }
+ if observation.Trades[1].BuyOrderID != 2 || observation.Trades[1].Quantity != "1" {
+ t.Fatalf("second fill = %#v, want order 2 for quantity 1", observation.Trades[1])
+ }
+ state, err := harness.snapshot()
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(state.Books) != 1 || len(state.Books[0].Bids) != 1 {
+ t.Fatalf("state = %#v, want one residual bid", state)
+ }
+ remaining := state.Books[0].Bids[0]
+ if remaining.OrderID != 2 || remaining.RemainingQuantity != "2" {
+ t.Fatalf("remaining bid = %#v, want order 2 quantity 2", remaining)
+ }
+}
+
+func TestUnfillableFOKPreservesNativeRejection(t *testing.T) {
+ harness, err := newAdapter(configWire{})
+ if err != nil {
+ t.Fatal(err)
+ }
+ observation, err := harness.apply(marketEvent{
+ Op: "new", Symbol: "TEST", OrderID: number("1"), Side: textPointer("BUY"),
+ OrderType: "FOK", Price: number("100"), Quantity: number("1"), Owner: int64Pointer(10),
+ }, 1)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if observation.Outcome.Status != "rejected" || observation.Outcome.Reason == nil ||
+ *observation.Outcome.Reason != "INVALID_ORDER" {
+ t.Fatalf("outcome = %#v, want the upstream FOK rejection", observation.Outcome)
+ }
+ if observation.RestingOrderCount != 0 || len(observation.Trades) != 0 {
+ t.Fatalf("observation = %#v, want no book mutation", observation)
+ }
+}
+
+func TestQuantityBeyondNativeSixPlaceBoundaryIsRejected(t *testing.T) {
+ places := uint32(12)
+ harness, err := newAdapter(configWire{QuantityDecimalPlaces: &places})
+ if err != nil {
+ t.Fatal(err)
+ }
+ observation, err := harness.apply(marketEvent{
+ Op: "new", Symbol: "TEST", OrderID: number("1"), Side: textPointer("BUY"),
+ OrderType: "LIMIT", Price: number("100"), Quantity: number("0.0000001"), Owner: int64Pointer(10),
+ }, 1)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if observation.Outcome.Status != "rejected" || observation.Outcome.Message == nil ||
+ !strings.Contains(*observation.Outcome.Message, "more than 6 native decimal places") {
+ t.Fatalf("outcome = %#v, want an explicit native-precision rejection", observation.Outcome)
+ }
+}
+
+func TestServerHandshakeAndEmptyCompletion(t *testing.T) {
+ input := strings.Join([]string{
+ `{"type":"hello","protocol":"tracebook.conformance","protocol_version":1,"config":{"matching_algorithm":"fifo","tick_size":"0.01","self_trade_policy":"NONE","quantity_decimal_places":12}}`,
+ `{"type":"finish","event_count":0}`,
+ "",
+ }, "\n")
+ var output bytes.Buffer
+ if status := serve(strings.NewReader(input), &output); status != 0 {
+ t.Fatalf("status = %d, output = %s", status, output.String())
+ }
+ decoder := json.NewDecoder(&output)
+ var ready map[string]any
+ var complete map[string]any
+ if err := decoder.Decode(&ready); err != nil {
+ t.Fatal(err)
+ }
+ if err := decoder.Decode(&complete); err != nil {
+ t.Fatal(err)
+ }
+ engine := ready["engine"].(map[string]any)
+ if ready["type"] != "ready" || engine["name"] != engineName || engine["version"] != engineVersion {
+ t.Fatalf("ready = %#v", ready)
+ }
+ if _, ok := engine["revision"]; ok {
+ t.Fatal("default handshake unexpectedly binds a revision")
+ }
+ if _, ok := engine["snapshot_id"]; ok {
+ t.Fatal("default handshake unexpectedly binds a snapshot")
+ }
+ if complete["type"] != "complete" || complete["event_count"] != float64(0) {
+ t.Fatalf("complete = %#v", complete)
+ }
+}
+
+func TestServerRejectsMissingNullAndMistypedRequiredFields(t *testing.T) {
+ for _, frame := range []string{
+ `{"type":"finish"}`, `{"type":"finish","event_count":null}`,
+ `{"type":"finish","event_count":false}`, `{"type":"finish","event_count":"0"}`,
+ `{"type":"finish","event_count":0.0}`, `{"type":"finish","event_count":-1}`,
+ `{"type":"snapshot"}`, `{"type":"snapshot","index":null}`,
+ `{"type":"snapshot","index":true}`, `{"type":"snapshot","index":"0"}`,
+ `{"type":"event","index":1}`, `{"type":"event","index":1,"event":null}`,
+ `{"type":"event","index":1,"event":[]}`, `{"type":"event","index":1,"event":false}`,
+ } {
+ t.Run(frame, func(t *testing.T) {
+ input := `{"type":"hello","protocol":"tracebook.conformance","protocol_version":1}` + "\n" + frame
+ var output bytes.Buffer
+ if status := serve(strings.NewReader(input), &output); status != 2 {
+ t.Fatalf("status = %d, output = %s", status, output.String())
+ }
+ lines := strings.Split(strings.TrimSpace(output.String()), "\n")
+ if len(lines) != 2 || !strings.Contains(lines[1], `"code":"PROTOCOL_ERROR"`) {
+ t.Fatalf("expected ready and one protocol error: %s", output.String())
+ }
+ })
+ }
+}
+
+func TestSideAggregateEnvelopeIncludesIOCAndFOKButNotMarket(t *testing.T) {
+ for _, orderType := range []string{"LIMIT", "IOC", "FOK", "MARKET"} {
+ t.Run(orderType, func(t *testing.T) {
+ harness := testAdapter(t, configWire{})
+ first := applyForTest(t, harness, limitEvent("1", "BUY", "0.01", "4700000000000"))
+ event := limitEvent("2", "BUY", "0.01", "4700000000000")
+ event.OrderType = orderType
+ got := applyForTest(t, harness, event)
+ if got.StateHash != first.StateHash || len(got.Trades) != 0 {
+ t.Fatalf("unsupported aggregate mutated the book: %#v", got)
+ }
+ if orderType == "MARKET" {
+ if got.Outcome.Status != "applied" {
+ t.Fatalf("market order lost its aggregate exemption: %#v", got)
+ }
+ } else if got.Outcome.Status != "rejected" || got.Outcome.Message == nil ||
+ !strings.Contains(*got.Outcome.Message, "side aggregate range") {
+ t.Fatalf("expected pre-mutation aggregate rejection: %#v", got)
+ }
+ })
+ }
+}
+
+func TestServerHandshakeReportsInjectedEvidenceIdentity(t *testing.T) {
+ previousRevision, previousSnapshot := engineRevision, engineSnapshot
+ t.Cleanup(func() {
+ engineRevision, engineSnapshot = previousRevision, previousSnapshot
+ })
+ engineRevision = "reviewed-source-revision"
+ engineSnapshot = "sha256:captured-source-snapshot"
+ input := `{"type":"hello","protocol":"tracebook.conformance","protocol_version":1,"config":{}}
+{"type":"finish","event_count":0}`
+ var output bytes.Buffer
+ if status := serve(strings.NewReader(input), &output); status != 0 {
+ t.Fatalf("status = %d, output = %s", status, output.String())
+ }
+ var ready struct {
+ Engine map[string]string `json:"engine"`
+ }
+ if err := json.NewDecoder(&output).Decode(&ready); err != nil {
+ t.Fatal(err)
+ }
+ if ready.Engine["revision"] != engineRevision || ready.Engine["snapshot_id"] != engineSnapshot ||
+ ready.Engine["name"] != engineName || ready.Engine["version"] != engineVersion {
+ t.Fatalf("bound identity = %#v", ready.Engine)
+ }
+}
+
+func TestAnonymousOwnersRemainExemptFromNativeSTPAfterReplace(t *testing.T) {
+ for _, algorithm := range []string{"fifo", "pro_rata"} {
+ for _, policy := range []string{"CANCEL_RESTING", "CANCEL_INCOMING"} {
+ for _, test := range []struct {
+ name string
+ makerOwner *int64
+ takerOwner *int64
+ selfMatch bool
+ }{
+ {"omitted", nil, nil, false},
+ {"explicit-anonymous", int64Pointer(-1), int64Pointer(-1), false},
+ {"mixed-anonymous", nil, int64Pointer(-1), false},
+ {"anonymous-vs-numeric", int64Pointer(-1), int64Pointer(1), false},
+ {"same-numeric", int64Pointer(1), int64Pointer(1), true},
+ {"same-negative-numeric", int64Pointer(-2), int64Pointer(-2), true},
+ } {
+ t.Run(algorithm+"/"+policy+"/"+test.name, func(t *testing.T) {
+ harness := testAdapter(t, configWire{MatchingAlgorithm: algorithm, SelfTradePolicy: policy})
+ maker := limitEvent("1", "BUY", "100", "2")
+ maker.Owner = test.makerOwner
+ for _, event := range []marketEvent{
+ maker,
+ {Op: "replace", Symbol: "TEST", OrderID: number("1"), Quantity: number("3")},
+ {Op: "reduce", Symbol: "TEST", OrderID: number("1"), Quantity: number("1")},
+ } {
+ if got := applyForTest(t, harness, event); got.Outcome.Status != "applied" {
+ t.Fatalf("%s outcome = %#v", event.Op, got.Outcome)
+ }
+ }
+ state, err := harness.snapshot()
+ if err != nil {
+ t.Fatal(err)
+ }
+ if got := state.Books[0].Bids[0].Owner; got != eventOwner(maker) {
+ t.Fatalf("owner after replace/reduce = %d, want %d", got, eventOwner(maker))
+ }
+ taker := limitEvent("2", "SELL", "100", "1")
+ taker.Owner = test.takerOwner
+ got := applyForTest(t, harness, taker)
+ if got.Outcome.Status != "applied" {
+ t.Fatalf("taker outcome = %#v", got.Outcome)
+ }
+ if !test.selfMatch {
+ if len(got.Trades) != 1 || got.Trades[0].BuyOrderID != 1 || got.Trades[0].Quantity != "1" {
+ t.Fatalf("anonymous crossing = %#v, want a native fill", got)
+ }
+ } else if len(got.Trades) != 0 {
+ t.Fatalf("same-owner order traded under %s: %#v", policy, got.Trades)
+ }
+ activeID := "1"
+ if test.selfMatch && policy == "CANCEL_RESTING" {
+ activeID = "2"
+ }
+ cancel := applyForTest(t, harness, marketEvent{Op: "cancel", Symbol: "TEST", OrderID: number(activeID)})
+ if cancel.Outcome.Status != "applied" || cancel.RestingOrderCount != 0 {
+ t.Fatalf("cancel after STP = %#v", cancel)
+ }
+ book := harness.books["TEST"]
+ if len(book.nativeToOwner) != 0 || len(book.nativeToSource) != 0 || len(book.sourceToNative) != 0 {
+ t.Fatal("inactive source and owner mappings were retained")
+ }
+ })
+ }
+ }
+ }
+}
+
+func TestQuantityPrecisionOnlyRoundsObservations(t *testing.T) {
+ places := uint32(0)
+ harness := testAdapter(t, configWire{QuantityDecimalPlaces: &places})
+ first := applyForTest(t, harness, limitEvent("1", "BUY", "100", "2.4"))
+ if first.Outcome.Status != "applied" {
+ t.Fatalf("first outcome = %#v", first.Outcome)
+ }
+ if got := harness.books["TEST"].engine.RestingOrders()[0].RemainingQty; got != 2_400_000 {
+ t.Fatalf("native quantity = %d, want the unrounded input 2400000", got)
+ }
+ second := applyForTest(t, harness, limitEvent("2", "SELL", "100", "3.5"))
+ if second.Outcome.Status != "applied" || len(second.Trades) != 1 || second.Trades[0].Quantity != "2" {
+ t.Fatalf("crossing = %#v, want the native 2.4 fill reported at zero decimal places", second)
+ }
+ state, err := harness.snapshot()
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(state.Books[0].Asks) != 1 || state.Books[0].Asks[0].RemainingQuantity != "1" {
+ t.Fatalf("state = %#v, want 1.1 native quantity reported as 1", state)
+ }
+ if got := harness.books["TEST"].engine.RestingOrders()[0].RemainingQty; got != 1_100_000 {
+ t.Fatalf("native remainder = %d, want 1100000", got)
+ }
+ if _, err := harness.config.quantityUnits(number("1.0000001")); err == nil {
+ t.Fatal("input beyond six native places must not be rounded into range")
+ }
+}
+
+func TestPriceTicksMatchReferenceBinary64Snapping(t *testing.T) {
+ config := testAdapter(t, configWire{}).config
+ for _, test := range []struct {
+ price string
+ ticks int64
+ }{
+ {"1.015", 101}, // Binary64 division is just below 101.5.
+ {"0.105", 10},
+ {"0.115", 12},
+ {"100.004", 10000},
+ {"100.006", 10001},
+ } {
+ t.Run(test.price, func(t *testing.T) {
+ got, err := config.priceTicks(number(test.price))
+ if err != nil || got != test.ticks {
+ t.Fatalf("ticks = %d, %v; want %d", got, err, test.ticks)
+ }
+ })
+ }
+ for _, price := range []string{"0.005", "0", "-1", "NaN", "Inf", "1e309", "92233720368547758.08"} {
+ if _, err := config.priceTicks(number(price)); err == nil {
+ t.Errorf("unsupported price %s was accepted", price)
+ }
+ }
+ for _, tick := range []string{"NaN", "Inf", "1e309", "1e-330"} {
+ if _, err := parseConfig(configWire{TickSize: tick}); err == nil {
+ t.Errorf("unsupported tick %s was accepted", tick)
+ }
+ }
+}
+
+func TestNativeQuantityIntegerBoundary(t *testing.T) {
+ config := testAdapter(t, configWire{}).config
+ got, err := config.quantityUnits(number("9223372036854.775807"))
+ if err != nil || got != math.MaxInt64 {
+ t.Fatalf("maximum native quantity = %d, %v", got, err)
+ }
+ for _, quantity := range []string{"9223372036854.775808", "0", "-1", "NaN"} {
+ if _, err := config.quantityUnits(number(quantity)); err == nil {
+ t.Errorf("unsupported quantity %s was accepted", quantity)
+ }
+ }
+}
+
+func TestUnsafeProRataProductIsRejectedBeforeMutation(t *testing.T) {
+ for _, orderType := range []string{"LIMIT", "MARKET", "IOC", "FOK"} {
+ t.Run(orderType, func(t *testing.T) {
+ harness := testAdapter(t, configWire{MatchingAlgorithm: "pro_rata"})
+ maker := applyForTest(t, harness, limitEvent("1", "SELL", "100", "4000"))
+ taker := limitEvent("2", "BUY", "100", "4000")
+ taker.OrderType = orderType
+ got := applyForTest(t, harness, taker)
+ if got.Outcome.Status != "rejected" || got.Outcome.Message == nil ||
+ !strings.Contains(*got.Outcome.Message, "pro-rata product range") {
+ t.Fatalf("unsafe product outcome = %#v", got.Outcome)
+ }
+ if got.StateHash != maker.StateHash || len(got.Trades) != 0 {
+ t.Fatalf("unsafe product mutated state: %#v", got)
+ }
+ })
+ }
+ for _, test := range []struct{ algorithm, quantity string }{
+ {"fifo", "4000"}, {"pro_rata", "3000"},
+ } {
+ t.Run(test.algorithm+"/safe-product", func(t *testing.T) {
+ harness := testAdapter(t, configWire{MatchingAlgorithm: test.algorithm})
+ applyForTest(t, harness, limitEvent("1", "SELL", "100", test.quantity))
+ got := applyForTest(t, harness, limitEvent("2", "BUY", "100", test.quantity))
+ if got.Outcome.Status != "applied" || len(got.Trades) != 1 || got.Trades[0].Quantity != test.quantity {
+ t.Fatalf("supported large crossing = %#v", got)
+ }
+ })
+ }
+}
+
+func TestSideAggregateAndReplacementCapacityPreserveExistingOrders(t *testing.T) {
+ harness := testAdapter(t, configWire{})
+ first := applyForTest(t, harness, limitEvent("1", "BUY", "0.01", "4700000000000"))
+ if first.Outcome.Status != "applied" {
+ t.Fatalf("large first order = %#v", first)
+ }
+ tooLarge := applyForTest(t, harness, limitEvent("2", "BUY", "0.01", "4700000000000"))
+ if tooLarge.Outcome.Status != "rejected" || tooLarge.StateHash != first.StateHash ||
+ tooLarge.Outcome.Message == nil || !strings.Contains(*tooLarge.Outcome.Message, "side aggregate range") {
+ t.Fatalf("aggregate overflow = %#v", tooLarge)
+ }
+ second := applyForTest(t, harness, limitEvent("2", "BUY", "0.01", "4000000000000"))
+ if second.Outcome.Status != "applied" {
+ t.Fatalf("supported aggregate = %#v", second)
+ }
+ replace := applyForTest(t, harness, marketEvent{Op: "replace", Symbol: "TEST",
+ OrderID: number("2"), Quantity: number("4700000000000")})
+ if replace.Outcome.Reason == nil || *replace.Outcome.Reason != "INVALID_REPLACEMENT" ||
+ replace.StateHash != second.StateHash {
+ t.Fatalf("unsafe replacement = %#v", replace)
+ }
+ noChange := applyForTest(t, harness, marketEvent{Op: "replace", Symbol: "TEST",
+ OrderID: number("2"), Quantity: number("4000000000000")})
+ if noChange.Outcome.Status != "applied" {
+ t.Fatalf("replacement must exclude the old quantity from its capacity check: %#v", noChange)
+ }
+}
+
+func TestNativeNotionalLimitAppliesOnlyToPriceBearingInstructions(t *testing.T) {
+ for _, orderType := range []string{"LIMIT", "IOC", "FOK", "MARKET"} {
+ t.Run(orderType, func(t *testing.T) {
+ harness := testAdapter(t, configWire{})
+ event := limitEvent("1", "BUY", "100000", "1000000")
+ event.OrderType = orderType
+ got := applyForTest(t, harness, event)
+ if orderType == "MARKET" {
+ if got.Outcome.Status != "applied" || got.RestingOrderCount != 0 {
+ t.Fatalf("market order should retain the native notional exemption: %#v", got)
+ }
+ } else if got.Outcome.Status != "rejected" || got.Outcome.Message == nil ||
+ !strings.Contains(*got.Outcome.Message, "order notional") || got.RestingOrderCount != 0 {
+ t.Fatalf("native notional overflow = %#v", got)
+ }
+ })
+ }
+}
+
+func TestNotionalOverflowReplacementPreservesNativeOrderAndSourceMetadata(t *testing.T) {
+ for _, test := range []struct {
+ name string
+ owner *int64
+ }{{"anonymous", nil}, {"identified", int64Pointer(8)}} {
+ t.Run(test.name, func(t *testing.T) {
+ harness := testAdapter(t, configWire{})
+ event := limitEvent("1", "BUY", "100", "1")
+ event.Owner = test.owner
+ first := applyForTest(t, harness, event)
+ book := harness.books["TEST"]
+ original := *book.engine.RestingOrders()[0]
+ got := applyForTest(t, harness, marketEvent{
+ Op: "replace", Symbol: "TEST", OrderID: number("1"),
+ Price: number("100000"), Quantity: number("1000000"),
+ })
+ if got.Outcome.Reason == nil || *got.Outcome.Reason != "INVALID_REPLACEMENT" ||
+ got.Outcome.Message == nil || !strings.Contains(*got.Outcome.Message, "order notional") {
+ t.Fatalf("overflow replacement outcome = %#v", got.Outcome)
+ }
+ if got.StateHash != first.StateHash || got.RestingOrderCount != 1 || len(got.Trades) != 0 {
+ t.Fatalf("overflow replacement changed the source state: %#v", got)
+ }
+ orders := book.engine.RestingOrders()
+ if len(orders) != 1 || !reflect.DeepEqual(original, *orders[0]) {
+ t.Fatalf("overflow replacement changed the native order: %#v", orders)
+ }
+ if len(book.sourceToNative) != 1 || book.sourceToNative[1] != original.ID ||
+ len(book.nativeToSource) != 1 || book.nativeToSource[original.ID] != 1 ||
+ len(book.nativeToOwner) != 1 || book.nativeToOwner[original.ID] != eventOwner(event) {
+ t.Fatal("overflow replacement changed source identity metadata")
+ }
+ })
+ }
+}
+
+func TestHighOutputPrecisionPreservesNativeDecimalRemainder(t *testing.T) {
+ places := uint32(18)
+ harness := testAdapter(t, configWire{QuantityDecimalPlaces: &places})
+ applyForTest(t, harness, limitEvent("1", "BUY", "100", "0.3"))
+ got := applyForTest(t, harness, limitEvent("2", "SELL", "100", "0.1"))
+ state, err := harness.snapshot()
+ if err != nil {
+ t.Fatal(err)
+ }
+ // The binary64 reference retains 0.19999999999999998 at 18 places.
+ // Preserve the native arithmetic here so this known difference is visible.
+ if got.Outcome.Status != "applied" || len(state.Books[0].Bids) != 1 ||
+ state.Books[0].Bids[0].RemainingQuantity != "0.2" {
+ t.Fatalf("native decimal remainder was not preserved: %#v", state)
+ }
+}
+
+func TestFineTickOutputPreservesNativeIntegerPrice(t *testing.T) {
+ harness := testAdapter(t, configWire{TickSize: "0.000000000000000001"})
+ got := applyForTest(t, harness, limitEvent("1", "BUY", "1", "0.000001"))
+ state, err := harness.snapshot()
+ if err != nil {
+ t.Fatal(err)
+ }
+ // The reference multiplies and rounds in binary64, yielding
+ // 0.9999999999999999. Native tick output must not imitate that arithmetic.
+ if got.Outcome.Status != "applied" || len(state.Books[0].Bids) != 1 ||
+ state.Books[0].Bids[0].Price != "0.999999999999999872" {
+ t.Fatalf("native fine-grid price was not preserved: %#v", state)
+ }
+}
diff --git a/integrations/intrepid_orderbook/go.mod b/integrations/intrepid_orderbook/go.mod
new file mode 100644
index 0000000..69ceb33
--- /dev/null
+++ b/integrations/intrepid_orderbook/go.mod
@@ -0,0 +1,8 @@
+module github.com/Taz33m/tracebook/integrations/intrepid_orderbook
+
+go 1.23.5
+
+require (
+ github.com/intrepidkarthi/orderbook v0.26.0
+ github.com/shopspring/decimal v1.4.0
+)
diff --git a/integrations/intrepid_orderbook/go.sum b/integrations/intrepid_orderbook/go.sum
new file mode 100644
index 0000000..13bb2d8
--- /dev/null
+++ b/integrations/intrepid_orderbook/go.sum
@@ -0,0 +1,4 @@
+github.com/intrepidkarthi/orderbook v0.26.0 h1:ePTONeCYlz4WQ/Qz/8YhvQ2OKsBTI/VHvclhgG3eLFI=
+github.com/intrepidkarthi/orderbook v0.26.0/go.mod h1:5HkWKqoS3UmnPaL5y3WIL8X/CynK3fIcY/T/ZdVq35s=
+github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k=
+github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME=
diff --git a/integrations/intrepid_orderbook/main.go b/integrations/intrepid_orderbook/main.go
new file mode 100644
index 0000000..3d27ba5
--- /dev/null
+++ b/integrations/intrepid_orderbook/main.go
@@ -0,0 +1,7 @@
+package main
+
+import "os"
+
+func main() {
+ os.Exit(serve(os.Stdin, os.Stdout))
+}
diff --git a/integrations/intrepid_orderbook/protocol.go b/integrations/intrepid_orderbook/protocol.go
new file mode 100644
index 0000000..b8f66a6
--- /dev/null
+++ b/integrations/intrepid_orderbook/protocol.go
@@ -0,0 +1,361 @@
+package main
+
+import (
+ "bufio"
+ "bytes"
+ "crypto/sha256"
+ "encoding/hex"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "sort"
+)
+
+const (
+ protocolName = "tracebook.conformance"
+ protocolVersion = 1
+ engineName = "intrepidkarthi/orderbook CLOB"
+ engineVersion = "v0.26.0@51d480cdb68b"
+)
+
+// Optional build-time identity binds an evidence run to its captured source.
+// Set with -ldflags '-X main.engineRevision=... -X main.engineSnapshot=...'.
+// Empty defaults preserve the unbound qualification handshake.
+var engineRevision string
+var engineSnapshot string
+
+func engineMetadata() map[string]string {
+ metadata := map[string]string{"name": engineName, "version": engineVersion, "language": "Go"}
+ if engineRevision != "" {
+ metadata["revision"] = engineRevision
+ }
+ if engineSnapshot != "" {
+ metadata["snapshot_id"] = engineSnapshot
+ }
+ return metadata
+}
+
+type configWire struct {
+ MatchingAlgorithm string `json:"matching_algorithm"`
+ TickSize string `json:"tick_size"`
+ SelfTradePolicy string `json:"self_trade_policy"`
+ QuantityDecimalPlaces *uint32 `json:"quantity_decimal_places"`
+}
+
+type helloFrame struct {
+ Type string `json:"type"`
+ Protocol string `json:"protocol"`
+ ProtocolVersion int `json:"protocol_version"`
+ Config configWire `json:"config"`
+}
+
+type marketEvent struct {
+ Op string `json:"op"`
+ Symbol string `json:"symbol"`
+ OrderID *json.Number `json:"order_id"`
+ Side *string `json:"side"`
+ OrderType string `json:"order_type"`
+ Price *json.Number `json:"price"`
+ Quantity *json.Number `json:"quantity"`
+ Owner *int64 `json:"owner"`
+ TimestampNS *uint64 `json:"timestamp_ns"`
+}
+
+type eventFrame struct {
+ Type string `json:"type"`
+ Index uint64 `json:"index"`
+ Event marketEvent `json:"event"`
+}
+
+type outcome struct {
+ Status string `json:"status"`
+ Reason *string `json:"reason"`
+ Message *string `json:"message"`
+}
+
+func applied() outcome { return outcome{Status: "applied"} }
+
+func rejected(reason, message string) outcome {
+ return outcome{Status: "rejected", Reason: &reason, Message: &message}
+}
+
+type tradeFill struct {
+ Symbol string `json:"symbol"`
+ BuyOrderID int64 `json:"buy_order_id"`
+ SellOrderID int64 `json:"sell_order_id"`
+ Price string `json:"price"`
+ Quantity string `json:"quantity"`
+}
+
+type restingOrder struct {
+ OrderID int64 `json:"order_id"`
+ Price string `json:"price"`
+ RemainingQuantity string `json:"remaining_quantity"`
+ Owner int64 `json:"owner"`
+ OrderType string `json:"order_type"`
+}
+
+type bookSnapshot struct {
+ Symbol string `json:"symbol"`
+ Bids []restingOrder `json:"bids"`
+ Asks []restingOrder `json:"asks"`
+}
+
+type bookState struct {
+ Books []bookSnapshot `json:"books"`
+}
+
+func (state bookState) orderCount() int {
+ count := 0
+ for _, book := range state.Books {
+ count += len(book.Bids) + len(book.Asks)
+ }
+ return count
+}
+
+func (state bookState) digest() (string, error) {
+ raw, err := json.Marshal(state)
+ if err != nil {
+ return "", err
+ }
+ decoder := json.NewDecoder(bytes.NewReader(raw))
+ decoder.UseNumber()
+ var value any
+ if err := decoder.Decode(&value); err != nil {
+ return "", err
+ }
+ canonical, err := canonicalJSON(value)
+ if err != nil {
+ return "", err
+ }
+ hash := sha256.Sum256(canonical)
+ return hex.EncodeToString(hash[:]), nil
+}
+
+type observationFrame struct {
+ Type string `json:"type"`
+ Index uint64 `json:"index"`
+ Outcome outcome `json:"outcome"`
+ Trades []tradeFill `json:"trades"`
+ StateHash string `json:"state_hash"`
+ RestingOrderCount int `json:"resting_order_count"`
+}
+
+func canonicalJSON(value any) ([]byte, error) {
+ switch typed := value.(type) {
+ case nil, bool, string, json.Number:
+ return json.Marshal(typed)
+ case []any:
+ var buffer bytes.Buffer
+ buffer.WriteByte('[')
+ for index, item := range typed {
+ if index > 0 {
+ buffer.WriteByte(',')
+ }
+ encoded, err := canonicalJSON(item)
+ if err != nil {
+ return nil, err
+ }
+ buffer.Write(encoded)
+ }
+ buffer.WriteByte(']')
+ return buffer.Bytes(), nil
+ case map[string]any:
+ keys := make([]string, 0, len(typed))
+ for key := range typed {
+ keys = append(keys, key)
+ }
+ sort.Strings(keys)
+ var buffer bytes.Buffer
+ buffer.WriteByte('{')
+ for index, key := range keys {
+ if index > 0 {
+ buffer.WriteByte(',')
+ }
+ encodedKey, _ := json.Marshal(key)
+ buffer.Write(encodedKey)
+ buffer.WriteByte(':')
+ encodedValue, err := canonicalJSON(typed[key])
+ if err != nil {
+ return nil, err
+ }
+ buffer.Write(encodedValue)
+ }
+ buffer.WriteByte('}')
+ return buffer.Bytes(), nil
+ default:
+ return nil, fmt.Errorf("unsupported canonical JSON value %T", value)
+ }
+}
+
+type serverError struct {
+ code string
+ message string
+}
+
+func (err *serverError) Error() string { return err.message }
+
+func protocolError(message string) error {
+ return &serverError{code: "PROTOCOL_ERROR", message: message}
+}
+
+func adapterError(err error) error {
+ return &serverError{code: "ADAPTER_ERROR", message: err.Error()}
+}
+
+func readObject(decoder *json.Decoder) (map[string]json.RawMessage, error) {
+ var object map[string]json.RawMessage
+ if err := decoder.Decode(&object); err != nil {
+ return nil, err
+ }
+ if object == nil {
+ return nil, errors.New("protocol message must be an object")
+ }
+ return object, nil
+}
+
+func frameType(object map[string]json.RawMessage) (string, error) {
+ var value string
+ if err := json.Unmarshal(object["type"], &value); err != nil || value == "" {
+ return "", errors.New("protocol message requires a string type")
+ }
+ return value, nil
+}
+
+func decodeObject(object map[string]json.RawMessage, target any) error {
+ raw, err := json.Marshal(object)
+ if err != nil {
+ return err
+ }
+ decoder := json.NewDecoder(bytes.NewReader(raw))
+ decoder.UseNumber()
+ return decoder.Decode(target)
+}
+
+func writeFrame(writer *bufio.Writer, value any) error {
+ if err := json.NewEncoder(writer).Encode(value); err != nil {
+ return err
+ }
+ return writer.Flush()
+}
+
+func serve(input io.Reader, output io.Writer) int {
+ decoder := json.NewDecoder(input)
+ decoder.UseNumber()
+ writer := bufio.NewWriter(output)
+ err := runServer(decoder, writer)
+ if err == nil {
+ return 0
+ }
+ var typed *serverError
+ if !errors.As(err, &typed) {
+ typed = &serverError{code: "PROTOCOL_ERROR", message: err.Error()}
+ }
+ _ = writeFrame(writer, map[string]any{
+ "type": "error", "code": typed.code, "message": typed.message,
+ })
+ return 2
+}
+
+func runServer(decoder *json.Decoder, writer *bufio.Writer) error {
+ first, err := readObject(decoder)
+ if err != nil {
+ return protocolError("expected hello message: " + err.Error())
+ }
+ var hello helloFrame
+ if err := decodeObject(first, &hello); err != nil {
+ return protocolError(err.Error())
+ }
+ if hello.Type != "hello" {
+ return protocolError("first message must be hello")
+ }
+ if hello.Protocol != protocolName {
+ return protocolError(fmt.Sprintf("protocol must be %q", protocolName))
+ }
+ if hello.ProtocolVersion != protocolVersion {
+ return protocolError(fmt.Sprintf("protocol_version must be %d", protocolVersion))
+ }
+ adapter, err := newAdapter(hello.Config)
+ if err != nil {
+ return protocolError(err.Error())
+ }
+ if err := writeFrame(writer, map[string]any{
+ "type": "ready",
+ "protocol": protocolName,
+ "protocol_version": protocolVersion,
+ "engine": engineMetadata(),
+ }); err != nil {
+ return adapterError(err)
+ }
+
+ var lastIndex uint64
+ for {
+ object, err := readObject(decoder)
+ if err != nil {
+ if errors.Is(err, io.EOF) {
+ return protocolError("protocol ended before finish")
+ }
+ return protocolError(err.Error())
+ }
+ kind, err := frameType(object)
+ if err != nil {
+ return protocolError(err.Error())
+ }
+ switch kind {
+ case "event":
+ if raw := object["event"]; len(raw) == 0 || bytes.Equal(bytes.TrimSpace(raw), []byte("null")) {
+ return protocolError("event payload must be an object")
+ }
+ var frame eventFrame
+ if err := decodeObject(object, &frame); err != nil {
+ return protocolError(err.Error())
+ }
+ if frame.Index == 0 || frame.Index != lastIndex+1 {
+ return protocolError("event indexes must be contiguous and start at 1")
+ }
+ observation, err := adapter.apply(frame.Event, frame.Index)
+ if err != nil {
+ return adapterError(err)
+ }
+ lastIndex = frame.Index
+ if err := writeFrame(writer, observation); err != nil {
+ return adapterError(err)
+ }
+ case "snapshot":
+ var frame struct {
+ Index *uint64 `json:"index"`
+ }
+ if err := decodeObject(object, &frame); err != nil {
+ return protocolError(err.Error())
+ }
+ if frame.Index == nil || *frame.Index != lastIndex {
+ return protocolError("snapshot index does not match the last event")
+ }
+ state, err := adapter.snapshot()
+ if err != nil {
+ return adapterError(err)
+ }
+ if err := writeFrame(writer, map[string]any{
+ "type": "snapshot", "index": lastIndex, "state": state,
+ }); err != nil {
+ return adapterError(err)
+ }
+ case "finish":
+ var frame struct {
+ EventCount *uint64 `json:"event_count"`
+ }
+ if err := decodeObject(object, &frame); err != nil {
+ return protocolError(err.Error())
+ }
+ if frame.EventCount == nil || *frame.EventCount != lastIndex {
+ return protocolError("finish event_count does not match the last event")
+ }
+ return writeFrame(writer, map[string]any{
+ "type": "complete", "event_count": lastIndex,
+ })
+ default:
+ return protocolError(fmt.Sprintf("unsupported protocol message type: %q", kind))
+ }
+ }
+}
diff --git a/integrations/intrepid_orderbook/regressions/fok-rejection-failure.json b/integrations/intrepid_orderbook/regressions/fok-rejection-failure.json
new file mode 100644
index 0000000..cdc1e1d
--- /dev/null
+++ b/integrations/intrepid_orderbook/regressions/fok-rejection-failure.json
@@ -0,0 +1,58 @@
+{
+ "artifact_type": "tracebook.conformance.failure",
+ "budget_exhausted": false,
+ "campaign_id": "sha256:61464afc70170352bc6eecb6ccb22f120d05166d6f5cb7926ce5d43adf8a6b41",
+ "campaign_seed": 42,
+ "candidate_engine": {
+ "language": "Go",
+ "name": "intrepidkarthi/orderbook CLOB",
+ "version": "v0.26.0@51d480cdb68b"
+ },
+ "config": {
+ "matching_algorithm": "fifo",
+ "quantity_decimal_places": 12,
+ "self_trade_policy": "NONE",
+ "tick_size": "0.01"
+ },
+ "expected_reduced_divergence": {
+ "candidate": "INVALID_ORDER",
+ "category": "outcome",
+ "event": {
+ "op": "new",
+ "order_id": 2,
+ "order_type": "FOK",
+ "owner": 4,
+ "price": 99.99,
+ "quantity": 1.0,
+ "side": "SELL",
+ "symbol": "BETA",
+ "timestamp_ns": 7
+ },
+ "event_index": 1,
+ "kind": "type_mismatch",
+ "message": "candidate applied/rejected semantics differ from the reference",
+ "path": "$.observation.outcome.reason",
+ "reference": null,
+ "snapshot_error": null
+ },
+ "failure_class": "order-outcome drift",
+ "failure_id": "failure-1ce2954857800b3a068d",
+ "generator_version": 2,
+ "one_minimal": true,
+ "original_divergence_event": 7,
+ "original_event_count": 7,
+ "original_trace_sha256": "sha256:be56a106f547a3ed17988bd8b6e448b91ed18e111678c9ac1dc9bd384fce7f8f",
+ "paths": {
+ "campaign": "campaign.json",
+ "minimization": "failure/minimization.json",
+ "original": "original.jsonl",
+ "reduced": "reduced.jsonl"
+ },
+ "profile": "fifo-full-v1",
+ "reduced_event_count": 1,
+ "reduced_trace_sha256": "sha256:395c88526cdd81445205298f8234d8b57afcef71534f8fd7312608fc43862706",
+ "schema_version": 1,
+ "target_category": "outcome",
+ "trace_index": 1,
+ "trace_seed": 2949826092126892291
+}
diff --git a/integrations/intrepid_orderbook/regressions/fok-rejection-reduced.jsonl b/integrations/intrepid_orderbook/regressions/fok-rejection-reduced.jsonl
new file mode 100644
index 0000000..fcee8c4
--- /dev/null
+++ b/integrations/intrepid_orderbook/regressions/fok-rejection-reduced.jsonl
@@ -0,0 +1 @@
+{"op":"new","order_id":2,"order_type":"FOK","owner":4,"price":99.99,"quantity":1.0,"side":"SELL","symbol":"BETA","timestamp_ns":7}
diff --git a/integrations/nautilus_trader/README.md b/integrations/nautilus_trader/README.md
new file mode 100644
index 0000000..6db3b56
--- /dev/null
+++ b/integrations/nautilus_trader/README.md
@@ -0,0 +1,116 @@
+# NautilusTrader L3 Book-Replay Integration
+
+This optional adapter checks NautilusTrader's native Rust `OrderBook` through
+Tracebook's separate `l3-book-replay-v1` surface. It does **not** run the
+matching-engine protocol and does not claim that NautilusTrader is a submitted
+order versus submitted order CLOB.
+
+The integration pins the pre-release Python binding whose public API exposes
+`OrderBook.simulate_fills(BookOrder)` directly:
+
+- package: `nautilus-trader==2.0.0rc3`;
+- tag: `v2.0.0rc3`;
+- source commit: `648970ce64a304d93da0a29320cb6e19b905fa39`;
+- repository: .
+
+The stable `1.231.0` release and the v2 release candidate are different Python
+binding generations. This adapter intentionally fails closed on any installed
+version other than the pin above.
+
+## What The Proof Exercises
+
+The adapter creates one native `L3_MBO` `OrderBook` per trace symbol. `add`,
+`update`, and `delete` events call the corresponding native book methods. A
+`probe` constructs a native `BookOrder` and calls the native
+`OrderBook.simulate_fills` method. Snapshots traverse `book.bids()`,
+`book.asks()`, and each level's FIFO `get_orders()` result.
+
+The 17-event bundled trace covers:
+
+- same-price FIFO insertion;
+- a size increase that retains its current queue slot;
+- ordered partial simulation across orders and price levels;
+- a price-changing update that requeues at its destination level;
+- deletion by source order ID;
+- an inactive-delete rejection at the normalized adapter boundary;
+- independent symbols; and
+- probes that leave the native book unchanged.
+
+No trace order is submitted to NautilusTrader's execution engine. The only
+liquidity is the exogenous L3 state built from deltas.
+
+The scheduled proof also runs generator version 1 with seed `20260831`: 25
+independent traces of 100 events each. The stable campaign ID is
+`sha256:09b1599eaeb474d98617acc7869ace26759ed5ab8350803f06197ef572864bab`.
+It reaches all 11 declared semantic capabilities and passes all 2,500 events.
+
+## Run The Pinned Proof
+
+Use a separate Python 3.12-3.14 environment because the v2 candidate does not
+support Tracebook's Python 3.10 floor:
+
+```bash
+python3.14 -m venv /tmp/tracebook-nautilus
+/tmp/tracebook-nautilus/bin/python -m pip install -e . \
+ "nautilus-trader==2.0.0rc3"
+
+/tmp/tracebook-nautilus/bin/python -m tracebook.book_replay run \
+ src/tracebook/book_replay/fixtures/l3-book-replay-v1.jsonl \
+ --output /tmp/nautilus-book-replay.json \
+ --timeout 10 \
+ --candidate /tmp/tracebook-nautilus/bin/python \
+ integrations/nautilus_trader/adapter.py
+```
+
+The command exits `0` and the pinned candidate produces:
+
+```json
+{
+ "candidate_engine": {
+ "language": "Rust/Python",
+ "name": "NautilusTrader L3 OrderBook",
+ "version": "2.0.0rc3"
+ },
+ "compared_events": 17,
+ "conformant": true,
+ "final_state_hash": "9e0af6be935dce940a87497788c7a9a799c71f05e34a0204c9d294fce611b002"
+}
+```
+
+The timeout also bounds the candidate's natural exit after its `complete`
+frame. A nonzero exit or a forced shutdown is a protocol failure, even if all
+event observations matched. The integration CI installs `pytest` explicitly
+before running the book-replay and integration contract tests; the base package
+does not include test dependencies.
+
+## Negative Control And Retained Evidence
+
+[`faulty_adapter.py`](faulty_adapter.py) subclasses the real pinned native
+adapter and injects one deliberate fault: a same-price size increase is
+implemented as delete-plus-add, moving the order to the back of its level.
+This file is a harness negative control, not a candidate integration.
+
+The generated campaign must exit `1`, localize the first mismatch to
+`$.state.books[0].bids[0].order_id`, and reduce it to the three events in
+[`regressions/upsize-requeue-reduced.jsonl`](regressions/upsize-requeue-reduced.jsonl).
+The retained metadata in
+[`regressions/upsize-requeue-failure.json`](regressions/upsize-requeue-failure.json)
+pins failure ID `failure-dfe5c23848b63211b655`, the reduced trace hash, and
+the expected localized divergence. CI regenerates both forms and compares them
+to the retained evidence. This proves the test surface can reject a known
+native queue-priority defect; it does not imply the unmodified upstream has
+that defect.
+
+## License Boundary
+
+NautilusTrader declares `LGPL-3.0-only` in its Cargo workspace. It is not a
+Tracebook dependency, no Nautilus source or binary is vendored here, and the
+normal Tracebook wheel does not import it. The optional adapter loads a
+user-installed Nautilus wheel in the candidate process and communicates with
+the Tracebook runner over NDJSON.
+
+That process boundary avoids adding an LGPL Rust crate to Tracebook's build,
+but it does not waive obligations for someone who redistributes a combined
+bundle. In particular, a distributed binary statically linking the Nautilus
+Rust crates needs an explicit LGPL section 4 compliance design. This note is a
+distribution warning, not legal advice.
diff --git a/integrations/nautilus_trader/__init__.py b/integrations/nautilus_trader/__init__.py
new file mode 100644
index 0000000..49966f3
--- /dev/null
+++ b/integrations/nautilus_trader/__init__.py
@@ -0,0 +1 @@
+"""Optional NautilusTrader L3 book-replay integration."""
diff --git a/integrations/nautilus_trader/adapter.py b/integrations/nautilus_trader/adapter.py
new file mode 100644
index 0000000..38bef21
--- /dev/null
+++ b/integrations/nautilus_trader/adapter.py
@@ -0,0 +1,179 @@
+#!/usr/bin/env python3
+"""NautilusTrader v2 L3 book adapter for Tracebook's book-replay protocol."""
+
+from __future__ import annotations
+
+from importlib import metadata as importlib_metadata
+
+from tracebook.book_replay import (
+ BookReplayObservation,
+ BookReplaySnapshot,
+ BookReplayState,
+ EngineMetadata,
+ Outcome,
+ RestingBookOrder,
+ SimulatedFill,
+ serve_book_replay_stdio,
+)
+from tracebook.conformance import canonical_decimal
+
+UPSTREAM_REPOSITORY = "https://github.com/nautechsystems/nautilus_trader.git"
+UPSTREAM_TAG = "v2.0.0rc3"
+UPSTREAM_COMMIT = "648970ce64a304d93da0a29320cb6e19b905fa39"
+UPSTREAM_VERSION = "2.0.0rc3"
+
+
+class NautilusTraderBookReplayAdapter:
+ """Translate the normalized profile onto NautilusTrader's native L3 book."""
+
+ def __init__(self, config):
+ from nautilus_trader.model import ( # pylint: disable=import-outside-toplevel
+ BookOrder,
+ BookType,
+ InstrumentId,
+ OrderBook,
+ OrderSide,
+ Price,
+ Quantity,
+ )
+
+ installed_version = importlib_metadata.version("nautilus-trader")
+ if installed_version != UPSTREAM_VERSION:
+ raise RuntimeError(
+ f"NautilusTrader adapter requires {UPSTREAM_VERSION}, found {installed_version}"
+ )
+ self.config = config
+ self.metadata = EngineMetadata(
+ "NautilusTrader L3 OrderBook",
+ installed_version,
+ "Rust/Python",
+ )
+ self._BookOrder = BookOrder
+ self._BookType = BookType
+ self._InstrumentId = InstrumentId
+ self._OrderBook = OrderBook
+ self._OrderSide = OrderSide
+ self._Price = Price
+ self._Quantity = Quantity
+ self._books = {}
+ self._orders = {}
+
+ def _book(self, symbol):
+ book = self._books.get(symbol)
+ if book is None:
+ instrument_id = self._InstrumentId.from_str(f"{symbol}.TRACEBOOK")
+ book = self._OrderBook(instrument_id, self._BookType.L3_MBO)
+ self._books[symbol] = book
+ self._orders[symbol] = {}
+ return book
+
+ def _native_order(self, event):
+ side = self._OrderSide.BUY if event.side == "BUY" else self._OrderSide.SELL
+ return self._BookOrder(
+ side,
+ self._Price.from_str(event.price),
+ self._Quantity.from_str(event.quantity),
+ event.order_id or 0,
+ )
+
+ def apply(self, event, index):
+ fills: tuple[SimulatedFill, ...] = ()
+ outcome = Outcome("applied")
+ known_orders = self._orders.get(event.symbol, {})
+ if event.op == "add":
+ if event.order_id in known_orders:
+ outcome = Outcome(
+ "rejected",
+ "DUPLICATE_ORDER_ID",
+ f"Order {event.order_id} is already present in {event.symbol}",
+ )
+ else:
+ book = self._book(event.symbol)
+ native_order = self._native_order(event)
+ book.add(native_order, 0, index, index)
+ self._orders[event.symbol][event.order_id] = native_order
+ elif event.op == "update":
+ if event.order_id not in known_orders:
+ outcome = Outcome(
+ "rejected",
+ "ORDER_NOT_ACTIVE",
+ f"Order {event.order_id} is not present in {event.symbol}",
+ )
+ else:
+ book = self._books[event.symbol]
+ old_order = known_orders[event.order_id]
+ native_order = self._native_order(event)
+ self._apply_native_update(book, old_order, native_order, index)
+ known_orders[event.order_id] = native_order
+ elif event.op == "delete":
+ if event.order_id not in known_orders:
+ outcome = Outcome(
+ "rejected",
+ "ORDER_NOT_ACTIVE",
+ f"Order {event.order_id} is not present in {event.symbol}",
+ )
+ else:
+ native_order = known_orders.pop(event.order_id)
+ self._books[event.symbol].delete(native_order, 0, index, index)
+ else:
+ book = self._books.get(event.symbol)
+ if book is not None:
+ probe = self._native_order(event)
+ fills = tuple(
+ SimulatedFill(
+ price=canonical_decimal(str(price)),
+ quantity=canonical_decimal(str(quantity)),
+ )
+ for price, quantity in book.simulate_fills(probe)
+ )
+
+ state = self.snapshot()
+ return BookReplayObservation(
+ index=index,
+ outcome=outcome,
+ fills=fills,
+ state_hash=state.digest(),
+ order_count=state.order_count,
+ )
+
+ @staticmethod
+ def _apply_native_update(book, old_order, native_order, index):
+ if old_order.side != native_order.side:
+ # Native OrderBook.update selects one ladder from the new side. A
+ # normalized side move is therefore delete+add at this boundary.
+ book.delete(old_order, 0, index, index)
+ book.add(native_order, 0, index, index)
+ else:
+ book.update(native_order, 0, index, index)
+
+ def snapshot(self):
+ snapshots = []
+ for symbol in sorted(self._books):
+ book = self._books[symbol]
+ snapshots.append(
+ BookReplaySnapshot(
+ symbol=symbol,
+ bids=self._snapshot_levels(book.bids()),
+ asks=self._snapshot_levels(book.asks()),
+ )
+ )
+ return BookReplayState(tuple(snapshots))
+
+ @staticmethod
+ def _snapshot_levels(levels):
+ return tuple(
+ RestingBookOrder(
+ order_id=order.order_id,
+ price=canonical_decimal(str(order.price)),
+ quantity=canonical_decimal(str(order.size)),
+ )
+ for level in levels
+ for order in level.get_orders()
+ )
+
+ def close(self):
+ """The native book objects own no external resources."""
+
+
+if __name__ == "__main__":
+ raise SystemExit(serve_book_replay_stdio(NautilusTraderBookReplayAdapter))
diff --git a/integrations/nautilus_trader/faulty_adapter.py b/integrations/nautilus_trader/faulty_adapter.py
new file mode 100644
index 0000000..9d0de16
--- /dev/null
+++ b/integrations/nautilus_trader/faulty_adapter.py
@@ -0,0 +1,41 @@
+#!/usr/bin/env python3
+"""Negative control that deliberately requeues same-level Nautilus upsizes."""
+
+from __future__ import annotations
+
+from decimal import Decimal
+
+from tracebook.book_replay import EngineMetadata, serve_book_replay_stdio
+
+from integrations.nautilus_trader.adapter import NautilusTraderBookReplayAdapter
+
+
+class FaultyNautilusTraderBookReplayAdapter(NautilusTraderBookReplayAdapter):
+ """Inject one documented queue-priority fault into the real native book."""
+
+ def __init__(self, config):
+ super().__init__(config)
+ self.metadata = EngineMetadata(
+ "NautilusTrader L3 negative control: upsize requeue",
+ self.metadata.version,
+ self.metadata.language,
+ )
+
+ @staticmethod
+ def _apply_native_update(book, old_order, native_order, index):
+ same_level = old_order.side == native_order.side and old_order.price == native_order.price
+ is_upsize = Decimal(str(native_order.size)) > Decimal(str(old_order.size))
+ if same_level and is_upsize:
+ book.delete(old_order, 0, index, index)
+ book.add(native_order, 0, index, index)
+ return
+ NautilusTraderBookReplayAdapter._apply_native_update(
+ book,
+ old_order,
+ native_order,
+ index,
+ )
+
+
+if __name__ == "__main__":
+ raise SystemExit(serve_book_replay_stdio(FaultyNautilusTraderBookReplayAdapter))
diff --git a/integrations/nautilus_trader/regressions/upsize-requeue-failure.json b/integrations/nautilus_trader/regressions/upsize-requeue-failure.json
new file mode 100644
index 0000000..4df73b6
--- /dev/null
+++ b/integrations/nautilus_trader/regressions/upsize-requeue-failure.json
@@ -0,0 +1,38 @@
+{
+ "artifact_type": "tracebook.book-replay.negative-control",
+ "budget_exhausted": false,
+ "campaign_id": "sha256:983b4500cf7665fddb7f20d3eb5b3b53817e5852dee3eac68ae37355cde39e22",
+ "candidate_engine": {
+ "language": "Rust/Python",
+ "name": "NautilusTrader L3 negative control: upsize requeue",
+ "version": "2.0.0rc3"
+ },
+ "failure_class": "book_state:value_mismatch",
+ "failure_id": "failure-dfe5c23848b63211b655",
+ "minimized_event_count": 3,
+ "minimized_trace_sha256": "d9eeb771c7c469c0762c67be7dc76a63a09f36d46c1dd2cda1dbb3cbb47c0b29",
+ "one_minimal": true,
+ "original_divergence_event": 3,
+ "original_event_count": 3,
+ "original_trace_sha256": "d9eeb771c7c469c0762c67be7dc76a63a09f36d46c1dd2cda1dbb3cbb47c0b29",
+ "reduced_divergence": {
+ "candidate": 2,
+ "category": "book_state",
+ "event": {
+ "op": "update",
+ "order_id": 1,
+ "price": "100",
+ "quantity": "4",
+ "side": "BUY",
+ "symbol": "TEST"
+ },
+ "event_index": 3,
+ "kind": "value_mismatch",
+ "message": "candidate L3 orders or queue priority differ from the reference",
+ "path": "$.state.books[0].bids[0].order_id",
+ "reference": 1,
+ "snapshot_error": null
+ },
+ "schema_version": 1,
+ "trace_index": 1
+}
diff --git a/integrations/nautilus_trader/regressions/upsize-requeue-reduced.jsonl b/integrations/nautilus_trader/regressions/upsize-requeue-reduced.jsonl
new file mode 100644
index 0000000..cd2597e
--- /dev/null
+++ b/integrations/nautilus_trader/regressions/upsize-requeue-reduced.jsonl
@@ -0,0 +1,3 @@
+{"op":"add","order_id":1,"price":"100","quantity":"2","side":"BUY","symbol":"TEST"}
+{"op":"add","order_id":2,"price":"100","quantity":"3","side":"BUY","symbol":"TEST"}
+{"op":"update","order_id":1,"price":"100","quantity":"4","side":"BUY","symbol":"TEST"}
diff --git a/integrations/orderbook_rs/README.md b/integrations/orderbook_rs/README.md
index 4b52f58..5e4329d 100644
--- a/integrations/orderbook_rs/README.md
+++ b/integrations/orderbook_rs/README.md
@@ -30,9 +30,19 @@ flowchart LR
```
The shared [`rust_protocol`](../rust_protocol) crate owns framing, protocol
-validation, and canonical SHA-256 state serialization. `adapter.rs` owns numeric
-conversion, source IDs, owners, lifecycle operations, trade translation, and
-complete queue snapshots. `orderbook-rs` performs all matching.
+validation, canonical SHA-256 state serialization, and exact quantity encoding.
+`adapter.rs` owns price conversion, source IDs, owners, lifecycle operations,
+trade translation, and complete queue snapshots. `orderbook-rs` performs all
+matching.
+
+Quantities use fixed twelve-place native lots, regardless of the configured
+output precision. `quantity_decimal_places` rounds observations only: a `2.4`
+buy crossed by a `3.5` sell at output precision zero leaves a native `1.1`
+remainder, reported as `1`. It must not become a two-unit remainder through
+input rounding. Nonrepresentable quantities are rejected before submission;
+invalid replacement quantities preserve the existing order. See the shared
+crate's [numeric boundary](../rust_protocol/README.md) for range and precision
+limits, including accepted configurations that remain unqualified.
## Run The Proof
@@ -220,8 +230,14 @@ artifacts under `target/` are excluded.
rounding, are stored as integer ticks, and return as canonical decimal
strings. This is intentionally not exact decimal division at half-tick
boundaries; for example, `1.015 / 0.01` snaps to `1.01`.
-- Quantities use fixed-point `u64` units at `quantity_decimal_places`; a value
- that rounds to zero or overflows that range is rejected.
+- Quantities use fixed-point `u64` units at 12 native decimal places. Inputs
+ requiring finer precision or exceeding that native range are rejected without
+ rounding. `quantity_decimal_places` controls output normalization only; it
+ never changes the quantity submitted to the engine.
+ If output normalization would turn a positive fill or remainder into zero,
+ the session fails with an adapter error before emitting an invalid observation.
+ Choose enough output precision for the trace; low-precision configurations
+ outside the recorded profiles are not qualified by those results.
- Real owners map deterministically to `Hash32`. Anonymous owners receive a
unique per-order identity so STP does not make unrelated anonymous orders
self-match or reject them for a missing user ID.
diff --git a/integrations/orderbook_rs/src/adapter.rs b/integrations/orderbook_rs/src/adapter.rs
index 078b736..c2e15ae 100644
--- a/integrations/orderbook_rs/src/adapter.rs
+++ b/integrations/orderbook_rs/src/adapter.rs
@@ -1,6 +1,6 @@
use crate::wire::{
- BookSnapshot, BookState, ConfigWire, MarketEvent, ObservationFrame, Outcome, RestingOrder,
- TradeFill,
+ BookSnapshot, BookState, ConfigWire, MarketEvent, ObservationFrame, Outcome, QuantityEncoding,
+ RestingOrder, TradeFill,
};
#[cfg(all(feature = "current", feature = "historical-issue-88"))]
compile_error!("select either current or historical-issue-88, not both");
@@ -19,8 +19,8 @@ use pricelevel::{
use pricelevel_issue_88::{
Hash32, Id, OrderType, OrderUpdate, PriceLevelSnapshot, Quantity, Side, TimeInForce,
};
-use rust_decimal::prelude::{FromPrimitive, ToPrimitive};
-use rust_decimal::{Decimal, RoundingStrategy};
+use rust_decimal::Decimal;
+use rust_decimal::prelude::FromPrimitive;
use std::collections::{BTreeMap, BTreeSet};
use std::str::FromStr;
use std::sync::{Arc, Mutex};
@@ -43,8 +43,7 @@ pub enum FaultMode {
struct AdapterConfig {
tick_size: Decimal,
tick_size_f64: f64,
- quantity_decimal_places: u32,
- quantity_scale: u64,
+ quantity_encoding: QuantityEncoding,
stp_mode: STPMode,
}
@@ -53,9 +52,7 @@ impl AdapterConfig {
if !matches!(config.matching_algorithm.as_str(), "fifo" | "pro_rata") {
return Err("matching_algorithm must be 'fifo' or 'pro_rata'".to_string());
}
- if config.quantity_decimal_places > 18 {
- return Err("quantity_decimal_places must be between 0 and 18".to_string());
- }
+ let quantity_encoding = QuantityEncoding::new(config.quantity_decimal_places)?;
let tick_size = Decimal::from_str(&config.tick_size)
.map_err(|_| "tick_size must be a positive finite decimal".to_string())?;
@@ -81,15 +78,10 @@ impl AdapterConfig {
);
}
};
- let quantity_scale = 10_u64
- .checked_pow(config.quantity_decimal_places)
- .ok_or_else(|| "quantity scale exceeds u64".to_string())?;
-
Ok(Self {
tick_size,
tick_size_f64,
- quantity_decimal_places: config.quantity_decimal_places,
- quantity_scale,
+ quantity_encoding,
stp_mode,
})
}
@@ -110,25 +102,7 @@ impl AdapterConfig {
}
fn quantity_units(&self, value: &serde_json::Number) -> Result {
- let quantity = Decimal::from_str(&value.to_string())
- .map_err(|_| "quantity must be a finite decimal".to_string())?;
- if quantity <= Decimal::ZERO {
- return Err("quantity must be positive".to_string());
- }
- let normalized = quantity.round_dp_with_strategy(
- self.quantity_decimal_places,
- RoundingStrategy::MidpointNearestEven,
- );
- let scaled = normalized
- .checked_mul(Decimal::from(self.quantity_scale))
- .ok_or_else(|| "quantity exceeds the adapter's integer range".to_string())?;
- let units = scaled
- .to_u64()
- .ok_or_else(|| "quantity exceeds the adapter's integer range".to_string())?;
- if units == 0 {
- return Err("quantity rounds to zero at the configured precision".to_string());
- }
- Ok(units)
+ self.quantity_encoding.units(value)
}
fn format_price(&self, ticks: u128) -> Result {
@@ -140,9 +114,8 @@ impl AdapterConfig {
Ok(canonical_decimal(price))
}
- fn format_quantity(&self, units: u64) -> String {
- let quantity = Decimal::from(units) / Decimal::from(self.quantity_scale);
- canonical_decimal(quantity)
+ fn format_quantity(&self, units: u64) -> Result {
+ self.quantity_encoding.format(units)
}
}
@@ -641,7 +614,7 @@ impl Adapter {
buy_order_id,
sell_order_id,
price: self.config.format_price(trade.price().as_u128())?,
- quantity: self.config.format_quantity(trade.quantity().as_u64()),
+ quantity: self.config.format_quantity(trade.quantity().as_u64())?,
});
}
}
@@ -682,7 +655,7 @@ impl Adapter {
Ok(RestingOrder {
order_id,
price: self.config.format_price(order.price().as_u128())?,
- remaining_quantity: self.config.format_quantity(visible_quantity(order)),
+ remaining_quantity: self.config.format_quantity(visible_quantity(order))?,
owner,
order_type: "LIMIT",
})
@@ -800,12 +773,124 @@ mod tests {
}
#[test]
- fn quantity_conversion_uses_half_even_rounding() {
+ fn quantity_conversion_rounds_only_observations() {
let config = config();
let low = serde_json::Number::from_str("1.2345").unwrap();
let high = serde_json::Number::from_str("1.2355").unwrap();
- assert_eq!(config.quantity_units(&low).unwrap(), 1_234);
- assert_eq!(config.quantity_units(&high).unwrap(), 1_236);
+ assert_eq!(config.quantity_units(&low).unwrap(), 1_234_500_000_000);
+ assert_eq!(config.quantity_units(&high).unwrap(), 1_235_500_000_000);
+ assert_eq!(config.format_quantity(1_234_500_000_000).unwrap(), "1.234");
+ assert_eq!(config.format_quantity(1_235_500_000_000).unwrap(), "1.236");
+ }
+
+ #[test]
+ fn output_precision_does_not_change_native_matching() {
+ let mut adapter = Adapter::new(ConfigWire {
+ quantity_decimal_places: 0,
+ ..ConfigWire::default()
+ })
+ .unwrap();
+ let values = [
+ serde_json::json!({"op":"new","symbol":"TEST","order_id":1,"side":"BUY","price":100,"quantity":2.4}),
+ serde_json::json!({"op":"new","symbol":"TEST","order_id":2,"side":"SELL","price":100,"quantity":3.5}),
+ ];
+ let mut last = None;
+ for (index, value) in values.into_iter().enumerate() {
+ let event = serde_json::from_value::(value).unwrap();
+ let observation = adapter.apply(&event, index as u64 + 1).unwrap();
+ assert_eq!(observation.outcome.status, "applied");
+ last = Some(observation);
+ }
+ let observation = last.unwrap();
+ assert_eq!(observation.trades.len(), 1);
+ assert_eq!(observation.trades[0].quantity, "2");
+ assert_eq!(
+ observation.state_hash,
+ "443d82df2af0902942c8614bcce6dc8d763a24db85e81917cc4843a9f4726300"
+ );
+ let state = adapter.snapshot().unwrap();
+ assert_eq!(state.books[0].asks.len(), 1);
+ assert_eq!(state.books[0].asks[0].remaining_quantity, "1");
+ }
+
+ #[test]
+ fn positive_fills_and_remainders_cannot_be_emitted_as_zero() {
+ for (maker, taker) in [(0.4, None), (1.0, Some(0.4)), (1.0, Some(0.6))] {
+ let mut adapter = Adapter::new(ConfigWire {
+ quantity_decimal_places: 0,
+ ..ConfigWire::default()
+ })
+ .unwrap();
+ let first = serde_json::from_value::(serde_json::json!({
+ "op":"new","symbol":"TEST","order_id":1,"side":"BUY","price":100,"quantity":maker
+ }))
+ .unwrap();
+ let result = if let Some(quantity) = taker {
+ adapter.apply(&first, 1).unwrap();
+ let second = serde_json::from_value::(serde_json::json!({
+ "op":"new","symbol":"TEST","order_id":2,"side":"SELL","price":100,"quantity":quantity
+ })).unwrap();
+ adapter.apply(&second, 2)
+ } else {
+ adapter.apply(&first, 1)
+ };
+ assert!(
+ result
+ .unwrap_err()
+ .contains("positive quantity cannot be represented")
+ );
+ }
+ }
+
+ #[test]
+ fn output_precision_does_not_round_reduce_or_replace_inputs() {
+ let mut adapter = Adapter::new(ConfigWire {
+ quantity_decimal_places: 0,
+ ..ConfigWire::default()
+ })
+ .unwrap();
+ for (index, value) in [
+ serde_json::json!({"op":"new","symbol":"TEST","order_id":1,"side":"BUY","price":100,"quantity":2.4}),
+ serde_json::json!({"op":"reduce","symbol":"TEST","order_id":1,"quantity":0.4}),
+ serde_json::json!({"op":"replace","symbol":"TEST","order_id":1,"quantity":3.5}),
+ serde_json::json!({"op":"new","symbol":"TEST","order_id":2,"side":"SELL","price":100,"quantity":1.4}),
+ ].into_iter().enumerate() {
+ let event = serde_json::from_value::(value).unwrap();
+ let observation = adapter.apply(&event, index as u64 + 1).unwrap();
+ assert_eq!(observation.outcome.status, "applied");
+ }
+ let state = adapter.snapshot().unwrap();
+ assert_eq!(state.books[0].bids.len(), 1);
+ assert_eq!(state.books[0].bids[0].remaining_quantity, "2");
+ }
+
+ #[test]
+ fn nonrepresentable_quantities_preserve_existing_state() {
+ let mut adapter = Adapter::new(ConfigWire::default()).unwrap();
+ let first = serde_json::from_value::(serde_json::json!({
+ "op":"new","symbol":"TEST","order_id":1,"side":"BUY","price":100,"quantity":2.4,"owner":9
+ })).unwrap();
+ let initial = adapter.apply(&first, 1).unwrap();
+ for quantity in ["0.0000000000001", "18446745"] {
+ for op in ["new", "reduce", "replace"] {
+ let event = serde_json::from_value::(serde_json::json!({
+ "op":op,"symbol":"TEST","order_id":if op == "new" {2} else {1},
+ "side":"BUY","price":100,"quantity":serde_json::Number::from_str(quantity).unwrap()
+ })).unwrap();
+ let observation = adapter.apply(&event, 2).unwrap();
+ assert_eq!(observation.outcome.status, "rejected");
+ assert_eq!(
+ observation.outcome.reason,
+ Some(if op == "replace" {
+ "INVALID_REPLACEMENT"
+ } else {
+ "INVALID_ORDER"
+ })
+ );
+ assert_eq!(observation.state_hash, initial.state_hash);
+ assert!(observation.trades.is_empty());
+ }
+ }
}
#[test]
diff --git a/integrations/rust_protocol/README.md b/integrations/rust_protocol/README.md
index 215605c..6e1a4ee 100644
--- a/integrations/rust_protocol/README.md
+++ b/integrations/rust_protocol/README.md
@@ -4,6 +4,26 @@ This private crate owns Tracebook protocol v1 framing, validation, canonical
book-state serialization, hashing, and the stdin/stdout server loop used by the
native Rust engine adapters.
+The crate also supplies the exact `QuantityEncoding` boundary shared by
+`orderbook_rs` and `gocronx_matcher`. Quantities become integer native lots at a
+fixed twelve decimal places, independently of `quantity_decimal_places`.
+That protocol setting controls half-even rounding of output quantities only;
+it never changes submitted order, reduction, or replacement quantities.
+
+The individual input range is positive `u64` lots: from `0.000000000001` through
+`18446744.073709551615`. Inputs with additional non-zero decimal places or a
+larger scaled value are rejected before native submission. Scientific notation
+and insignificant trailing zeros are handled exactly. Parsing uses checked
+integer arithmetic rather than a decimal parser that could silently round a
+long mantissa into range. Each native engine retains its own capacity limits.
+
+The fixed twelve-place scale preserves the native lots used by existing
+default-precision qualification workloads. Other numeric configurations remain
+unqualified. For example, output precision eighteen exposes the native exact
+remainder `0.2` after `0.3 - 0.1`, while the binary64 reference reports
+`0.19999999999999998`. The adapters preserve native results, including such
+differences; they do not imitate the reference's arithmetic to obtain a pass.
+
Engine-specific crates implement `EngineAdapter` and provide a constructor.
They must not copy protocol frames or server behavior locally. This keeps every
Rust candidate on the same observation contract while leaving matching and
diff --git a/integrations/rust_protocol/src/lib.rs b/integrations/rust_protocol/src/lib.rs
index 337572e..c3ee386 100644
--- a/integrations/rust_protocol/src/lib.rs
+++ b/integrations/rust_protocol/src/lib.rs
@@ -3,8 +3,10 @@ use serde_json::{Number, Value};
use sha2::{Digest, Sha256};
use std::io::{self, Write};
+mod quantity;
mod server;
+pub use quantity::QuantityEncoding;
pub use server::{EngineAdapter, EngineIdentity, run};
pub const PROTOCOL_NAME: &str = "tracebook.conformance";
diff --git a/integrations/rust_protocol/src/quantity.rs b/integrations/rust_protocol/src/quantity.rs
new file mode 100644
index 0000000..25ab00d
--- /dev/null
+++ b/integrations/rust_protocol/src/quantity.rs
@@ -0,0 +1,220 @@
+use serde_json::Number;
+
+const NATIVE_DECIMAL_PLACES: u32 = 12;
+
+/// Exact u64 lots shared by the two native Rust adapters.
+/// Output precision never changes the quantity submitted to either engine.
+#[derive(Clone)]
+pub struct QuantityEncoding {
+ output_decimal_places: u32,
+}
+
+impl QuantityEncoding {
+ pub fn new(output_decimal_places: u32) -> Result {
+ if output_decimal_places > 18 {
+ return Err("quantity_decimal_places must be between 0 and 18".to_string());
+ }
+ Ok(Self {
+ output_decimal_places,
+ })
+ }
+
+ pub fn units(&self, value: &Number) -> Result {
+ let text = value.to_string();
+ if text.starts_with('-') {
+ return Err("quantity must be positive".to_string());
+ }
+ let range_error = || "quantity exceeds the adapter's integer range".to_string();
+ let (mantissa, exponent) = match text.split_once(['e', 'E']) {
+ Some((mantissa, exponent)) => (
+ mantissa,
+ exponent.parse::().map_err(|_| range_error())?,
+ ),
+ None => (text.as_str(), 0),
+ };
+ let fractional_places = mantissa.split_once('.').map_or(0, |(_, part)| part.len());
+ let digits = mantissa.replace('.', "");
+ let significant = digits.trim_start_matches('0');
+ if significant.is_empty() {
+ return Err("quantity must be positive".to_string());
+ }
+ let coefficient = significant.trim_end_matches('0');
+ let trailing_zeros = significant.len() - coefficient.len();
+ let shift = exponent
+ .checked_sub(i64::try_from(fractional_places).map_err(|_| range_error())?)
+ .and_then(|value| value.checked_add(i64::from(NATIVE_DECIMAL_PLACES)))
+ .and_then(|value| value.checked_add(i64::try_from(trailing_zeros).ok()?))
+ .ok_or_else(range_error)?;
+ if shift < 0 {
+ return Err(format!(
+ "quantity requires more than {NATIVE_DECIMAL_PLACES} native decimal places"
+ ));
+ }
+ // Parse only the significant digits after removing exact decimal zeros.
+ // No binary float or fixed-precision decimal parser may round this input.
+ let coefficient = coefficient.parse::().map_err(|_| range_error())?;
+ let multiplier = u32::try_from(shift)
+ .ok()
+ .and_then(|shift| 10_u64.checked_pow(shift))
+ .ok_or_else(range_error)?;
+ coefficient.checked_mul(multiplier).ok_or_else(range_error)
+ }
+
+ pub fn format(&self, units: u64) -> Result {
+ if units == 0 {
+ return Err("quantity must be positive".to_string());
+ }
+ let places = self.output_decimal_places.min(NATIVE_DECIMAL_PLACES);
+ let divisor = 10_u64.pow(NATIVE_DECIMAL_PLACES - places);
+ let mut rounded = units / divisor;
+ let remainder = units % divisor;
+ if divisor > 1
+ && (remainder > divisor / 2 || (remainder == divisor / 2 && rounded % 2 == 1))
+ {
+ // With divisor >= 10, incrementing the quotient cannot overflow.
+ rounded += 1;
+ }
+ if rounded == 0 {
+ return Err(format!(
+ "positive quantity cannot be represented at quantity_decimal_places={}",
+ self.output_decimal_places
+ ));
+ }
+ let scale = 10_u64.pow(places);
+ let whole = rounded / scale;
+ let fraction = rounded % scale;
+ if fraction == 0 {
+ return Ok(whole.to_string());
+ }
+ Ok(
+ format!("{whole}.{fraction:0width$}", width = places as usize)
+ .trim_end_matches('0')
+ .to_string(),
+ )
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use std::str::FromStr;
+
+ fn number(text: &str) -> Number {
+ Number::from_str(text).unwrap()
+ }
+
+ #[test]
+ fn output_precision_does_not_round_submitted_lots() {
+ for places in [0, 3, 12, 18] {
+ let encoding = QuantityEncoding::new(places).unwrap();
+ assert_eq!(
+ encoding.units(&number("1.2345")).unwrap(),
+ 1_234_500_000_000
+ );
+ assert_eq!(encoding.units(&number("1e-12")).unwrap(), 1);
+ }
+ }
+
+ #[test]
+ fn scientific_notation_and_insignificant_zeros_remain_exact() {
+ let encoding = QuantityEncoding::new(12).unwrap();
+ for (text, units) in [
+ ("1E+1", 10_000_000_000_000),
+ ("1.234e-9", 1_234),
+ (
+ "1234000000000000000000000000000000000e-36",
+ 1_234_000_000_000,
+ ),
+ ("1.000000000000000000000000000000000000", 1_000_000_000_000),
+ ] {
+ assert_eq!(encoding.units(&number(text)).unwrap(), units, "{text}");
+ }
+ }
+
+ #[test]
+ fn nonrepresentable_inputs_cannot_round_into_the_native_domain() {
+ let encoding = QuantityEncoding::new(0).unwrap();
+ for text in [
+ "0.0000000000011",
+ "1e-13",
+ "1.00000000000000000000000000001",
+ "100000000000000000000000000001e-29",
+ ] {
+ assert!(
+ encoding
+ .units(&number(text))
+ .unwrap_err()
+ .contains("more than 12"),
+ "{text}"
+ );
+ }
+ }
+
+ #[test]
+ fn positive_u64_range_is_checked_without_truncation() {
+ let encoding = QuantityEncoding::new(12).unwrap();
+ assert_eq!(
+ encoding.units(&number("18446744.073709551615")).unwrap(),
+ u64::MAX
+ );
+ for text in [
+ "0",
+ "-1",
+ "18446744.073709551616",
+ "1e99",
+ "1e9999999999999999999",
+ ] {
+ assert!(encoding.units(&number(text)).is_err(), "{text}");
+ }
+ assert_eq!(encoding.format(u64::MAX).unwrap(), "18446744.073709551615");
+ }
+
+ #[test]
+ fn observation_rounding_is_half_even_and_canonical() {
+ for (places, text, expected) in [
+ (3, "1.2345", "1.234"),
+ (3, "1.2355", "1.236"),
+ (0, "2.5", "2"),
+ (0, "3.5", "4"),
+ (18, "0.2", "0.2"),
+ (12, "1e-12", "0.000000000001"),
+ ] {
+ let encoding = QuantityEncoding::new(places).unwrap();
+ assert_eq!(
+ encoding
+ .format(encoding.units(&number(text)).unwrap())
+ .unwrap(),
+ expected
+ );
+ }
+ assert!(QuantityEncoding::new(19).is_err());
+ }
+
+ #[test]
+ fn unrepresentable_positive_observations_fail_instead_of_emitting_zero() {
+ for (places, text) in [(0, "0.4"), (0, "0.5"), (3, "0.0004"), (11, "1e-12")] {
+ let encoding = QuantityEncoding::new(places).unwrap();
+ let units = encoding.units(&number(text)).unwrap();
+ assert!(units > 0);
+ assert!(
+ encoding
+ .format(units)
+ .unwrap_err()
+ .contains("cannot be represented")
+ );
+ }
+ }
+
+ #[test]
+ fn zero_observations_report_nonpositive_quantity_not_precision_loss() {
+ for places in [0, 12, 18] {
+ assert_eq!(
+ QuantityEncoding::new(places)
+ .unwrap()
+ .format(0)
+ .unwrap_err(),
+ "quantity must be positive"
+ );
+ }
+ }
+}
diff --git a/pyproject.toml b/pyproject.toml
index 7403bea..e40ce2f 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -86,6 +86,7 @@ tracebook = ["py.typed"]
"tracebook.corpus.fixtures" = ["coinbase-btcusd-synthetic-v1/*"]
"tracebook.conformance.fixtures.v1" = ["*.json", "*.jsonl"]
"tracebook.conformance.fixtures.v2" = ["*.json", "*.jsonl"]
+"tracebook.book_replay.fixtures" = ["*.jsonl"]
"tracebook.visualization.web" = ["*.html", "*.css", "*.js"]
[tool.pytest.ini_options]
@@ -95,6 +96,9 @@ addopts = "-q"
[tool.black]
line-length = 100
target-version = ["py310"]
+# This runner is hash-bound by the frozen v2 cohort and cannot be reformatted
+# without invalidating the preregistration.
+extend-exclude = "^/experiments/agent_qualification_generalization.py$"
[tool.mypy]
python_version = "3.10"
@@ -103,6 +107,7 @@ check_untyped_defs = true
no_implicit_optional = true
warn_redundant_casts = true
warn_unused_configs = true
+exclude = "^experiments/private/"
[tool.coverage.run]
source = ["tracebook"]
diff --git a/src/tracebook/book_replay/__init__.py b/src/tracebook/book_replay/__init__.py
new file mode 100644
index 0000000..b17fd36
--- /dev/null
+++ b/src/tracebook/book_replay/__init__.py
@@ -0,0 +1,78 @@
+"""A separate, explicitly non-CLOB L3 book-replay conformance surface."""
+
+from .campaign import (
+ BOOK_REPLAY_CAPABILITIES,
+ BOOK_REPLAY_GENERATOR_VERSION,
+ BookReplayCampaignFailure,
+ BookReplayCampaignResult,
+ BookReplayCampaignTrace,
+ BookReplayCoverage,
+ generate_book_replay_trace,
+ measure_book_replay_coverage,
+ run_book_replay_campaign,
+)
+from .compare import BookReplayDivergence, BookReplayReport, run_book_replay
+from .external import (
+ BookReplayProtocolError,
+ ExternalBookReplayAdapter,
+ ExternalBookReplayAdapterFactory,
+)
+from .model import (
+ ARTIFACT_SCHEMA_VERSION,
+ PROFILE_NAME,
+ PROTOCOL_NAME,
+ PROTOCOL_VERSION,
+ BookReplayConfig,
+ BookReplayError,
+ BookReplayEvent,
+ BookReplayObservation,
+ BookReplaySnapshot,
+ BookReplayState,
+ EngineMetadata,
+ Outcome,
+ RestingBookOrder,
+ SimulatedFill,
+ load_book_replay_events,
+)
+from .minimize import BookReplayMinimizationResult, minimize_book_replay_failure
+from .protocol import BookReplayAdapter, BookReplayAdapterFactory, serve_book_replay_stdio
+from .reference import ReferenceBookReplayAdapter
+
+__all__ = [
+ "ARTIFACT_SCHEMA_VERSION",
+ "BOOK_REPLAY_CAPABILITIES",
+ "BOOK_REPLAY_GENERATOR_VERSION",
+ "PROFILE_NAME",
+ "PROTOCOL_NAME",
+ "PROTOCOL_VERSION",
+ "BookReplayAdapter",
+ "BookReplayAdapterFactory",
+ "BookReplayCampaignFailure",
+ "BookReplayCampaignResult",
+ "BookReplayCampaignTrace",
+ "BookReplayConfig",
+ "BookReplayCoverage",
+ "BookReplayDivergence",
+ "BookReplayError",
+ "BookReplayEvent",
+ "BookReplayObservation",
+ "BookReplayMinimizationResult",
+ "BookReplayProtocolError",
+ "BookReplayReport",
+ "BookReplaySnapshot",
+ "BookReplayState",
+ "EngineMetadata",
+ "ExternalBookReplayAdapter",
+ "ExternalBookReplayAdapterFactory",
+ "Outcome",
+ "ReferenceBookReplayAdapter",
+ "RestingBookOrder",
+ "SimulatedFill",
+ "load_book_replay_events",
+ "generate_book_replay_trace",
+ "measure_book_replay_coverage",
+ "minimize_book_replay_failure",
+ "run_book_replay",
+ "run_book_replay_campaign",
+ "serve_book_replay_stdio",
+]
diff --git a/src/tracebook/book_replay/__main__.py b/src/tracebook/book_replay/__main__.py
new file mode 100644
index 0000000..6a14a89
--- /dev/null
+++ b/src/tracebook/book_replay/__main__.py
@@ -0,0 +1,5 @@
+"""Run the book-replay CLI with ``python -m tracebook.book_replay``."""
+
+from .cli import main
+
+raise SystemExit(main())
diff --git a/src/tracebook/book_replay/artifacts.py b/src/tracebook/book_replay/artifacts.py
new file mode 100644
index 0000000..376ffec
--- /dev/null
+++ b/src/tracebook/book_replay/artifacts.py
@@ -0,0 +1,287 @@
+"""Reserve and atomically publish new L3 report and trace files."""
+
+from __future__ import annotations
+
+import os
+import uuid
+from contextlib import ExitStack, contextmanager
+from pathlib import Path
+from typing import Iterable, Iterator, Optional
+
+from .model import BookReplayError
+
+
+def _identity(stat: os.stat_result) -> tuple[int, int]:
+ return stat.st_dev, stat.st_ino
+
+
+def _write_payload(descriptor: int, payload: bytes) -> None:
+ with os.fdopen(os.dup(descriptor), "wb") as stream:
+ stream.write(payload)
+ stream.flush()
+ os.fsync(stream.fileno())
+
+
+class OutputReservation:
+ """Own a sidecar reservation while the destination itself stays absent."""
+
+ def __init__(self, destination: str | Path) -> None:
+ path = Path(destination).expanduser()
+ self._requested_target = path.absolute()
+ try:
+ self.target = path.parent.resolve() / path.name
+ except RuntimeError as exc:
+ raise BookReplayError(f"cannot resolve book-replay path: {exc}") from exc
+ self._lock_name = f".{self.target.name}.tracebook-in-progress"
+ self._stage_directory_name = f".tracebook-stage-{uuid.uuid4().hex}"
+ self._stage_name = "payload"
+ self._parent_fd: Optional[int] = None
+ self._stage_directory_fd: Optional[int] = None
+ self._stage_directory_identity: Optional[tuple[int, int]] = None
+ self._lock_fd: Optional[int] = None
+ self._stage_fd: Optional[int] = None
+ self._published = False
+
+ def __enter__(self) -> "OutputReservation":
+ if not all(operation in os.supports_dir_fd for operation in (os.open, os.stat, os.link)):
+ raise BookReplayError(
+ "L3 artifact publication requires descriptor-relative file operations"
+ )
+ try:
+ self.target.parent.mkdir(parents=True, exist_ok=True)
+ self._parent_fd = os.open(
+ self.target.parent,
+ os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0),
+ )
+ self._require_absent()
+ flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0)
+ self._lock_fd = os.open(self._lock_name, flags, 0o600, dir_fd=self._parent_fd)
+ os.mkdir(self._stage_directory_name, mode=0o700, dir_fd=self._parent_fd)
+ # Retain the identity even if opening the newly created directory
+ # fails. Cleanup must not remove a replacement at the same name.
+ try:
+ self._stage_directory_identity = _identity(
+ os.stat(
+ self._stage_directory_name, dir_fd=self._parent_fd, follow_symlinks=False
+ )
+ )
+ except OSError as exc:
+ # mkdir gives no inode/descriptor. Do not guess ownership or
+ # remove a replacement when this first identity read fails.
+ residue = self.target.parent / self._stage_directory_name
+ raise BookReplayError(
+ "staging directory ownership could not be verified; "
+ "automatic directory cleanup skipped; "
+ f"possible staging residue at original path {str(residue)!r}. "
+ f"Inspect its location and ownership before manual cleanup: {exc}"
+ ) from exc
+ stage_directory_fd = os.open(
+ self._stage_directory_name,
+ os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0),
+ dir_fd=self._parent_fd,
+ )
+ try:
+ if _identity(os.fstat(stage_directory_fd)) != self._stage_directory_identity:
+ raise BookReplayError(f"book-replay output reservation changed: {self.target}")
+ except BaseException:
+ os.close(stage_directory_fd)
+ raise
+ self._stage_directory_fd = stage_directory_fd
+ self._stage_fd = os.open(
+ self._stage_name, flags, 0o400, dir_fd=self._stage_directory_fd
+ )
+ # Keep the source entry in a private, non-writable directory. The
+ # write descriptor remains usable, but another normal file writer
+ # cannot replace the source entry in the validate/link window.
+ # Linking relative to this directory's descriptor also survives a
+ # competing rename/replacement of its entry in the output parent.
+ os.fchmod(self._stage_directory_fd, 0o500)
+ self.validate()
+ except BaseException:
+ try:
+ self.close()
+ except OSError:
+ pass
+ raise
+ return self
+
+ def __exit__(self, exc_type, exc_value, traceback) -> None:
+ try:
+ self.close()
+ except OSError:
+ if exc_type is None:
+ raise
+
+ def _require_absent(self) -> None:
+ try:
+ os.stat(self.target.name, dir_fd=self._parent_fd, follow_symlinks=False)
+ except FileNotFoundError:
+ return
+ raise BookReplayError(f"book-replay output already exists: {self.target}")
+
+ def _owns_entry(
+ self, name: str, descriptor: Optional[int], *, directory: Optional[int] = None
+ ) -> bool:
+ directory = self._parent_fd if directory is None else directory
+ if directory is None or descriptor is None:
+ return False
+ try:
+ return _identity(os.stat(name, dir_fd=directory, follow_symlinks=False)) == _identity(
+ os.fstat(descriptor)
+ )
+ except OSError:
+ return False
+
+ def validate(self) -> None:
+ if self._parent_fd is None:
+ raise BookReplayError("book-replay output reservation is closed")
+ if (
+ not self._same_parent()
+ or not self._owns_entry(self._lock_name, self._lock_fd)
+ or not self._owns_entry(self._stage_directory_name, self._stage_directory_fd)
+ or not self._owns_entry(
+ self._stage_name, self._stage_fd, directory=self._stage_directory_fd
+ )
+ ):
+ raise BookReplayError(f"book-replay output reservation changed: {self.target}")
+ self._require_absent()
+
+ def _owns_stage_directory(self) -> bool:
+ if self._parent_fd is None or self._stage_directory_identity is None:
+ return False
+ try:
+ return (
+ _identity(
+ os.stat(
+ self._stage_directory_name, dir_fd=self._parent_fd, follow_symlinks=False
+ )
+ )
+ == self._stage_directory_identity
+ )
+ except OSError:
+ return False
+
+ def _same_parent(self) -> bool:
+ if self._parent_fd is None:
+ return False
+ try:
+ expected = _identity(os.fstat(self._parent_fd))
+ return (
+ _identity(self.target.parent.stat(follow_symlinks=False)) == expected
+ and _identity(self._requested_target.parent.stat()) == expected
+ )
+ except OSError:
+ return False
+
+ def stage(self, payload: bytes) -> None:
+ self.validate()
+ if self._stage_fd is None:
+ raise BookReplayError("book-replay output reservation is closed")
+ _write_payload(self._stage_fd, payload)
+
+ def publish(self) -> None:
+ self.validate()
+ # A hard link publishes the complete staged inode atomically and fails
+ # if anything appeared at the destination after validation.
+ # Mark the attempt first: interruption immediately after a successful
+ # link still needs rollback, which checks the destination's inode.
+ self._published = True
+ os.link(
+ self._stage_name,
+ self.target.name,
+ src_dir_fd=self._stage_directory_fd,
+ dst_dir_fd=self._parent_fd,
+ follow_symlinks=False,
+ )
+ self.validate_published()
+
+ def validate_published(self) -> None:
+ if (
+ not self._published
+ or not self._same_parent()
+ or not self._owns_entry(self.target.name, self._stage_fd)
+ ):
+ raise BookReplayError(f"book-replay output changed during publication: {self.target}")
+
+ def rollback(self) -> None:
+ if self._published and self._owns_entry(self.target.name, self._stage_fd):
+ os.unlink(self.target.name, dir_fd=self._parent_fd)
+ self._published = False
+
+ def close(self) -> None:
+ error: Optional[OSError] = None
+ if self._stage_directory_fd is not None:
+ try:
+ os.fchmod(self._stage_directory_fd, 0o700)
+ except OSError as exc:
+ error = exc
+ for name, descriptor, directory in (
+ (self._stage_name, self._stage_fd, self._stage_directory_fd),
+ (self._lock_name, self._lock_fd, self._parent_fd),
+ ):
+ if descriptor is not None:
+ try:
+ if self._owns_entry(name, descriptor, directory=directory):
+ os.unlink(name, dir_fd=directory)
+ except OSError as exc:
+ error = error or exc
+ finally:
+ try:
+ os.close(descriptor)
+ except OSError as exc:
+ error = error or exc
+ self._stage_fd = self._lock_fd = None
+ try:
+ if self._owns_stage_directory():
+ os.rmdir(self._stage_directory_name, dir_fd=self._parent_fd)
+ except OSError as exc:
+ error = error or exc
+ self._stage_directory_identity = None
+ if self._stage_directory_fd is not None:
+ try:
+ os.close(self._stage_directory_fd)
+ except OSError as exc:
+ error = error or exc
+ self._stage_directory_fd = None
+ if self._parent_fd is not None:
+ try:
+ os.close(self._parent_fd)
+ except OSError as exc:
+ error = error or exc
+ self._parent_fd = None
+ if error is not None:
+ raise error
+
+
+@contextmanager
+def reserve_outputs(*destinations: Optional[str]) -> Iterator[dict[str, OutputReservation]]:
+ """Reserve every output before candidate work, including conditional traces."""
+ with ExitStack() as stack:
+ outputs = {
+ destination: stack.enter_context(OutputReservation(destination))
+ for destination in destinations
+ if destination is not None
+ }
+ # Also catch a destination colliding with another output's sidecar.
+ for output in outputs.values():
+ output.validate()
+ yield outputs
+
+
+def publish_outputs(payloads: Iterable[tuple[OutputReservation, bytes]]) -> None:
+ """Stage all bytes, then publish in order; roll back this attempt on failure."""
+ prepared = list(payloads)
+ try:
+ for output, payload in prepared:
+ output.stage(payload)
+ for output, _payload in prepared:
+ output.publish()
+ for output, _payload in prepared:
+ output.validate_published()
+ except BaseException:
+ for output, _payload in reversed(prepared):
+ try:
+ output.rollback()
+ except OSError:
+ pass
+ raise
diff --git a/src/tracebook/book_replay/campaign.py b/src/tracebook/book_replay/campaign.py
new file mode 100644
index 0000000..f860485
--- /dev/null
+++ b/src/tracebook/book_replay/campaign.py
@@ -0,0 +1,533 @@
+"""Deterministic stateful campaigns for L3 book-replay adapters."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from dataclasses import dataclass, replace
+from decimal import Decimal
+from numbers import Integral
+from typing import Dict, List, Optional, Sequence, Tuple, TypeVar
+
+from .compare import BookReplayReport, run_book_replay
+from .minimize import BookReplayMinimizationResult, minimize_book_replay_failure
+from .model import (
+ ARTIFACT_SCHEMA_VERSION,
+ PROFILE_NAME,
+ BookReplayConfig,
+ BookReplayError,
+ BookReplayEvent,
+ BookReplayState,
+ EngineMetadata,
+ trace_sha256,
+)
+from .protocol import BookReplayAdapterFactory
+from .reference import ReferenceBookReplayAdapter
+
+BOOK_REPLAY_GENERATOR_VERSION = 1
+BOOK_REPLAY_CAPABILITIES = (
+ "l3-add",
+ "same-level-fifo",
+ "same-level-upsize",
+ "price-relocation",
+ "l3-delete",
+ "duplicate-active-order-id",
+ "inactive-lifecycle-request",
+ "buy-fill-probe",
+ "sell-fill-probe",
+ "multi-level-depth",
+ "multiple-symbols",
+)
+_MASK_64 = (1 << 64) - 1
+_GOLDEN_GAMMA = 0x9E3779B97F4A7C15
+_CHOICE = TypeVar("_CHOICE")
+
+
+class _SplitMix64:
+ """Specified PRNG whose output is independent of Python's random module."""
+
+ def __init__(self, seed: int) -> None:
+ self._state = seed & _MASK_64
+
+ def next_u64(self) -> int:
+ self._state = (self._state + _GOLDEN_GAMMA) & _MASK_64
+ value = self._state
+ value = ((value ^ (value >> 30)) * 0xBF58476D1CE4E5B9) & _MASK_64
+ value = ((value ^ (value >> 27)) * 0x94D049BB133111EB) & _MASK_64
+ return (value ^ (value >> 31)) & _MASK_64
+
+ def randbelow(self, bound: int) -> int:
+ if bound <= 0:
+ raise ValueError("bound must be positive")
+ return self.next_u64() % bound
+
+ def choice(self, values: Sequence[_CHOICE]) -> _CHOICE:
+ if not values:
+ raise ValueError("cannot choose from an empty sequence")
+ return values[self.randbelow(len(values))]
+
+
+def _positive_int(value: int, field_name: str) -> int:
+ if isinstance(value, bool) or not isinstance(value, Integral) or value <= 0:
+ raise BookReplayError(f"{field_name} must be a positive integer")
+ return int(value)
+
+
+def _campaign_seed(value: int) -> int:
+ if isinstance(value, bool) or not isinstance(value, Integral) or not 0 <= value <= _MASK_64:
+ raise BookReplayError("seed must be an integer between 0 and 2^64-1")
+ return int(value)
+
+
+def _trace_seed(seed: int, trace_index: int) -> int:
+ mixer = _SplitMix64(seed ^ ((trace_index * _GOLDEN_GAMMA) & _MASK_64))
+ return mixer.next_u64()
+
+
+def _semantic_scaffold() -> Tuple[BookReplayEvent, ...]:
+ """Guarantee every v1 capability before random exploration starts."""
+ rows = (
+ ("add", "TEST", 1, "BUY", "100", "2"),
+ ("add", "TEST", 2, "BUY", "100", "3"),
+ ("update", "TEST", 1, "BUY", "100", "4"),
+ ("add", "TEST", 3, "BUY", "99", "5"),
+ ("add", "TEST", 10, "SELL", "101", "1.5"),
+ ("add", "TEST", 11, "SELL", "101", "2.5"),
+ ("add", "TEST", 12, "SELL", "102", "4"),
+ ("probe", "TEST", None, "BUY", "102", "5"),
+ ("probe", "TEST", None, "SELL", "99", "8"),
+ ("update", "TEST", 2, "BUY", "99", "3"),
+ ("probe", "TEST", None, "SELL", "99", "12"),
+ ("delete", "TEST", 1, None, None, None),
+ ("delete", "TEST", 10, None, None, None),
+ ("probe", "TEST", None, "BUY", "101", "3"),
+ ("add", "ALT", 1, "SELL", "50", "7"),
+ ("probe", "ALT", None, "BUY", "50", "2"),
+ ("delete", "TEST", 999, None, None, None),
+ ("add", "TEST", 2, "BUY", "99", "1"),
+ )
+ return tuple(
+ BookReplayEvent(
+ op=op,
+ symbol=symbol,
+ order_id=order_id,
+ side=side,
+ price=price,
+ quantity=quantity,
+ )
+ for op, symbol, order_id, side, price, quantity in rows
+ )
+
+
+@dataclass(frozen=True)
+class _ActiveOrder:
+ symbol: str
+ order_id: int
+ side: str
+ price: str
+ quantity: str
+
+
+def _active_orders(state: BookReplayState) -> Tuple[_ActiveOrder, ...]:
+ active = []
+ for book in state.books:
+ for side, orders in (("BUY", book.bids), ("SELL", book.asks)):
+ for order in orders:
+ active.append(
+ _ActiveOrder(
+ book.symbol,
+ order.order_id,
+ side,
+ order.price,
+ order.quantity,
+ )
+ )
+ return tuple(sorted(active, key=lambda item: (item.symbol, item.order_id)))
+
+
+def _new_event(
+ rng: _SplitMix64,
+ next_ids: Dict[str, int],
+) -> BookReplayEvent:
+ symbol = rng.choice(tuple(next_ids))
+ order_id = next_ids[symbol]
+ next_ids[symbol] += 1
+ return BookReplayEvent(
+ "add",
+ symbol,
+ order_id,
+ rng.choice(("BUY", "SELL")),
+ rng.choice(("98", "99", "100", "101", "102")),
+ rng.choice(("0.5", "1", "2", "3", "5")),
+ )
+
+
+def _generated_event(
+ rng: _SplitMix64,
+ next_ids: Dict[str, int],
+ active: Tuple[_ActiveOrder, ...],
+) -> BookReplayEvent:
+ if not active:
+ return _new_event(rng, next_ids)
+ roll = rng.randbelow(100)
+ selected = rng.choice(active)
+ if roll < 35:
+ return _new_event(rng, next_ids)
+ if roll < 60:
+ mode = rng.randbelow(4)
+ side = selected.side if mode < 3 else ("SELL" if selected.side == "BUY" else "BUY")
+ price = selected.price if mode in {0, 1} else rng.choice(("98", "99", "100", "101", "102"))
+ quantity = (
+ str(Decimal(selected.quantity) + Decimal("1"))
+ if mode == 0
+ else rng.choice(("0.5", "1", "2", "3", "5"))
+ )
+ return BookReplayEvent(
+ "update",
+ selected.symbol,
+ selected.order_id,
+ side,
+ price,
+ quantity,
+ )
+ if roll < 74:
+ return BookReplayEvent("delete", selected.symbol, selected.order_id)
+ if roll < 91:
+ return BookReplayEvent(
+ "probe",
+ rng.choice(tuple(next_ids)),
+ side=rng.choice(("BUY", "SELL")),
+ price=rng.choice(("98", "99", "100", "101", "102")),
+ quantity=rng.choice(("0.5", "1", "2", "4", "8")),
+ )
+ if roll < 95:
+ return BookReplayEvent(
+ "add",
+ selected.symbol,
+ selected.order_id,
+ selected.side,
+ selected.price,
+ selected.quantity,
+ )
+ symbol = rng.choice(tuple(next_ids))
+ unknown_id = 1_000_000 + next_ids[symbol] + rng.randbelow(10_000)
+ if roll < 98:
+ return BookReplayEvent("delete", symbol, unknown_id)
+ return BookReplayEvent(
+ "update",
+ symbol,
+ unknown_id,
+ rng.choice(("BUY", "SELL")),
+ rng.choice(("98", "99", "100", "101", "102")),
+ rng.choice(("1", "2", "3")),
+ )
+
+
+def generate_book_replay_trace(
+ seed: int,
+ event_count: int,
+) -> Tuple[BookReplayEvent, ...]:
+ """Generate one stateful trace without consulting candidate behavior."""
+ seed = _campaign_seed(seed)
+ event_count = _positive_int(event_count, "event_count")
+ rng = _SplitMix64(seed)
+ scaffold = _semantic_scaffold()
+ next_ids = {"ALPHA": 1, "BETA": 1}
+ reference = ReferenceBookReplayAdapter(BookReplayConfig())
+ events: List[BookReplayEvent] = []
+ active: Tuple[_ActiveOrder, ...] = ()
+ try:
+ for index in range(1, event_count + 1):
+ event = scaffold[index - 1] if index <= len(scaffold) else None
+ if event is None:
+ event = _generated_event(rng, next_ids, active)
+ events.append(event)
+ reference.apply(event, index)
+ active = _active_orders(reference.snapshot())
+ finally:
+ reference.close()
+ return tuple(events)
+
+
+@dataclass(frozen=True)
+class BookReplayCoverage:
+ """Source-event capability coverage reached by compared campaign prefixes."""
+
+ covered: Tuple[str, ...]
+
+ @property
+ def uncovered(self) -> Tuple[str, ...]:
+ return tuple(name for name in BOOK_REPLAY_CAPABILITIES if name not in self.covered)
+
+ @property
+ def complete(self) -> bool:
+ return not self.uncovered
+
+ def to_dict(self) -> dict:
+ return {
+ "covered": len(self.covered),
+ "expected": len(BOOK_REPLAY_CAPABILITIES),
+ "capabilities": list(self.covered),
+ "uncovered": list(self.uncovered),
+ "complete": self.complete,
+ }
+
+
+def measure_book_replay_coverage(
+ traces: Sequence[Sequence[BookReplayEvent]],
+) -> BookReplayCoverage:
+ """Measure which v1 operations occurred before comparison stopped."""
+ covered = set()
+ symbols = set()
+ for trace in traces:
+ active: Dict[Tuple[str, int], BookReplayEvent] = {}
+ for event in trace:
+ symbols.add(event.symbol)
+ key = (event.symbol, event.order_id or 0)
+ if event.op == "add":
+ covered.add("l3-add")
+ if key in active:
+ covered.add("duplicate-active-order-id")
+ continue
+ peers = [
+ order
+ for order in active.values()
+ if order.symbol == event.symbol and order.side == event.side
+ ]
+ if any(order.price == event.price for order in peers):
+ covered.add("same-level-fifo")
+ if peers and any(order.price != event.price for order in peers):
+ covered.add("multi-level-depth")
+ active[key] = event
+ elif event.op == "update":
+ previous = active.get(key)
+ if previous is None:
+ covered.add("inactive-lifecycle-request")
+ continue
+ if previous.side == event.side and previous.price == event.price:
+ if Decimal(event.quantity or "0") > Decimal(previous.quantity or "0"):
+ covered.add("same-level-upsize")
+ elif previous.price != event.price:
+ covered.add("price-relocation")
+ active[key] = event
+ elif event.op == "delete":
+ if key in active:
+ covered.add("l3-delete")
+ del active[key]
+ else:
+ covered.add("inactive-lifecycle-request")
+ elif event.side == "BUY":
+ covered.add("buy-fill-probe")
+ else:
+ covered.add("sell-fill-probe")
+ if len(symbols) > 1:
+ covered.add("multiple-symbols")
+ return BookReplayCoverage(tuple(name for name in BOOK_REPLAY_CAPABILITIES if name in covered))
+
+
+@dataclass(frozen=True)
+class BookReplayCampaignTrace:
+ index: int
+ seed: int
+ events: Tuple[BookReplayEvent, ...]
+ report: BookReplayReport
+
+ def to_dict(self) -> dict:
+ return {
+ "index": self.index,
+ "seed": self.seed,
+ "event_count": len(self.events),
+ "trace_sha256": trace_sha256(self.events),
+ "compared_events": self.report.compared_events,
+ "conformant": self.report.conformant,
+ "divergence": self.report.divergence.to_dict() if self.report.divergence else None,
+ }
+
+
+@dataclass(frozen=True)
+class BookReplayCampaignFailure:
+ trace: BookReplayCampaignTrace
+ original_events: Tuple[BookReplayEvent, ...]
+ original_report: BookReplayReport
+ minimization: BookReplayMinimizationResult
+
+ def to_dict(self) -> dict:
+ divergence = self.original_report.divergence
+ return {
+ "trace_index": self.trace.index,
+ "failure_class": (
+ f"{divergence.category}:{divergence.kind}" if divergence else "unknown"
+ ),
+ "original_divergence_event": divergence.event_index if divergence else None,
+ "original_event_count": len(self.original_events),
+ "original_trace_sha256": trace_sha256(self.original_events),
+ "original_events": [event.to_dict() for event in self.original_events],
+ "original_report": self.original_report.to_dict(),
+ "minimized_event_count": len(self.minimization.events),
+ "minimized_trace_sha256": trace_sha256(self.minimization.events),
+ "minimized_events": [event.to_dict() for event in self.minimization.events],
+ "minimization": self.minimization.to_dict(),
+ "one_minimal": self.minimization.one_minimal,
+ "budget_exhausted": self.minimization.budget_exhausted,
+ }
+
+
+@dataclass(frozen=True)
+class BookReplayCampaignResult:
+ seed: int
+ requested_traces: int
+ events_per_trace: int
+ max_minimize_runs: int
+ candidate_engine: EngineMetadata
+ traces: Tuple[BookReplayCampaignTrace, ...]
+ failure: Optional[BookReplayCampaignFailure]
+ coverage: BookReplayCoverage
+
+ @property
+ def conformant(self) -> bool:
+ return self.failure is None
+
+ @property
+ def campaign_id(self) -> str:
+ payload = {
+ "generator_version": BOOK_REPLAY_GENERATOR_VERSION,
+ "profile": PROFILE_NAME,
+ "seed": self.seed,
+ "requested_traces": self.requested_traces,
+ "events_per_trace": self.events_per_trace,
+ }
+ encoded = json.dumps(
+ payload,
+ sort_keys=True,
+ separators=(",", ":"),
+ allow_nan=False,
+ ).encode("utf-8")
+ return "sha256:" + hashlib.sha256(encoded).hexdigest()
+
+ @property
+ def failure_id(self) -> Optional[str]:
+ if self.failure is None:
+ return None
+ divergence = self.failure.minimization.report.divergence
+ if divergence is None:
+ raise BookReplayError("campaign failure has no reduced divergence")
+ payload = {
+ "campaign_id": self.campaign_id,
+ "trace_index": self.failure.trace.index,
+ "original_trace_sha256": trace_sha256(self.failure.original_events),
+ "reduced_trace_sha256": trace_sha256(self.failure.minimization.events),
+ "reduced_divergence": divergence.to_dict(),
+ }
+ encoded = json.dumps(
+ payload,
+ sort_keys=True,
+ separators=(",", ":"),
+ allow_nan=False,
+ ).encode("utf-8")
+ return "failure-" + hashlib.sha256(encoded).hexdigest()[:20]
+
+ def to_dict(self) -> dict:
+ minimization_runs = self.failure.minimization.runs if self.failure else 0
+ failure = self.failure.to_dict() if self.failure else None
+ if failure is not None:
+ failure["failure_id"] = self.failure_id
+ return {
+ "schema_version": ARTIFACT_SCHEMA_VERSION,
+ "artifact_type": "tracebook.book-replay.campaign",
+ "generator_version": BOOK_REPLAY_GENERATOR_VERSION,
+ "campaign_id": self.campaign_id,
+ "profile": PROFILE_NAME,
+ "config": BookReplayConfig().to_dict(),
+ "seed": self.seed,
+ "requested_traces": self.requested_traces,
+ "completed_traces": len(self.traces),
+ "events_per_trace": self.events_per_trace,
+ "generated_events": sum(len(trace.events) for trace in self.traces),
+ "max_minimize_runs": self.max_minimize_runs,
+ "candidate_runs": len(self.traces) + minimization_runs,
+ "candidate_engine": self.candidate_engine.to_dict(),
+ "stopped_at_first_divergence": True,
+ "conformant": self.conformant,
+ "semantic_coverage": self.coverage.to_dict(),
+ "traces": [trace.to_dict() for trace in self.traces],
+ "failure": failure,
+ }
+
+
+def run_book_replay_campaign(
+ candidate_factory: BookReplayAdapterFactory,
+ seed: int = 1337,
+ traces: int = 25,
+ events_per_trace: int = 100,
+ max_minimize_runs: int = 100,
+) -> BookReplayCampaignResult:
+ """Run generated traces until completion or the first minimized divergence."""
+ seed = _campaign_seed(seed)
+ traces = _positive_int(traces, "traces")
+ events_per_trace = _positive_int(events_per_trace, "events_per_trace")
+ max_minimize_runs = _positive_int(max_minimize_runs, "max_minimize_runs")
+ config = BookReplayConfig()
+ results = []
+ candidate_engine: Optional[EngineMetadata] = None
+ failure: Optional[BookReplayCampaignFailure] = None
+ for trace_index in range(1, traces + 1):
+ generated_seed = _trace_seed(seed, trace_index)
+ events = generate_book_replay_trace(generated_seed, events_per_trace)
+ trace_name = f"campaign:{PROFILE_NAME}:{trace_index}"
+ report = run_book_replay(events, candidate_factory, config, trace_name=trace_name)
+ if candidate_engine is None:
+ candidate_engine = report.candidate_engine
+ elif candidate_engine != report.candidate_engine:
+ raise BookReplayError("candidate engine metadata changed between campaign traces")
+ trace_result = BookReplayCampaignTrace(trace_index, generated_seed, events, report)
+ results.append(trace_result)
+ if not report.conformant:
+ if report.divergence is None:
+ raise BookReplayError("non-conformant campaign report has no divergence")
+ original_events = events[: report.divergence.event_index]
+ original_report = replace(
+ report,
+ trace_hash=trace_sha256(original_events),
+ event_count=len(original_events),
+ )
+ minimization = minimize_book_replay_failure(
+ original_events,
+ candidate_factory,
+ config=config,
+ max_runs=max_minimize_runs,
+ trace_name=trace_name,
+ expected_candidate_engine=candidate_engine,
+ )
+ failure = BookReplayCampaignFailure(
+ trace_result,
+ original_events,
+ original_report,
+ minimization,
+ )
+ break
+ if candidate_engine is None:
+ raise BookReplayError("campaign completed without candidate metadata")
+ compared = tuple(trace.events[: trace.report.compared_events] for trace in results)
+ return BookReplayCampaignResult(
+ seed,
+ traces,
+ events_per_trace,
+ max_minimize_runs,
+ candidate_engine,
+ tuple(results),
+ failure,
+ measure_book_replay_coverage(compared),
+ )
+
+
+__all__ = [
+ "BOOK_REPLAY_CAPABILITIES",
+ "BOOK_REPLAY_GENERATOR_VERSION",
+ "BookReplayCampaignFailure",
+ "BookReplayCampaignResult",
+ "BookReplayCampaignTrace",
+ "BookReplayCoverage",
+ "generate_book_replay_trace",
+ "measure_book_replay_coverage",
+ "run_book_replay_campaign",
+]
diff --git a/src/tracebook/book_replay/cli.py b/src/tracebook/book_replay/cli.py
new file mode 100644
index 0000000..127d609
--- /dev/null
+++ b/src/tracebook/book_replay/cli.py
@@ -0,0 +1,214 @@
+"""CLI for the isolated L3 book-replay conformance surface."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import shlex
+import sys
+from importlib import resources
+from pathlib import Path
+from typing import List, Optional
+
+from .._version import __version__
+from .artifacts import OutputReservation, publish_outputs, reserve_outputs
+from .campaign import run_book_replay_campaign
+from .compare import run_book_replay
+from .external import ExternalBookReplayAdapterFactory
+from .minimize import minimize_book_replay_failure
+from .model import PROFILE_NAME, BookReplayConfig, BookReplayError, load_book_replay_events
+
+
+def _add_candidate_arguments(parser: argparse.ArgumentParser) -> None:
+ parser.add_argument("--timeout", type=float, default=5.0)
+ candidate = parser.add_mutually_exclusive_group(required=True)
+ candidate.add_argument("--candidate-cmd", help="Candidate command as one shell-style string.")
+ candidate.add_argument(
+ "--candidate",
+ nargs=argparse.REMAINDER,
+ help="Adapter command and arguments; this must be the final CLI option.",
+ )
+
+
+def _build_parser() -> argparse.ArgumentParser:
+ parser = argparse.ArgumentParser(
+ prog="python -m tracebook.book_replay",
+ description=(
+ "Compare an external L3 order-book mirror without claiming matching-engine "
+ "conformance."
+ ),
+ )
+ parser.add_argument("--version", action="version", version=f"%(prog)s {__version__}")
+ commands = parser.add_subparsers(dest="command", required=True)
+
+ sample = commands.add_parser("sample", help="Copy the bundled L3 replay trace.")
+ sample.add_argument("destination", help="New directory for the profile trace.")
+
+ run = commands.add_parser("run", help="Compare one L3 delta/probe JSONL trace.")
+ run.add_argument("events")
+ run.add_argument("--output")
+ run.add_argument("--profile", choices=[PROFILE_NAME], default=PROFILE_NAME)
+ _add_candidate_arguments(run)
+
+ minimize = commands.add_parser(
+ "minimize",
+ help="Reduce one divergent L3 trace while preserving its failure category.",
+ )
+ minimize.add_argument("events")
+ minimize.add_argument("--events-output", required=True)
+ minimize.add_argument("--output")
+ minimize.add_argument("--max-runs", type=int, default=100)
+ minimize.add_argument("--profile", choices=[PROFILE_NAME], default=PROFILE_NAME)
+ _add_candidate_arguments(minimize)
+
+ campaign = commands.add_parser(
+ "campaign",
+ help="Generate deterministic L3 traces and minimize the first divergence.",
+ )
+ campaign.add_argument("--output", required=True)
+ campaign.add_argument("--reduced-events-output")
+ campaign.add_argument("--seed", type=int, default=1337)
+ campaign.add_argument("--traces", type=int, default=25)
+ campaign.add_argument("--events-per-trace", type=int, default=100)
+ campaign.add_argument("--max-minimize-runs", type=int, default=100)
+ _add_candidate_arguments(campaign)
+ return parser
+
+
+def _candidate_factory(args) -> ExternalBookReplayAdapterFactory:
+ command = (
+ shlex.split(args.candidate_cmd) if args.candidate_cmd is not None else list(args.candidate)
+ )
+ if command and command[0] == "--":
+ command = command[1:]
+ if not command:
+ raise BookReplayError("--candidate requires a command")
+ return ExternalBookReplayAdapterFactory(command, timeout_seconds=args.timeout)
+
+
+def _copy_sample(destination: str) -> Path:
+ root = Path(destination)
+ target = root / f"{PROFILE_NAME}.jsonl"
+ if root.exists() and not root.is_dir():
+ raise BookReplayError("sample destination must be a directory")
+ source = resources.files("tracebook.book_replay.fixtures").joinpath(f"{PROFILE_NAME}.jsonl")
+ with reserve_outputs(str(target)) as outputs:
+ publish_outputs([(outputs[str(target)], source.read_bytes())])
+ return target
+
+
+def _emit(
+ payload: dict,
+ output: Optional[str],
+ outputs: dict[str, OutputReservation],
+ *,
+ events=None,
+ events_output: Optional[str] = None,
+) -> None:
+ rendered = json.dumps(payload, indent=2, sort_keys=True, allow_nan=False) + "\n"
+ files = []
+ if events_output is not None and events is not None:
+ event_bytes = "".join(
+ json.dumps(event.to_dict(), sort_keys=True, separators=(",", ":"), allow_nan=False)
+ + "\n"
+ for event in events
+ ).encode("utf-8")
+ files.append((outputs[events_output], event_bytes))
+ if output is not None:
+ files.append((outputs[output], rendered.encode("utf-8")))
+ # Publish the report last so a successful report never precedes its trace.
+ publish_outputs(files)
+ if output is None:
+ print(rendered, end="")
+ else:
+ print(f"Report written: {Path(output)}")
+
+
+def _resolve_path(path: str) -> Path:
+ try:
+ return Path(path).expanduser().resolve()
+ except RuntimeError as exc:
+ raise BookReplayError(f"cannot resolve book-replay path: {exc}") from exc
+
+
+def _require_distinct_paths(*paths: Optional[str]) -> None:
+ resolved = [_resolve_path(path) for path in paths if path is not None]
+ if len(resolved) != len(set(resolved)) or any(
+ left in right.parents or right in left.parents
+ for index, left in enumerate(resolved)
+ for right in resolved[index + 1 :]
+ ):
+ raise BookReplayError(
+ "input and output paths must be distinct and cannot contain one another"
+ )
+
+
+def main(argv: Optional[List[str]] = None) -> int:
+ args = _build_parser().parse_args(argv)
+ try:
+ if args.command == "sample":
+ target = _copy_sample(args.destination)
+ print(f"Book-replay trace copied to: {target}")
+ print(f"Profile: {PROFILE_NAME}")
+ return 0
+
+ if args.command == "campaign":
+ _require_distinct_paths(args.output, args.reduced_events_output)
+ with reserve_outputs(args.output, args.reduced_events_output) as outputs:
+ campaign = run_book_replay_campaign(
+ _candidate_factory(args),
+ seed=args.seed,
+ traces=args.traces,
+ events_per_trace=args.events_per_trace,
+ max_minimize_runs=args.max_minimize_runs,
+ )
+ _emit(
+ campaign.to_dict(),
+ args.output,
+ outputs,
+ events=campaign.failure.minimization.events if campaign.failure else None,
+ events_output=args.reduced_events_output,
+ )
+ if campaign.failure is not None and args.reduced_events_output is not None:
+ print(f"Reduced events written: {Path(args.reduced_events_output)}")
+ if campaign.failure is None:
+ return 0
+ return 2 if campaign.failure.minimization.report.operational_failure else 1
+
+ events_path = _resolve_path(args.events)
+ events_output = args.events_output if args.command == "minimize" else None
+ _require_distinct_paths(str(events_path), args.output, events_output)
+ with reserve_outputs(args.output, events_output) as outputs:
+ if args.command == "minimize":
+ minimization = minimize_book_replay_failure(
+ load_book_replay_events(str(events_path)),
+ _candidate_factory(args),
+ config=BookReplayConfig(args.profile),
+ max_runs=args.max_runs,
+ trace_name=str(events_path),
+ )
+ _emit(
+ minimization.to_dict(),
+ args.output,
+ outputs,
+ events=minimization.events,
+ events_output=args.events_output,
+ )
+ print(f"Minimized events written: {Path(args.events_output)}")
+ return 2 if minimization.report.operational_failure else 1
+ report = run_book_replay(
+ load_book_replay_events(str(events_path)),
+ _candidate_factory(args),
+ config=BookReplayConfig(args.profile),
+ trace_name=str(events_path),
+ )
+ _emit(report.to_dict(), args.output, outputs)
+ if report.operational_failure:
+ return 2
+ return 0 if report.conformant else 1
+ except (BookReplayError, OSError, TypeError, ValueError) as exc:
+ print(f"Error: {exc}", file=sys.stderr)
+ return 2
+
+
+__all__ = ["main"]
diff --git a/src/tracebook/book_replay/compare.py b/src/tracebook/book_replay/compare.py
new file mode 100644
index 0000000..c64e726
--- /dev/null
+++ b/src/tracebook/book_replay/compare.py
@@ -0,0 +1,404 @@
+"""Compare one L3 replay trace and localize the first observable drift."""
+
+from __future__ import annotations
+
+from contextlib import suppress
+from dataclasses import dataclass, replace
+from typing import Any, Iterable, Optional
+
+from ..conformance.model import first_difference
+from .model import (
+ ARTIFACT_SCHEMA_VERSION,
+ PROTOCOL_VERSION,
+ BookReplayConfig,
+ BookReplayError,
+ BookReplayEvent,
+ BookReplayObservation,
+ BookReplayState,
+ EngineMetadata,
+ trace_sha256,
+)
+from .protocol import BookReplayAdapter, BookReplayAdapterFactory
+from .reference import ReferenceBookReplayAdapter
+
+
+@dataclass(frozen=True)
+class BookReplayDivergence:
+ """First observable disagreement in a book-replay trace."""
+
+ event_index: int
+ category: str
+ kind: str
+ path: str
+ message: str
+ event: Optional[dict]
+ reference: Any
+ candidate: Any
+ snapshot_error: Optional[str] = None
+ close_error: Optional[str] = None
+
+ def to_dict(self) -> dict:
+ payload = {
+ "event_index": self.event_index,
+ "category": self.category,
+ "kind": self.kind,
+ "path": self.path,
+ "message": self.message,
+ "event": self.event,
+ "reference": self.reference,
+ "candidate": self.candidate,
+ "snapshot_error": self.snapshot_error,
+ }
+ if self.close_error is not None:
+ payload["close_error"] = self.close_error
+ return payload
+
+
+@dataclass(frozen=True)
+class BookReplayReport:
+ """Stable JSON artifact for one isolated L3 replay comparison."""
+
+ config: BookReplayConfig
+ trace_name: Optional[str]
+ trace_hash: str
+ event_count: int
+ compared_events: int
+ reference_engine: EngineMetadata
+ candidate_engine: EngineMetadata
+ conformant: bool
+ final_state_hash: Optional[str]
+ divergence: Optional[BookReplayDivergence]
+
+ @property
+ def operational_failure(self) -> bool:
+ return self.divergence is not None and (
+ self.divergence.category == "protocol"
+ or self.divergence.snapshot_error is not None
+ or self.divergence.close_error is not None
+ )
+
+ def to_dict(self) -> dict:
+ return {
+ "schema_version": ARTIFACT_SCHEMA_VERSION,
+ "artifact_type": "tracebook.book-replay.report",
+ "protocol_version": PROTOCOL_VERSION,
+ "trace": {
+ "name": self.trace_name,
+ "sha256": self.trace_hash,
+ "event_count": self.event_count,
+ },
+ "config": self.config.to_dict(),
+ "reference_engine": self.reference_engine.to_dict(),
+ "candidate_engine": self.candidate_engine.to_dict(),
+ "compared_events": self.compared_events,
+ "conformant": self.conformant,
+ "final_state_hash": self.final_state_hash,
+ "divergence": self.divergence.to_dict() if self.divergence else None,
+ }
+
+
+def _divergence(
+ event: BookReplayEvent,
+ index: int,
+ category: str,
+ difference: dict,
+ message: str,
+) -> BookReplayDivergence:
+ return BookReplayDivergence(
+ event_index=index,
+ category=category,
+ kind=difference["kind"],
+ path=difference["path"],
+ message=message,
+ event=event.to_dict(),
+ reference=difference.get("reference"),
+ candidate=difference.get("candidate"),
+ )
+
+
+def _compare_observations(
+ event: BookReplayEvent,
+ index: int,
+ reference: BookReplayObservation,
+ candidate: BookReplayObservation,
+) -> Optional[BookReplayDivergence]:
+ reference_outcome = {
+ "status": reference.outcome.status,
+ "reason": reference.outcome.reason,
+ }
+ candidate_outcome = {
+ "status": candidate.outcome.status,
+ "reason": candidate.outcome.reason,
+ }
+ difference = first_difference(reference_outcome, candidate_outcome, "$.observation.outcome")
+ if difference is not None:
+ return _divergence(
+ event,
+ index,
+ "outcome",
+ difference,
+ "candidate delta acceptance differs from the reference",
+ )
+
+ difference = first_difference(
+ [fill.to_dict() for fill in reference.fills],
+ [fill.to_dict() for fill in candidate.fills],
+ "$.observation.fills",
+ )
+ if difference is not None:
+ return _divergence(
+ event,
+ index,
+ "simulated_fills",
+ difference,
+ "candidate simulated fills differ from the reference",
+ )
+
+ if reference.order_count != candidate.order_count:
+ return BookReplayDivergence(
+ event_index=index,
+ category="book_state",
+ kind="value_mismatch",
+ path="$.observation.order_count",
+ message="candidate mirrored-order count differs from the reference",
+ event=event.to_dict(),
+ reference=reference.order_count,
+ candidate=candidate.order_count,
+ )
+ if reference.state_hash != candidate.state_hash:
+ return BookReplayDivergence(
+ event_index=index,
+ category="book_state",
+ kind="value_mismatch",
+ path="$.observation.state_hash",
+ message="candidate mirrored state differs from the reference",
+ event=event.to_dict(),
+ reference=reference.state_hash,
+ candidate=candidate.state_hash,
+ )
+ return None
+
+
+def _localize_state(
+ divergence: BookReplayDivergence,
+ reference: ReferenceBookReplayAdapter,
+ candidate: BookReplayAdapter,
+ candidate_observation: Optional[BookReplayObservation],
+) -> BookReplayDivergence:
+ try:
+ reference_state = reference.snapshot()
+ candidate_state = candidate.snapshot()
+ if not isinstance(candidate_state, BookReplayState):
+ raise BookReplayError("candidate snapshot must return BookReplayState")
+ except Exception as exc:
+ return BookReplayDivergence(
+ event_index=divergence.event_index,
+ category="protocol",
+ kind="adapter_error",
+ path="$.state",
+ message=f"candidate snapshot failed: {exc}",
+ event=divergence.event,
+ reference="valid candidate snapshot",
+ candidate=None,
+ snapshot_error=f"candidate snapshot failed: {exc}",
+ )
+
+ candidate_digest = candidate_state.digest()
+ if candidate_observation is not None and candidate_digest != candidate_observation.state_hash:
+ return BookReplayDivergence(
+ event_index=divergence.event_index,
+ category="protocol",
+ kind="invalid_state_hash",
+ path="$.observation.state_hash",
+ message="candidate state_hash does not describe its snapshot",
+ event=divergence.event,
+ reference=candidate_digest,
+ candidate=candidate_observation.state_hash,
+ )
+ difference = first_difference(reference_state.to_dict(), candidate_state.to_dict(), "$.state")
+ if difference is None:
+ return divergence
+ return BookReplayDivergence(
+ event_index=divergence.event_index,
+ category="book_state",
+ kind=difference["kind"],
+ path=difference["path"],
+ message="candidate L3 orders or queue priority differ from the reference",
+ event=divergence.event,
+ reference=difference.get("reference"),
+ candidate=difference.get("candidate"),
+ )
+
+
+def _protocol_divergence(
+ event: BookReplayEvent,
+ index: int,
+ exc: Exception,
+) -> BookReplayDivergence:
+ return BookReplayDivergence(
+ event_index=index,
+ category="protocol",
+ kind="adapter_error",
+ path="$",
+ message=str(exc),
+ event=event.to_dict(),
+ reference="valid observation",
+ candidate=None,
+ )
+
+
+def run_book_replay(
+ events: Iterable[BookReplayEvent],
+ candidate_factory: BookReplayAdapterFactory,
+ config: Optional[BookReplayConfig] = None,
+ trace_name: Optional[str] = None,
+) -> BookReplayReport:
+ """Compare one candidate book mirror and stop at its first drift."""
+ normalized_events = list(events)
+ if any(not isinstance(event, BookReplayEvent) for event in normalized_events):
+ raise BookReplayError("book-replay traces contain only BookReplayEvent values")
+ config = config or BookReplayConfig()
+ reference = ReferenceBookReplayAdapter(config)
+ candidate = candidate_factory(config)
+ candidate_metadata = getattr(candidate, "metadata", None)
+ if not isinstance(candidate_metadata, EngineMetadata):
+ with suppress(Exception):
+ candidate.close()
+ reference.close()
+ raise BookReplayError("candidate adapter metadata must be EngineMetadata")
+
+ divergence: Optional[BookReplayDivergence] = None
+ compared_events = 0
+ final_state_hash: Optional[str] = None
+ last_candidate_observation: Optional[BookReplayObservation] = None
+ close_error: Optional[Exception] = None
+
+ try:
+ for index, event in enumerate(normalized_events, 1):
+ reference_observation = reference.apply(event, index)
+ try:
+ candidate_observation = candidate.apply(event, index)
+ if not isinstance(candidate_observation, BookReplayObservation):
+ raise BookReplayError("candidate apply() must return BookReplayObservation")
+ if candidate_observation.index != index:
+ raise BookReplayError(
+ f"candidate observation index {candidate_observation.index} "
+ f"does not match event {index}"
+ )
+ except Exception as exc:
+ divergence = _protocol_divergence(event, index, exc)
+ compared_events = index
+ break
+ last_candidate_observation = candidate_observation
+ compared_events = index
+ divergence = _compare_observations(
+ event,
+ index,
+ reference_observation,
+ candidate_observation,
+ )
+ if divergence is not None:
+ if divergence.category == "book_state":
+ divergence = _localize_state(
+ divergence,
+ reference,
+ candidate,
+ candidate_observation,
+ )
+ break
+
+ if divergence is None:
+ reference_state = reference.snapshot()
+ final_state_hash = reference_state.digest()
+ try:
+ candidate_state = candidate.snapshot()
+ if not isinstance(candidate_state, BookReplayState):
+ raise BookReplayError("candidate snapshot must return BookReplayState")
+ candidate_digest = candidate_state.digest()
+ if (
+ last_candidate_observation is not None
+ and candidate_digest != last_candidate_observation.state_hash
+ ):
+ hashed_event = normalized_events[-1]
+ divergence = BookReplayDivergence(
+ event_index=len(normalized_events),
+ category="protocol",
+ kind="invalid_state_hash",
+ path="$.observation.state_hash",
+ message="candidate final state_hash does not describe its snapshot",
+ event=hashed_event.to_dict(),
+ reference=candidate_digest,
+ candidate=last_candidate_observation.state_hash,
+ )
+ else:
+ difference = first_difference(
+ reference_state.to_dict(),
+ candidate_state.to_dict(),
+ "$.state",
+ )
+ if difference is not None:
+ state_event = normalized_events[-1] if normalized_events else None
+ divergence = BookReplayDivergence(
+ event_index=len(normalized_events),
+ category="book_state",
+ kind=difference["kind"],
+ path=difference["path"],
+ message="candidate final L3 state differs from the reference",
+ event=state_event.to_dict() if state_event else None,
+ reference=difference.get("reference"),
+ candidate=difference.get("candidate"),
+ )
+ except Exception as exc:
+ failed_event = normalized_events[-1] if normalized_events else None
+ divergence = BookReplayDivergence(
+ event_index=len(normalized_events),
+ category="protocol",
+ kind="adapter_error",
+ path="$.state",
+ message=f"candidate final snapshot failed: {exc}",
+ event=failed_event.to_dict() if failed_event else None,
+ reference="valid final snapshot",
+ candidate=None,
+ )
+ finally:
+ try:
+ candidate.close()
+ except Exception as exc:
+ close_error = exc
+ reference.close()
+
+ if close_error is not None:
+ if divergence is None:
+ closed_event = normalized_events[compared_events - 1] if compared_events else None
+ divergence = BookReplayDivergence(
+ event_index=compared_events,
+ category="protocol",
+ kind="adapter_close_error",
+ path="$",
+ message=f"candidate close failed: {close_error}",
+ event=closed_event.to_dict() if closed_event else None,
+ reference="clean adapter shutdown",
+ candidate=None,
+ )
+ else:
+ divergence = replace(divergence, close_error=f"candidate close failed: {close_error}")
+
+ return BookReplayReport(
+ config=config,
+ trace_name=trace_name,
+ trace_hash=trace_sha256(normalized_events),
+ event_count=len(normalized_events),
+ compared_events=compared_events,
+ reference_engine=reference.metadata,
+ candidate_engine=candidate_metadata,
+ conformant=divergence is None,
+ final_state_hash=final_state_hash,
+ divergence=divergence,
+ )
+
+
+__all__ = [
+ "BookReplayDivergence",
+ "BookReplayReport",
+ "run_book_replay",
+]
diff --git a/src/tracebook/book_replay/external.py b/src/tracebook/book_replay/external.py
new file mode 100644
index 0000000..f2a286b
--- /dev/null
+++ b/src/tracebook/book_replay/external.py
@@ -0,0 +1,324 @@
+"""External-process client for the isolated book-replay protocol."""
+
+from __future__ import annotations
+
+import json
+import math
+import queue
+import subprocess # nosec B404
+import threading
+from collections import deque
+from pathlib import Path
+from typing import Deque, Mapping, Optional, Sequence
+
+from ..conformance.model import ConformanceError
+from .model import (
+ PROTOCOL_NAME,
+ PROTOCOL_VERSION,
+ BookReplayConfig,
+ BookReplayError,
+ BookReplayEvent,
+ BookReplayObservation,
+ BookReplayState,
+ EngineMetadata,
+)
+
+_END_OF_STREAM = object()
+
+
+class BookReplayProtocolError(BookReplayError):
+ """Raised when an external book adapter violates or times out on protocol."""
+
+
+class ExternalBookReplayAdapter:
+ """Drive one external book mirror over newline-delimited JSON."""
+
+ def __init__(
+ self,
+ command: Sequence[str],
+ config: BookReplayConfig,
+ timeout_seconds: float = 5.0,
+ cwd: Optional[str | Path] = None,
+ ) -> None:
+ if isinstance(command, (str, bytes)) or not command:
+ raise BookReplayError("candidate command must be a non-empty argument list")
+ normalized_command = []
+ for argument in command:
+ if not isinstance(argument, str) or not argument:
+ raise BookReplayError("candidate command arguments must be non-empty strings")
+ normalized_command.append(argument)
+ try:
+ normalized_timeout = float(timeout_seconds)
+ except (TypeError, ValueError, OverflowError) as exc:
+ raise BookReplayError("timeout_seconds must be a positive finite number") from exc
+ if (
+ isinstance(timeout_seconds, bool)
+ or not isinstance(timeout_seconds, (int, float))
+ or not math.isfinite(normalized_timeout)
+ or normalized_timeout <= 0
+ ):
+ raise BookReplayError("timeout_seconds must be a positive finite number")
+
+ self.command = tuple(normalized_command)
+ self.timeout_seconds = normalized_timeout
+ self._closed = False
+ self._broken = False
+ self._last_index = 0
+ self._stdout_queue: queue.Queue[object] = queue.Queue(maxsize=64)
+ self._stderr_lines: Deque[str] = deque(maxlen=64)
+ self._stderr_lock = threading.Lock()
+ try:
+ self._process = subprocess.Popen( # nosec B603
+ self.command,
+ cwd=str(cwd) if cwd is not None else None,
+ stdin=subprocess.PIPE,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ text=True,
+ encoding="utf-8",
+ bufsize=1,
+ )
+ except OSError as exc:
+ raise BookReplayProtocolError(
+ f"unable to start candidate command {self.command!r}: {exc}"
+ ) from exc
+ stdout = self._process.stdout
+ stderr = self._process.stderr
+ if stdout is None or stderr is None:
+ self._process.kill()
+ self._process.wait()
+ raise BookReplayProtocolError("candidate process pipes were not created")
+ self._stdout_thread = threading.Thread(
+ target=self._read_stdout,
+ args=(stdout,),
+ name="tracebook-book-replay-stdout",
+ daemon=True,
+ )
+ self._stderr_thread = threading.Thread(
+ target=self._read_stderr,
+ args=(stderr,),
+ name="tracebook-book-replay-stderr",
+ daemon=True,
+ )
+ self._stdout_thread.start()
+ self._stderr_thread.start()
+
+ try:
+ self._send(
+ {
+ "type": "hello",
+ "protocol": PROTOCOL_NAME,
+ "protocol_version": PROTOCOL_VERSION,
+ "config": config.to_dict(),
+ }
+ )
+ ready = self._receive("ready")
+ if ready.get("protocol") != PROTOCOL_NAME:
+ raise BookReplayProtocolError("ready message reported the wrong protocol")
+ if ready.get("protocol_version") != PROTOCOL_VERSION:
+ raise BookReplayProtocolError("ready message reported the wrong protocol version")
+ try:
+ self.metadata = EngineMetadata.from_dict(ready.get("engine", {}))
+ except ConformanceError as exc:
+ raise BookReplayProtocolError(f"invalid ready engine metadata: {exc}") from exc
+ except Exception:
+ self._broken = True
+ self._shutdown()
+ raise
+
+ def _read_stdout(self, stream) -> None:
+ try:
+ for line in stream:
+ self._stdout_queue.put(line)
+ finally:
+ self._stdout_queue.put(_END_OF_STREAM)
+
+ def _read_stderr(self, stream) -> None:
+ for line in stream:
+ with self._stderr_lock:
+ self._stderr_lines.append(line[-4096:])
+
+ def _stderr_tail(self) -> str:
+ with self._stderr_lock:
+ rendered = "".join(self._stderr_lines).strip()
+ return rendered[-4000:] if rendered else ""
+
+ def _send(self, payload: Mapping[str, object]) -> None:
+ if self._closed or self._process.poll() is not None:
+ raise BookReplayProtocolError(
+ f"candidate process is not running{self._process_context()}"
+ )
+ process_input = self._process.stdin
+ if process_input is None:
+ self._broken = True
+ raise BookReplayProtocolError("candidate process input pipe is unavailable")
+ try:
+ process_input.write(json.dumps(payload, separators=(",", ":"), allow_nan=False) + "\n")
+ process_input.flush()
+ except (BrokenPipeError, OSError) as exc:
+ self._broken = True
+ raise BookReplayProtocolError(
+ f"candidate process closed its input{self._process_context()}"
+ ) from exc
+
+ def _receive(self, expected_type: str) -> dict:
+ try:
+ item = self._stdout_queue.get(timeout=self.timeout_seconds)
+ except queue.Empty as exc:
+ self._broken = True
+ raise BookReplayProtocolError(
+ f"candidate timed out waiting for {expected_type!r} after "
+ f"{self.timeout_seconds:g}s{self._process_context()}"
+ ) from exc
+ if item is _END_OF_STREAM:
+ self._broken = True
+ raise BookReplayProtocolError(
+ f"candidate exited before {expected_type!r}{self._process_context()}"
+ )
+ if not isinstance(item, str):
+ self._broken = True
+ raise BookReplayProtocolError("candidate produced an invalid protocol frame")
+ try:
+ message = json.loads(item)
+ except json.JSONDecodeError as exc:
+ self._broken = True
+ raise BookReplayProtocolError(
+ f"candidate stdout was not valid JSON: {item.strip()!r}"
+ ) from exc
+ if not isinstance(message, dict):
+ self._broken = True
+ raise BookReplayProtocolError("candidate protocol frame must be a JSON object")
+ if message.get("type") == "error":
+ self._broken = True
+ raise BookReplayProtocolError(
+ f"candidate reported {message.get('code', 'ERROR')}: "
+ f"{message.get('message', '')}"
+ )
+ if message.get("type") != expected_type:
+ self._broken = True
+ raise BookReplayProtocolError(
+ f"expected candidate message {expected_type!r}, "
+ f"received {message.get('type')!r}"
+ )
+ return message
+
+ def _process_context(self) -> str:
+ return_code = self._process.poll()
+ details = f" (exit code {return_code})" if return_code is not None else ""
+ stderr = self._stderr_tail()
+ return details + (f"; stderr: {stderr}" if stderr else "")
+
+ def apply(self, event: BookReplayEvent, index: int) -> BookReplayObservation:
+ self._send({"type": "event", "index": index, "event": event.to_dict()})
+ observation = BookReplayObservation.from_dict(self._receive("observation"))
+ if observation.index != index:
+ self._broken = True
+ raise BookReplayProtocolError(
+ f"candidate observation index {observation.index} does not match {index}"
+ )
+ self._last_index = index
+ return observation
+
+ def snapshot(self) -> BookReplayState:
+ self._send({"type": "snapshot", "index": self._last_index})
+ message = self._receive("snapshot")
+ if message.get("index") != self._last_index:
+ self._broken = True
+ raise BookReplayProtocolError("candidate snapshot reported the wrong event index")
+ return BookReplayState.from_dict(message.get("state", {}))
+
+ def close(self) -> None:
+ if self._closed:
+ return
+ close_error: Optional[BookReplayProtocolError] = None
+ was_broken = self._broken
+ try:
+ if not self._broken and self._process.poll() is None:
+ self._send({"type": "finish", "event_count": self._last_index})
+ complete = self._receive("complete")
+ if complete.get("event_count") != self._last_index:
+ raise BookReplayProtocolError(
+ "candidate complete message reported the wrong event_count"
+ )
+ elif not self._broken:
+ raise BookReplayProtocolError("candidate exited before finish")
+ except (BookReplayProtocolError, OSError) as exc:
+ self._broken = True
+ close_error = (
+ exc
+ if isinstance(exc, BookReplayProtocolError)
+ else BookReplayProtocolError(f"candidate shutdown failed: {exc}")
+ )
+ finally:
+ self._closed = True
+ shutdown_timed_out = self._shutdown(
+ timeout_seconds=self.timeout_seconds if not self._broken else 0.5
+ )
+ if not was_broken and close_error is None:
+ if shutdown_timed_out:
+ close_error = BookReplayProtocolError(
+ f"candidate timed out exiting after complete ({self.timeout_seconds:g}s)"
+ )
+ elif self._process.returncode != 0:
+ close_error = BookReplayProtocolError(
+ f"candidate exited with code {self._process.returncode} after complete"
+ )
+ if close_error is not None:
+ raise close_error
+
+ def _shutdown(self, *, timeout_seconds: float = 0.5) -> bool:
+ timed_out = False
+ if self._process.poll() is None:
+ try:
+ self._process.wait(timeout=timeout_seconds)
+ except subprocess.TimeoutExpired:
+ timed_out = True
+ self._process.terminate()
+ try:
+ self._process.wait(timeout=0.5)
+ except subprocess.TimeoutExpired:
+ self._process.kill()
+ self._process.wait(timeout=0.5)
+ if self._process.stdin is not None:
+ try:
+ self._process.stdin.close()
+ except OSError:
+ pass
+ self._stdout_thread.join(timeout=0.2)
+ self._stderr_thread.join(timeout=0.2)
+ for stream in (self._process.stdout, self._process.stderr):
+ if stream is not None:
+ try:
+ stream.close()
+ except OSError:
+ pass
+ return timed_out
+
+
+class ExternalBookReplayAdapterFactory:
+ """Start a fresh external book adapter for each comparison."""
+
+ def __init__(
+ self,
+ command: Sequence[str],
+ timeout_seconds: float = 5.0,
+ cwd: Optional[str | Path] = None,
+ ) -> None:
+ self.command = tuple(command)
+ self.timeout_seconds = timeout_seconds
+ self.cwd = cwd
+
+ def __call__(self, config: BookReplayConfig) -> ExternalBookReplayAdapter:
+ return ExternalBookReplayAdapter(
+ self.command,
+ config,
+ timeout_seconds=self.timeout_seconds,
+ cwd=self.cwd,
+ )
+
+
+__all__ = [
+ "BookReplayProtocolError",
+ "ExternalBookReplayAdapter",
+ "ExternalBookReplayAdapterFactory",
+]
diff --git a/src/tracebook/book_replay/fixtures/__init__.py b/src/tracebook/book_replay/fixtures/__init__.py
new file mode 100644
index 0000000..f752d3c
--- /dev/null
+++ b/src/tracebook/book_replay/fixtures/__init__.py
@@ -0,0 +1 @@
+"""Bundled deterministic traces for the book-replay profile."""
diff --git a/src/tracebook/book_replay/fixtures/l3-book-replay-v1.jsonl b/src/tracebook/book_replay/fixtures/l3-book-replay-v1.jsonl
new file mode 100644
index 0000000..0d99d2a
--- /dev/null
+++ b/src/tracebook/book_replay/fixtures/l3-book-replay-v1.jsonl
@@ -0,0 +1,17 @@
+{"op":"add","symbol":"TEST","order_id":1,"side":"BUY","price":"100","quantity":"2"}
+{"op":"add","symbol":"TEST","order_id":2,"side":"BUY","price":"100","quantity":"3"}
+{"op":"update","symbol":"TEST","order_id":1,"side":"BUY","price":"100","quantity":"4"}
+{"op":"add","symbol":"TEST","order_id":3,"side":"BUY","price":"99","quantity":"5"}
+{"op":"add","symbol":"TEST","order_id":10,"side":"SELL","price":"101","quantity":"1.5"}
+{"op":"add","symbol":"TEST","order_id":11,"side":"SELL","price":"101","quantity":"2.5"}
+{"op":"add","symbol":"TEST","order_id":12,"side":"SELL","price":"102","quantity":"4"}
+{"op":"probe","symbol":"TEST","side":"BUY","price":"102","quantity":"5"}
+{"op":"probe","symbol":"TEST","side":"SELL","price":"99","quantity":"8"}
+{"op":"update","symbol":"TEST","order_id":2,"side":"BUY","price":"99","quantity":"3"}
+{"op":"probe","symbol":"TEST","side":"SELL","price":"99","quantity":"12"}
+{"op":"delete","symbol":"TEST","order_id":1}
+{"op":"delete","symbol":"TEST","order_id":10}
+{"op":"probe","symbol":"TEST","side":"BUY","price":"101","quantity":"3"}
+{"op":"add","symbol":"ALT","order_id":1,"side":"SELL","price":"50","quantity":"7"}
+{"op":"probe","symbol":"ALT","side":"BUY","price":"50","quantity":"2"}
+{"op":"delete","symbol":"TEST","order_id":999}
diff --git a/src/tracebook/book_replay/minimize.py b/src/tracebook/book_replay/minimize.py
new file mode 100644
index 0000000..377c19e
--- /dev/null
+++ b/src/tracebook/book_replay/minimize.py
@@ -0,0 +1,172 @@
+"""Deterministically reduce an L3 book-replay divergence."""
+
+from __future__ import annotations
+
+import math
+from dataclasses import dataclass, replace
+from numbers import Integral
+from typing import Iterable, Optional, Tuple
+
+from .compare import BookReplayReport, run_book_replay
+from .model import (
+ ARTIFACT_SCHEMA_VERSION,
+ BookReplayConfig,
+ BookReplayError,
+ BookReplayEvent,
+ EngineMetadata,
+ trace_sha256,
+)
+from .protocol import BookReplayAdapterFactory
+
+
+@dataclass(frozen=True)
+class BookReplayMinimizationResult:
+ """A reduced trace and the report proving it still diverges."""
+
+ events: Tuple[BookReplayEvent, ...]
+ original_event_count: int
+ runs: int
+ target_category: str
+ one_minimal: bool
+ budget_exhausted: bool
+ report: BookReplayReport
+
+ def to_dict(self) -> dict:
+ minimized_count = len(self.events)
+ reduction_percent = (
+ 100.0 * (self.original_event_count - minimized_count) / self.original_event_count
+ if self.original_event_count
+ else 0.0
+ )
+ return {
+ "schema_version": ARTIFACT_SCHEMA_VERSION,
+ "artifact_type": "tracebook.book-replay.minimization",
+ "original_event_count": self.original_event_count,
+ "minimized_event_count": minimized_count,
+ "reduction_percent": reduction_percent,
+ "runs": self.runs,
+ "target_category": self.target_category,
+ "one_minimal": self.one_minimal,
+ "budget_exhausted": self.budget_exhausted,
+ "minimized_trace_sha256": trace_sha256(self.events),
+ "book_replay_report": self.report.to_dict(),
+ }
+
+
+def _reject_operational_failure(
+ report: BookReplayReport,
+ target_category: str,
+ run_number: int,
+) -> None:
+ divergence = report.divergence
+ if target_category != "protocol" and divergence is not None and report.operational_failure:
+ kind = "adapter_close_error" if divergence.close_error is not None else divergence.kind
+ message = divergence.close_error or divergence.message
+ raise BookReplayError(
+ "candidate protocol failure during semantic minimization "
+ f"run {run_number}: {kind}: {message}"
+ )
+
+
+def minimize_book_replay_failure(
+ events: Iterable[BookReplayEvent],
+ candidate_factory: BookReplayAdapterFactory,
+ config: Optional[BookReplayConfig] = None,
+ max_runs: int = 100,
+ trace_name: Optional[str] = None,
+ expected_candidate_engine: Optional[EngineMetadata] = None,
+) -> BookReplayMinimizationResult:
+ """Delta-debug a failure while preserving its first-divergence category."""
+ if isinstance(max_runs, bool) or not isinstance(max_runs, Integral) or max_runs <= 0:
+ raise BookReplayError("max_runs must be a positive integer")
+ max_runs = int(max_runs)
+ config = config or BookReplayConfig()
+ original = tuple(events)
+ initial = run_book_replay(original, candidate_factory, config, trace_name=trace_name)
+ runs = 1
+ if initial.conformant or initial.divergence is None:
+ raise BookReplayError("trace is conformant and cannot be minimized")
+ candidate_engine = expected_candidate_engine or initial.candidate_engine
+ if initial.candidate_engine != candidate_engine:
+ raise BookReplayError("candidate engine metadata changed during minimization")
+
+ target = initial.divergence.category
+ _reject_operational_failure(initial, target, runs)
+ prefix_length = initial.divergence.event_index
+ current = original[:prefix_length] if prefix_length > 0 else original
+ current_report = (
+ replace(
+ initial,
+ trace_name=None,
+ trace_hash=trace_sha256(current),
+ event_count=len(current),
+ )
+ if current != original
+ else initial
+ )
+ granularity = 2
+ one_minimal = len(current) == 0
+
+ while len(current) >= 2 and runs < max_runs:
+ chunk_size = int(math.ceil(len(current) / granularity))
+ reduced = False
+ completed_round = True
+ for start in range(0, len(current), chunk_size):
+ if runs >= max_runs:
+ completed_round = False
+ break
+ trial = current[:start] + current[start + chunk_size :]
+ if not trial:
+ continue
+ report = run_book_replay(trial, candidate_factory, config)
+ runs += 1
+ if report.candidate_engine != candidate_engine:
+ raise BookReplayError("candidate engine metadata changed during minimization")
+ _reject_operational_failure(report, target, runs)
+ if (
+ not report.conformant
+ and report.divergence is not None
+ and report.divergence.category == target
+ ):
+ current = trial
+ current_report = report
+ granularity = max(2, granularity - 1)
+ reduced = True
+ break
+ if reduced:
+ one_minimal = len(current) == 0
+ continue
+ if not completed_round:
+ break
+ if granularity >= len(current):
+ one_minimal = True
+ break
+ granularity = min(len(current), granularity * 2)
+
+ if len(current) == 1 and not one_minimal and runs < max_runs:
+ empty_report = run_book_replay((), candidate_factory, config)
+ runs += 1
+ if empty_report.candidate_engine != candidate_engine:
+ raise BookReplayError("candidate engine metadata changed during minimization")
+ _reject_operational_failure(empty_report, target, runs)
+ if (
+ not empty_report.conformant
+ and empty_report.divergence is not None
+ and empty_report.divergence.category == target
+ ):
+ current = ()
+ current_report = empty_report
+ one_minimal = True
+
+ return BookReplayMinimizationResult(
+ events=current,
+ original_event_count=len(original),
+ runs=runs,
+ target_category=target,
+ one_minimal=one_minimal,
+ budget_exhausted=runs >= max_runs and not one_minimal,
+ report=current_report,
+ )
+
+
+__all__ = ["BookReplayMinimizationResult", "minimize_book_replay_failure"]
diff --git a/src/tracebook/book_replay/model.py b/src/tracebook/book_replay/model.py
new file mode 100644
index 0000000..57b42c5
--- /dev/null
+++ b/src/tracebook/book_replay/model.py
@@ -0,0 +1,395 @@
+"""Versioned contracts for deterministic L3 order-book replay checks."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from dataclasses import dataclass
+from decimal import Decimal
+from numbers import Integral
+from typing import Any, Iterable, Mapping, Optional, Tuple, cast
+
+from ..conformance.model import EngineMetadata, Outcome, canonical_decimal
+
+PROTOCOL_NAME = "tracebook.book-replay"
+PROTOCOL_VERSION = 1
+ARTIFACT_SCHEMA_VERSION = 1
+PROFILE_NAME = "l3-book-replay-v1"
+_HASH_LENGTH = 64
+
+
+class BookReplayError(ValueError):
+ """Raised when a book-replay trace, contract, or adapter is invalid."""
+
+
+def _positive_int(value: Any, field_name: str) -> int:
+ if isinstance(value, bool) or not isinstance(value, Integral) or value <= 0:
+ raise BookReplayError(f"{field_name} must be a positive integer")
+ return int(value)
+
+
+def _nonnegative_int(value: Any, field_name: str) -> int:
+ if isinstance(value, bool) or not isinstance(value, Integral) or value < 0:
+ raise BookReplayError(f"{field_name} must be a non-negative integer")
+ return int(value)
+
+
+def _positive_decimal(value: Any, field_name: str) -> str:
+ try:
+ normalized = canonical_decimal(value)
+ except ValueError as exc:
+ raise BookReplayError(str(exc)) from exc
+ if Decimal(normalized) <= 0:
+ raise BookReplayError(f"{field_name} must be positive")
+ return normalized
+
+
+def _symbol(value: Any) -> str:
+ if not isinstance(value, str) or not value.strip():
+ raise BookReplayError("symbol must be a non-empty string")
+ return value.strip()
+
+
+def _side(value: Any) -> str:
+ if not isinstance(value, str):
+ raise BookReplayError("side must be BUY or SELL")
+ normalized = value.strip().upper()
+ if normalized not in {"BUY", "SELL"}:
+ raise BookReplayError("side must be BUY or SELL")
+ return normalized
+
+
+@dataclass(frozen=True)
+class BookReplayConfig:
+ """The immutable semantic profile selected for one comparison."""
+
+ profile: str = PROFILE_NAME
+
+ def __post_init__(self) -> None:
+ if self.profile != PROFILE_NAME:
+ raise BookReplayError(f"profile must be {PROFILE_NAME!r}")
+
+ def to_dict(self) -> dict:
+ return {"profile": self.profile}
+
+ @classmethod
+ def from_dict(cls, data: Mapping[str, Any]) -> "BookReplayConfig":
+ if not isinstance(data, Mapping):
+ raise BookReplayError("config must be an object")
+ return cls(profile=data.get("profile", PROFILE_NAME))
+
+
+@dataclass(frozen=True)
+class BookReplayEvent:
+ """One L3 delta or non-mutating simulated-fill probe."""
+
+ op: str
+ symbol: str
+ order_id: Optional[int] = None
+ side: Optional[str] = None
+ price: Optional[str] = None
+ quantity: Optional[str] = None
+
+ def __post_init__(self) -> None:
+ if not isinstance(self.op, str):
+ raise BookReplayError("op must be add, update, delete, or probe")
+ op = self.op.strip().lower()
+ if op not in {"add", "update", "delete", "probe"}:
+ raise BookReplayError("op must be add, update, delete, or probe")
+ object.__setattr__(self, "op", op)
+ object.__setattr__(self, "symbol", _symbol(self.symbol))
+
+ if op in {"add", "update", "delete"}:
+ object.__setattr__(self, "order_id", _positive_int(self.order_id, "order_id"))
+ elif self.order_id is not None:
+ raise BookReplayError("probe events cannot contain order_id")
+
+ if op in {"add", "update", "probe"}:
+ object.__setattr__(self, "side", _side(self.side))
+ object.__setattr__(self, "price", _positive_decimal(self.price, "price"))
+ object.__setattr__(
+ self,
+ "quantity",
+ _positive_decimal(self.quantity, "quantity"),
+ )
+ elif any(value is not None for value in (self.side, self.price, self.quantity)):
+ raise BookReplayError("delete events contain only op, symbol, and order_id")
+
+ def to_dict(self) -> dict:
+ payload: dict[str, Any] = {"op": self.op, "symbol": self.symbol}
+ if self.order_id is not None:
+ payload["order_id"] = self.order_id
+ if self.side is not None:
+ payload["side"] = self.side
+ if self.price is not None:
+ payload["price"] = self.price
+ if self.quantity is not None:
+ payload["quantity"] = self.quantity
+ return payload
+
+ @classmethod
+ def from_mapping(cls, data: Mapping[str, Any]) -> "BookReplayEvent":
+ if not isinstance(data, Mapping):
+ raise BookReplayError("book-replay event must be an object")
+ op = data.get("op", "")
+ normalized_op = op.strip().lower() if isinstance(op, str) else op
+ allowed = {
+ "add": {"op", "symbol", "order_id", "side", "price", "quantity"},
+ "update": {"op", "symbol", "order_id", "side", "price", "quantity"},
+ "delete": {"op", "symbol", "order_id"},
+ "probe": {"op", "symbol", "side", "price", "quantity"},
+ }.get(normalized_op)
+ if allowed is not None:
+ unknown = set(data) - allowed
+ if unknown:
+ raise BookReplayError(
+ f"{normalized_op} event contains unsupported fields: {sorted(unknown)!r}"
+ )
+ return cls(
+ op=cast(str, op),
+ symbol=cast(str, data.get("symbol", "")),
+ order_id=cast(Optional[int], data.get("order_id")),
+ side=cast(Optional[str], data.get("side")),
+ price=cast(Optional[str], data.get("price")),
+ quantity=cast(Optional[str], data.get("quantity")),
+ )
+
+
+@dataclass(frozen=True)
+class SimulatedFill:
+ """One price/quantity segment returned by a non-mutating probe."""
+
+ price: str
+ quantity: str
+
+ def __post_init__(self) -> None:
+ object.__setattr__(self, "price", _positive_decimal(self.price, "fill price"))
+ object.__setattr__(self, "quantity", _positive_decimal(self.quantity, "fill quantity"))
+
+ def to_dict(self) -> dict:
+ return {"price": self.price, "quantity": self.quantity}
+
+ @classmethod
+ def from_dict(cls, data: Mapping[str, Any]) -> "SimulatedFill":
+ if not isinstance(data, Mapping):
+ raise BookReplayError("simulated fill must be an object")
+ return cls(price=cast(str, data.get("price")), quantity=cast(str, data.get("quantity")))
+
+
+@dataclass(frozen=True)
+class RestingBookOrder:
+ """One mirrored L3 order in queue position."""
+
+ order_id: int
+ price: str
+ quantity: str
+
+ def __post_init__(self) -> None:
+ object.__setattr__(self, "order_id", _positive_int(self.order_id, "order_id"))
+ object.__setattr__(self, "price", _positive_decimal(self.price, "order price"))
+ object.__setattr__(self, "quantity", _positive_decimal(self.quantity, "order quantity"))
+
+ def to_dict(self) -> dict:
+ return {"order_id": self.order_id, "price": self.price, "quantity": self.quantity}
+
+ @classmethod
+ def from_dict(cls, data: Mapping[str, Any]) -> "RestingBookOrder":
+ if not isinstance(data, Mapping):
+ raise BookReplayError("resting order must be an object")
+ return cls(
+ order_id=cast(int, data.get("order_id")),
+ price=cast(str, data.get("price")),
+ quantity=cast(str, data.get("quantity")),
+ )
+
+
+@dataclass(frozen=True)
+class BookReplaySnapshot:
+ """Both price-time queues for one mirrored symbol."""
+
+ symbol: str
+ bids: Tuple[RestingBookOrder, ...] = ()
+ asks: Tuple[RestingBookOrder, ...] = ()
+
+ def __post_init__(self) -> None:
+ object.__setattr__(self, "symbol", _symbol(self.symbol))
+ for name in ("bids", "asks"):
+ orders = getattr(self, name)
+ if not isinstance(orders, tuple) or any(
+ not isinstance(order, RestingBookOrder) for order in orders
+ ):
+ raise BookReplayError(f"{name} must be a tuple of RestingBookOrder values")
+ order_ids = [order.order_id for order in self.bids + self.asks]
+ if len(order_ids) != len(set(order_ids)):
+ raise BookReplayError(f"book {self.symbol!r} contains duplicate order ids")
+
+ def to_dict(self) -> dict:
+ return {
+ "symbol": self.symbol,
+ "bids": [order.to_dict() for order in self.bids],
+ "asks": [order.to_dict() for order in self.asks],
+ }
+
+ @classmethod
+ def from_dict(cls, data: Mapping[str, Any]) -> "BookReplaySnapshot":
+ if not isinstance(data, Mapping):
+ raise BookReplayError("book snapshot must be an object")
+ bids = data.get("bids", [])
+ asks = data.get("asks", [])
+ if not isinstance(bids, list) or not isinstance(asks, list):
+ raise BookReplayError("book bids and asks must be arrays")
+ return cls(
+ symbol=cast(str, data.get("symbol", "")),
+ bids=tuple(RestingBookOrder.from_dict(item) for item in bids),
+ asks=tuple(RestingBookOrder.from_dict(item) for item in asks),
+ )
+
+
+@dataclass(frozen=True)
+class BookReplayState:
+ """Canonical mirrored state, including L3 queue order."""
+
+ books: Tuple[BookReplaySnapshot, ...] = ()
+
+ def __post_init__(self) -> None:
+ if not isinstance(self.books, tuple) or any(
+ not isinstance(book, BookReplaySnapshot) for book in self.books
+ ):
+ raise BookReplayError("books must be a tuple of BookReplaySnapshot values")
+ symbols = [book.symbol for book in self.books]
+ if symbols != sorted(symbols) or len(symbols) != len(set(symbols)):
+ raise BookReplayError("book snapshots must have unique symbols in sorted order")
+
+ @property
+ def order_count(self) -> int:
+ return sum(len(book.bids) + len(book.asks) for book in self.books)
+
+ def to_dict(self) -> dict:
+ return {"books": [book.to_dict() for book in self.books]}
+
+ @classmethod
+ def from_dict(cls, data: Mapping[str, Any]) -> "BookReplayState":
+ if not isinstance(data, Mapping):
+ raise BookReplayError("state must be an object")
+ books = data.get("books", [])
+ if not isinstance(books, list):
+ raise BookReplayError("state books must be an array")
+ return cls(tuple(BookReplaySnapshot.from_dict(item) for item in books))
+
+ def digest(self) -> str:
+ return hashlib.sha256(_canonical_json(self.to_dict())).hexdigest()
+
+
+@dataclass(frozen=True)
+class BookReplayObservation:
+ """Comparable result emitted after one delta or probe."""
+
+ index: int
+ outcome: Outcome
+ fills: Tuple[SimulatedFill, ...]
+ state_hash: str
+ order_count: int
+
+ def __post_init__(self) -> None:
+ object.__setattr__(self, "index", _positive_int(self.index, "observation index"))
+ if not isinstance(self.outcome, Outcome):
+ raise BookReplayError("observation outcome must be an Outcome")
+ if not isinstance(self.fills, tuple) or any(
+ not isinstance(fill, SimulatedFill) for fill in self.fills
+ ):
+ raise BookReplayError("observation fills must be a tuple of SimulatedFill values")
+ if (
+ not isinstance(self.state_hash, str)
+ or len(self.state_hash) != _HASH_LENGTH
+ or any(character not in "0123456789abcdef" for character in self.state_hash)
+ ):
+ raise BookReplayError("state_hash must be a lowercase SHA-256 hex digest")
+ object.__setattr__(
+ self,
+ "order_count",
+ _nonnegative_int(self.order_count, "order_count"),
+ )
+
+ def to_dict(self, include_type: bool = False) -> dict:
+ payload = {
+ "index": self.index,
+ "outcome": self.outcome.to_dict(),
+ "fills": [fill.to_dict() for fill in self.fills],
+ "state_hash": self.state_hash,
+ "order_count": self.order_count,
+ }
+ if include_type:
+ payload = {"type": "observation", **payload}
+ return payload
+
+ @classmethod
+ def from_dict(cls, data: Mapping[str, Any]) -> "BookReplayObservation":
+ if not isinstance(data, Mapping):
+ raise BookReplayError("observation must be an object")
+ fills = data.get("fills", [])
+ if not isinstance(fills, list):
+ raise BookReplayError("observation fills must be an array")
+ return cls(
+ index=cast(int, data.get("index")),
+ outcome=Outcome.from_dict(data.get("outcome", {})),
+ fills=tuple(SimulatedFill.from_dict(item) for item in fills),
+ state_hash=cast(str, data.get("state_hash")),
+ order_count=cast(int, data.get("order_count")),
+ )
+
+
+def load_book_replay_events(path: str) -> list[BookReplayEvent]:
+ """Load a newline-delimited JSON book-replay trace."""
+ events = []
+ with open(path, "r", encoding="utf-8") as stream:
+ for line_number, line in enumerate(stream, 1):
+ if not line.strip():
+ continue
+ try:
+ payload = json.loads(line)
+ except json.JSONDecodeError as exc:
+ raise BookReplayError(f"invalid JSON on line {line_number}: {exc.msg}") from exc
+ try:
+ events.append(BookReplayEvent.from_mapping(payload))
+ except (TypeError, ValueError) as exc:
+ raise BookReplayError(f"invalid event on line {line_number}: {exc}") from exc
+ return events
+
+
+def trace_sha256(events: Iterable[BookReplayEvent]) -> str:
+ """Hash normalized events independently of input whitespace."""
+ digest = hashlib.sha256()
+ for event in events:
+ digest.update(_canonical_json(event.to_dict()))
+ digest.update(b"\n")
+ return digest.hexdigest()
+
+
+def _canonical_json(payload: Any) -> bytes:
+ return json.dumps(
+ payload,
+ sort_keys=True,
+ separators=(",", ":"),
+ ensure_ascii=False,
+ allow_nan=False,
+ ).encode("utf-8")
+
+
+__all__ = [
+ "ARTIFACT_SCHEMA_VERSION",
+ "PROFILE_NAME",
+ "PROTOCOL_NAME",
+ "PROTOCOL_VERSION",
+ "BookReplayConfig",
+ "BookReplayError",
+ "BookReplayEvent",
+ "BookReplayObservation",
+ "BookReplaySnapshot",
+ "BookReplayState",
+ "EngineMetadata",
+ "Outcome",
+ "RestingBookOrder",
+ "SimulatedFill",
+ "load_book_replay_events",
+ "trace_sha256",
+]
diff --git a/src/tracebook/book_replay/protocol.py b/src/tracebook/book_replay/protocol.py
new file mode 100644
index 0000000..efba6b7
--- /dev/null
+++ b/src/tracebook/book_replay/protocol.py
@@ -0,0 +1,161 @@
+"""NDJSON protocol helpers for external L3 book-replay adapters."""
+
+from __future__ import annotations
+
+import json
+import sys
+from typing import Callable, Optional, Protocol, TextIO
+
+from .model import (
+ PROTOCOL_NAME,
+ PROTOCOL_VERSION,
+ BookReplayConfig,
+ BookReplayEvent,
+ BookReplayObservation,
+ BookReplayState,
+ EngineMetadata,
+)
+
+
+class BookReplayAdapter(Protocol):
+ """Minimal interface shared by in-process and external book mirrors."""
+
+ metadata: EngineMetadata
+
+ def apply(self, event: BookReplayEvent, index: int) -> BookReplayObservation: ...
+
+ def snapshot(self) -> BookReplayState: ...
+
+ def close(self) -> None: ...
+
+
+BookReplayAdapterFactory = Callable[[BookReplayConfig], BookReplayAdapter]
+
+
+class _RequestError(ValueError):
+ """Invalid client input, distinct from exceptions inside an adapter."""
+
+
+def _message(line: str) -> dict:
+ try:
+ message = json.loads(line)
+ except json.JSONDecodeError as exc:
+ raise _RequestError(str(exc)) from exc
+ if not isinstance(message, dict):
+ raise _RequestError("protocol message must be an object")
+ return message
+
+
+def _index(value: object, name: str) -> int:
+ if isinstance(value, bool) or not isinstance(value, int) or value < 0:
+ raise _RequestError(f"{name} must be a non-negative integer")
+ return value
+
+
+def _write_message(stream: TextIO, payload: dict) -> None:
+ stream.write(json.dumps(payload, separators=(",", ":"), allow_nan=False) + "\n")
+ stream.flush()
+
+
+def serve_book_replay_stdio(
+ adapter_factory: BookReplayAdapterFactory,
+ input_stream: Optional[TextIO] = None,
+ output_stream: Optional[TextIO] = None,
+) -> int:
+ """Serve one adapter over the isolated book-replay protocol."""
+ source = input_stream or sys.stdin
+ sink = output_stream or sys.stdout
+ adapter: Optional[BookReplayAdapter] = None
+ last_index = 0
+ try:
+ first_line = source.readline()
+ if not first_line:
+ raise _RequestError("expected hello message")
+ hello = _message(first_line)
+ if hello.get("type") != "hello":
+ raise _RequestError("first message must be hello")
+ if hello.get("protocol") != PROTOCOL_NAME:
+ raise _RequestError(f"protocol must be {PROTOCOL_NAME!r}")
+ if _index(hello.get("protocol_version"), "protocol_version") != PROTOCOL_VERSION:
+ raise _RequestError(f"protocol_version must be {PROTOCOL_VERSION}")
+ try:
+ config = BookReplayConfig.from_dict(hello.get("config", {}))
+ except (TypeError, ValueError) as exc:
+ raise _RequestError(str(exc)) from exc
+ adapter = adapter_factory(config)
+ _write_message(
+ sink,
+ {
+ "type": "ready",
+ "protocol": PROTOCOL_NAME,
+ "protocol_version": PROTOCOL_VERSION,
+ "engine": adapter.metadata.to_dict(),
+ },
+ )
+
+ for line in source:
+ if not line.strip():
+ continue
+ message = _message(line)
+ message_type = message.get("type")
+ if message_type == "event":
+ index = _index(message.get("index"), "event index")
+ if index != last_index + 1:
+ raise _RequestError("event indexes must be contiguous and start at 1")
+ try:
+ event = BookReplayEvent.from_mapping(message.get("event", {}))
+ except (TypeError, ValueError) as exc:
+ raise _RequestError(str(exc)) from exc
+ observation = adapter.apply(event, index)
+ if observation.index != index:
+ raise ValueError("adapter returned the wrong observation index")
+ last_index = index
+ _write_message(sink, observation.to_dict(include_type=True))
+ elif message_type == "snapshot":
+ if _index(message.get("index"), "snapshot index") != last_index:
+ raise _RequestError("snapshot index does not match the last event")
+ _write_message(
+ sink,
+ {
+ "type": "snapshot",
+ "index": last_index,
+ "state": adapter.snapshot().to_dict(),
+ },
+ )
+ elif message_type == "finish":
+ if _index(message.get("event_count"), "finish event_count") != last_index:
+ raise _RequestError("finish event_count does not match the last event")
+ # Completion includes shutdown. Detach first so a failing close
+ # produces one error and is never retried by the cleanup path.
+ closing, adapter = adapter, None
+ closing.close()
+ _write_message(sink, {"type": "complete", "event_count": last_index})
+ return 0
+ else:
+ raise _RequestError(f"unsupported protocol message type: {message_type!r}")
+ raise _RequestError("protocol ended before finish")
+ except _RequestError as exc:
+ try:
+ _write_message(sink, {"type": "error", "code": "PROTOCOL_ERROR", "message": str(exc)})
+ except BrokenPipeError:
+ pass
+ return 2
+ except Exception as exc:
+ try:
+ _write_message(sink, {"type": "error", "code": "ADAPTER_ERROR", "message": str(exc)})
+ except BrokenPipeError:
+ pass
+ return 2
+ finally:
+ if adapter is not None:
+ try:
+ adapter.close()
+ except Exception as exc:
+ print(f"Adapter close error: {exc}", file=sys.stderr)
+
+
+__all__ = [
+ "BookReplayAdapter",
+ "BookReplayAdapterFactory",
+ "serve_book_replay_stdio",
+]
diff --git a/src/tracebook/book_replay/reference.py b/src/tracebook/book_replay/reference.py
new file mode 100644
index 0000000..fe1c54f
--- /dev/null
+++ b/src/tracebook/book_replay/reference.py
@@ -0,0 +1,207 @@
+"""Inspectable reference semantics for L3 book replay."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from decimal import Decimal, localcontext
+from typing import Dict, List, Tuple, cast
+
+from .._version import __version__
+from .model import (
+ BookReplayConfig,
+ BookReplayError,
+ BookReplayEvent,
+ BookReplayObservation,
+ BookReplaySnapshot,
+ BookReplayState,
+ EngineMetadata,
+ Outcome,
+ RestingBookOrder,
+ SimulatedFill,
+ canonical_decimal,
+)
+
+
+@dataclass
+class _Order:
+ order_id: int
+ side: str
+ price: Decimal
+ quantity: Decimal
+
+
+class _Book:
+ def __init__(self) -> None:
+ self.levels: Dict[str, Dict[Decimal, List[_Order]]] = {"BUY": {}, "SELL": {}}
+ self.orders: Dict[int, _Order] = {}
+
+
+class ReferenceBookReplayAdapter:
+ """Mirror L3 deltas and simulate fills without crossing submitted orders."""
+
+ def __init__(
+ self,
+ config: BookReplayConfig,
+ engine_name: str = "tracebook-book-replay-reference",
+ engine_version: str = __version__,
+ ) -> None:
+ if not isinstance(config, BookReplayConfig):
+ raise BookReplayError("reference adapter requires BookReplayConfig")
+ self.config = config
+ self.metadata = EngineMetadata(engine_name, engine_version, "Python")
+ self._books: Dict[str, _Book] = {}
+
+ def apply(self, event: BookReplayEvent, index: int) -> BookReplayObservation:
+ """Apply one delta or probe and emit its canonical observation."""
+ if not isinstance(event, BookReplayEvent):
+ raise BookReplayError("reference adapter requires BookReplayEvent values")
+
+ fills: Tuple[SimulatedFill, ...] = ()
+ outcome = Outcome("applied")
+ if event.op == "add":
+ outcome = self._add(event)
+ elif event.op == "update":
+ outcome = self._update(event)
+ elif event.op == "delete":
+ outcome = self._delete(event)
+ else:
+ fills = self._probe(event)
+
+ state = self.snapshot()
+ return BookReplayObservation(index, outcome, fills, state.digest(), state.order_count)
+
+ def _add(self, event: BookReplayEvent) -> Outcome:
+ book = self._books.setdefault(event.symbol, _Book())
+ order_id = cast(int, event.order_id)
+ if order_id in book.orders:
+ return Outcome(
+ "rejected",
+ "DUPLICATE_ORDER_ID",
+ f"Order {order_id} is already present in {event.symbol}",
+ )
+ side = cast(str, event.side)
+ price = Decimal(cast(str, event.price))
+ order = _Order(order_id, side, price, Decimal(cast(str, event.quantity)))
+ book.orders[order_id] = order
+ book.levels[side].setdefault(price, []).append(order)
+ return Outcome("applied")
+
+ def _update(self, event: BookReplayEvent) -> Outcome:
+ book = self._books.get(event.symbol)
+ order_id = cast(int, event.order_id)
+ if book is None or order_id not in book.orders:
+ return Outcome(
+ "rejected",
+ "ORDER_NOT_ACTIVE",
+ f"Order {order_id} is not present in {event.symbol}",
+ )
+
+ order = book.orders[order_id]
+ new_side = cast(str, event.side)
+ new_price = Decimal(cast(str, event.price))
+ new_quantity = Decimal(cast(str, event.quantity))
+ if order.side == new_side and order.price == new_price:
+ # A mirrored update does not decide venue priority. Like NautilusTrader's
+ # BookLevel update, replacing the value under the same key keeps its slot.
+ order.quantity = new_quantity
+ return Outcome("applied")
+
+ self._remove_from_level(book, order)
+ order.side = new_side
+ order.price = new_price
+ order.quantity = new_quantity
+ book.levels[new_side].setdefault(new_price, []).append(order)
+ return Outcome("applied")
+
+ def _delete(self, event: BookReplayEvent) -> Outcome:
+ book = self._books.get(event.symbol)
+ order_id = cast(int, event.order_id)
+ if book is None or order_id not in book.orders:
+ return Outcome(
+ "rejected",
+ "ORDER_NOT_ACTIVE",
+ f"Order {order_id} is not present in {event.symbol}",
+ )
+ order = book.orders.pop(order_id)
+ self._remove_from_level(book, order)
+ return Outcome("applied")
+
+ @staticmethod
+ def _remove_from_level(book: _Book, order: _Order) -> None:
+ levels = book.levels[order.side]
+ level = levels[order.price]
+ level.remove(order)
+ if not level:
+ del levels[order.price]
+
+ def _probe(self, event: BookReplayEvent) -> Tuple[SimulatedFill, ...]:
+ book = self._books.get(event.symbol)
+ if book is None:
+ return ()
+
+ incoming_side = cast(str, event.side)
+ resting_side = "SELL" if incoming_side == "BUY" else "BUY"
+ limit_price = Decimal(cast(str, event.price))
+ remaining = Decimal(cast(str, event.quantity))
+ levels = book.levels[resting_side]
+ prices = sorted(levels, reverse=resting_side == "BUY")
+ fills = []
+ for price in prices:
+ if incoming_side == "BUY" and price > limit_price:
+ break
+ if incoming_side == "SELL" and price < limit_price:
+ break
+ for order in levels[price]:
+ quantity = min(order.quantity, remaining)
+ fills.append(
+ SimulatedFill(
+ price=canonical_decimal(price),
+ quantity=canonical_decimal(quantity),
+ )
+ )
+ # Align both coefficients before subtracting. The ambient
+ # Decimal context must not erase small FIFO fills.
+ with localcontext() as context:
+ exponent = min(
+ cast(int, remaining.as_tuple().exponent),
+ cast(int, quantity.as_tuple().exponent),
+ )
+ context.prec = max(remaining.adjusted(), quantity.adjusted()) - exponent + 2
+ remaining -= quantity
+ if remaining == 0:
+ return tuple(fills)
+ return tuple(fills)
+
+ def snapshot(self) -> BookReplayState:
+ """Return every mirrored order in price-time priority order."""
+ snapshots = []
+ for symbol in sorted(self._books):
+ book = self._books[symbol]
+ snapshots.append(
+ BookReplaySnapshot(
+ symbol=symbol,
+ bids=self._snapshot_side(book, "BUY"),
+ asks=self._snapshot_side(book, "SELL"),
+ )
+ )
+ return BookReplayState(tuple(snapshots))
+
+ @staticmethod
+ def _snapshot_side(book: _Book, side: str) -> Tuple[RestingBookOrder, ...]:
+ orders = []
+ for price in sorted(book.levels[side], reverse=side == "BUY"):
+ for order in book.levels[side][price]:
+ orders.append(
+ RestingBookOrder(
+ order_id=order.order_id,
+ price=canonical_decimal(order.price),
+ quantity=canonical_decimal(order.quantity),
+ )
+ )
+ return tuple(orders)
+
+ def close(self) -> None:
+ """Reference adapters own no external resources."""
+
+
+__all__ = ["ReferenceBookReplayAdapter"]
diff --git a/src/tracebook/conformance/external.py b/src/tracebook/conformance/external.py
index e4f1d48..c33b668 100644
--- a/src/tracebook/conformance/external.py
+++ b/src/tracebook/conformance/external.py
@@ -253,19 +253,28 @@ def close(self) -> None:
)
finally:
self._closed = True
- self._shutdown()
- if not was_broken and close_error is None and self._process.returncode != 0:
- close_error = AdapterProtocolError(
- f"candidate exited with code {self._process.returncode} after complete"
+ shutdown_timed_out = self._shutdown(
+ timeout_seconds=self.timeout_seconds if not self._broken else 0.5
)
+ if not was_broken and close_error is None:
+ if shutdown_timed_out:
+ close_error = AdapterProtocolError(
+ f"candidate timed out exiting after complete ({self.timeout_seconds:g}s)"
+ )
+ elif self._process.returncode != 0:
+ close_error = AdapterProtocolError(
+ f"candidate exited with code {self._process.returncode} after complete"
+ )
if close_error is not None:
raise close_error
- def _shutdown(self) -> None:
+ def _shutdown(self, *, timeout_seconds: float = 0.5) -> bool:
+ timed_out = False
if self._process.poll() is None:
try:
- self._process.wait(timeout=0.5)
+ self._process.wait(timeout=timeout_seconds)
except subprocess.TimeoutExpired:
+ timed_out = True
self._process.terminate()
try:
self._process.wait(timeout=0.5)
@@ -285,6 +294,7 @@ def _shutdown(self) -> None:
stream.close()
except OSError:
pass
+ return timed_out
class ExternalProcessAdapterFactory:
diff --git a/src/tracebook/conformance/model.py b/src/tracebook/conformance/model.py
index 9890bc8..0fd86f5 100644
--- a/src/tracebook/conformance/model.py
+++ b/src/tracebook/conformance/model.py
@@ -54,7 +54,10 @@ def canonical_decimal(value: Any, decimal_places: Optional[int] = None) -> str:
raise ConformanceError(f"numeric value cannot be normalized: {value!r}") from exc
if number == 0:
return "0"
- return format(number.normalize(), "f")
+ # Decimal.normalize() applies the ambient precision and can silently round
+ # an otherwise exact value. Strip insignificant zeros from fixed-point text.
+ rendered = format(number, "f")
+ return rendered.rstrip("0").rstrip(".") if "." in rendered else rendered
def _positive_int(value: Any, field_name: str) -> int:
diff --git a/src/tracebook/visualization/dashboard.py b/src/tracebook/visualization/dashboard.py
index 2d1d46a..2daac08 100644
--- a/src/tracebook/visualization/dashboard.py
+++ b/src/tracebook/visualization/dashboard.py
@@ -193,7 +193,7 @@ def set_order_book_manager(self, manager: OrderBookManager):
def _create_app(self) -> dash.Dash:
"""Create the Dash application."""
- app = dash.Dash(__name__)
+ app = dash.Dash(__name__, title="Tracebook · Performance Dashboard")
app.layout = html.Div( # type: ignore[misc] # Dash types layout as read-only
[
diff --git a/tests/fixtures/faulty_book_replay_adapter.py b/tests/fixtures/faulty_book_replay_adapter.py
new file mode 100644
index 0000000..8525798
--- /dev/null
+++ b/tests/fixtures/faulty_book_replay_adapter.py
@@ -0,0 +1,54 @@
+#!/usr/bin/env python3
+"""Protocol fixture that visibly requeues one same-level size increase."""
+
+from tracebook.book_replay import (
+ BookReplayObservation,
+ BookReplaySnapshot,
+ BookReplayState,
+ EngineMetadata,
+ ReferenceBookReplayAdapter,
+ serve_book_replay_stdio,
+)
+
+
+class FaultyBookReplayAdapter:
+ def __init__(self, config, engine_name="faulty-book-replay-test-adapter"):
+ self._inner = ReferenceBookReplayAdapter(config)
+ self._reorder = False
+ self.metadata = EngineMetadata(
+ engine_name,
+ "1",
+ "Python",
+ )
+
+ def apply(self, event, index):
+ observation = self._inner.apply(event, index)
+ if event.op == "update" and event.order_id == 1:
+ self._reorder = True
+ state = self.snapshot()
+ return BookReplayObservation(
+ index,
+ observation.outcome,
+ observation.fills,
+ state.digest(),
+ state.order_count,
+ )
+
+ def snapshot(self):
+ state = self._inner.snapshot()
+ if not self._reorder:
+ return state
+ books = []
+ for book in state.books:
+ bids = list(book.bids)
+ if book.symbol == "TEST" and len(bids) >= 2:
+ bids[0], bids[1] = bids[1], bids[0]
+ books.append(BookReplaySnapshot(book.symbol, tuple(bids), book.asks))
+ return BookReplayState(tuple(books))
+
+ def close(self):
+ self._inner.close()
+
+
+if __name__ == "__main__":
+ raise SystemExit(serve_book_replay_stdio(FaultyBookReplayAdapter))
diff --git a/tests/test_agent_qualification_experiment.py b/tests/test_agent_qualification_experiment.py
new file mode 100644
index 0000000..24e2e4a
--- /dev/null
+++ b/tests/test_agent_qualification_experiment.py
@@ -0,0 +1,404 @@
+import hashlib
+import json
+from pathlib import Path
+
+import pytest
+
+from experiments import agent_qualification
+
+
+def _file_hash(path: Path) -> str:
+ return hashlib.sha256(path.read_bytes()).hexdigest()
+
+
+def _manifest(tmp_path: Path, *, status: str = "ready") -> Path:
+ source = tmp_path / "source"
+ source.mkdir(parents=True)
+ (source / "engine.txt").write_text("price-time\n")
+ gold = tmp_path / "gold.json"
+ gold.write_text('{"terminal_class":"conformant"}\n')
+ manifest = {
+ "protocol": agent_qualification.PROTOCOL_ID,
+ "protocol_sha256": _file_hash(agent_qualification.PROTOCOL_PATH),
+ "prompt_sha256": _file_hash(agent_qualification.PROMPT_PATH),
+ "cases": [
+ {
+ "id": "c1",
+ "source": str(source),
+ "snapshot_id": "snapshot-c1",
+ "revision": "revision-c1",
+ "commands": "test-engine",
+ "declared_claim": "FIFO matching",
+ "excluded_scope": "latency",
+ "snapshot_sha256": agent_qualification._snapshot_digest(source),
+ "gold_manifest": str(gold),
+ "gold_sha256": _file_hash(gold),
+ "dependency_cache": {"kind": "none"},
+ "native_test_tool": {"kind": "none"},
+ "status": status,
+ }
+ ],
+ }
+ path = tmp_path / "cases.json"
+ path.write_text(json.dumps(manifest))
+ return path
+
+
+def _add_frozen_fixture(
+ manifest_path: Path,
+ fixture_root: Path,
+ *,
+ field_name: str,
+ kind: str,
+) -> None:
+ source = fixture_root / "source"
+ source.mkdir(parents=True)
+ (source / "package.bin").write_bytes(b"frozen dependency")
+ case = json.loads(manifest_path.read_text())["cases"][0]
+ inventory = agent_qualification._tree_inventory(source)
+ tree_hash = agent_qualification._snapshot_digest(source)
+ fixture_manifest = {
+ "kind": kind,
+ "case_id": case["id"],
+ "snapshot_sha256": case["snapshot_sha256"],
+ "tree_sha256": tree_hash,
+ "file_count": len(inventory),
+ "total_bytes": sum(item["bytes"] for item in inventory),
+ "files": inventory,
+ }
+ fixture_manifest_path = fixture_root / "manifest.json"
+ fixture_manifest_path.write_text(json.dumps(fixture_manifest))
+ payload = json.loads(manifest_path.read_text())
+ payload["cases"][0][field_name] = {
+ "kind": kind,
+ "source": str(source),
+ "manifest": str(fixture_manifest_path),
+ "manifest_sha256": _file_hash(fixture_manifest_path),
+ "tree_sha256": tree_hash,
+ }
+ manifest_path.write_text(json.dumps(payload))
+
+
+def test_validate_manifest_rejects_snapshot_drift(tmp_path):
+ manifest_path = _manifest(tmp_path)
+ manifest = agent_qualification.validate_manifest(manifest_path)
+ assert manifest["protocol"] == agent_qualification.PROTOCOL_ID
+
+ source = Path(manifest["cases"][0]["source"])
+ (source / "engine.txt").write_text("changed\n")
+
+ with pytest.raises(agent_qualification.EvaluationError, match="snapshot changed"):
+ agent_qualification.validate_manifest(manifest_path)
+
+
+def test_inventory_and_digest_both_ignore_git_metadata(tmp_path):
+ source = tmp_path / "source"
+ git = source / ".git"
+ git.mkdir(parents=True)
+ (source / "engine.txt").write_text("price-time\n")
+ (git / "index").write_text("first\n")
+ first_digest = agent_qualification._snapshot_digest(source)
+
+ assert [item["path"] for item in agent_qualification._tree_inventory(source)] == ["engine.txt"]
+ (git / "index").write_text("changed\n")
+ assert agent_qualification._snapshot_digest(source) == first_digest
+
+
+def test_validate_manifest_rejects_gold_drift(tmp_path):
+ manifest_path = _manifest(tmp_path)
+ manifest = agent_qualification.validate_manifest(manifest_path)
+
+ Path(manifest["cases"][0]["gold_manifest"]).write_text(
+ '{"terminal_class":"candidate-defect"}\n'
+ )
+
+ with pytest.raises(agent_qualification.EvaluationError, match="gold manifest changed"):
+ agent_qualification.validate_manifest(manifest_path)
+
+
+def test_frozen_cache_copy_is_verified_and_isolated(tmp_path, monkeypatch):
+ manifest_path = _manifest(tmp_path)
+ _add_frozen_fixture(
+ manifest_path,
+ tmp_path / "fixtures" / "nuget",
+ field_name="dependency_cache",
+ kind="nuget-global-packages-v1",
+ )
+ monkeypatch.setattr(agent_qualification, "PRIVATE_ROOT", tmp_path)
+
+ manifest = agent_qualification.validate_manifest(manifest_path)
+ case = manifest["cases"][0]
+ scratch = tmp_path / "scratch"
+ agent_qualification._clean_agent_environment(scratch)
+ prepared = agent_qualification._prepare_case_fixtures(case, scratch)
+
+ assert (
+ prepared["dependency_cache"]["initial_tree_sha256"]
+ == case["dependency_cache"]["tree_sha256"]
+ )
+ copied = scratch / "nuget" / "package.bin"
+ copied.write_bytes(b"mutated run copy")
+ source = Path(case["dependency_cache"]["source"]) / "package.bin"
+ assert source.read_bytes() == b"frozen dependency"
+ agent_qualification._record_final_fixture_state(prepared, scratch)
+ assert prepared["dependency_cache"]["mutated"] is True
+
+
+def test_frozen_cache_rejects_tree_drift(tmp_path, monkeypatch):
+ manifest_path = _manifest(tmp_path)
+ fixture_root = tmp_path / "fixtures" / "nuget"
+ _add_frozen_fixture(
+ manifest_path,
+ fixture_root,
+ field_name="dependency_cache",
+ kind="nuget-global-packages-v1",
+ )
+ monkeypatch.setattr(agent_qualification, "PRIVATE_ROOT", tmp_path)
+ (fixture_root / "source" / "package.bin").write_bytes(b"drift")
+
+ with pytest.raises(agent_qualification.EvaluationError, match="manifest binding"):
+ agent_qualification.validate_manifest(manifest_path)
+
+
+def test_render_prompt_keeps_baseline_blind():
+ case = {
+ "id": "c1",
+ "snapshot_id": "snapshot-c1",
+ "revision": "revision-c1",
+ "commands": "test-engine",
+ "declared_claim": "FIFO matching",
+ "excluded_scope": "latency",
+ }
+
+ baseline = agent_qualification.render_prompt(case, "baseline")
+ docs = agent_qualification.render_prompt(case, "docs")
+
+ assert "tracebook" not in baseline.lower()
+ assert "tracebook-conformance==0.6.0" in docs
+ assert "{{" not in baseline
+ assert "snapshot-c1" in baseline
+
+
+def test_plan_is_seeded_and_excludes_pending_cases(tmp_path):
+ manifest_path = _manifest(tmp_path)
+ pending_path = _manifest(tmp_path / "pending", status="pending-gold")
+ first_path = tmp_path / "first.json"
+ second_path = tmp_path / "second.json"
+ pending_plan_path = tmp_path / "pending-plan.json"
+
+ first = agent_qualification.create_plan(
+ manifest_path, first_path, repetitions=3, conditions=("baseline",)
+ )
+ second = agent_qualification.create_plan(
+ manifest_path, second_path, repetitions=3, conditions=("baseline",)
+ )
+ pending = agent_qualification.create_plan(
+ pending_path,
+ pending_plan_path,
+ repetitions=3,
+ conditions=("baseline",),
+ )
+
+ assert first["entries"] == second["entries"]
+ assert len(first["entries"]) == 6
+ assert {entry["agent"] for entry in first["entries"]} == {"codex", "claude"}
+ assert pending["entries"] == []
+
+
+def test_skill_condition_is_locked_before_baseline(tmp_path):
+ manifest_path = _manifest(tmp_path)
+
+ with pytest.raises(agent_qualification.EvaluationError, match="remain locked"):
+ agent_qualification.execute_run(
+ manifest_path,
+ case_id="c1",
+ agent="codex",
+ repetition=1,
+ condition="skill",
+ runs_path=tmp_path / "runs",
+ timeout_seconds=1,
+ )
+
+
+def test_claude_permissions_delegate_paths_to_fail_closed_sandbox(tmp_path):
+ workspace = tmp_path / "workspace"
+ scratch = tmp_path / "scratch"
+ workspace.mkdir()
+ scratch.mkdir()
+
+ settings = agent_qualification._claude_settings(workspace, scratch)
+
+ assert settings["permissions"]["allow"] == [
+ f"Read(/{workspace}/**)",
+ f"Edit(/{workspace}/**)",
+ "Bash",
+ "WebSearch",
+ "WebFetch",
+ ]
+ assert f"Read(/{Path.home()}/**)" in settings["permissions"]["deny"]
+ assert f"Edit(/{workspace}/.git/**)" in settings["permissions"]["deny"]
+ assert str(workspace) in settings["sandbox"]["filesystem"]["allowWrite"]
+ assert str(workspace / ".git") in settings["sandbox"]["filesystem"]["denyWrite"]
+ assert str(Path.home()) in settings["sandbox"]["filesystem"]["denyRead"]
+ for candidate in (
+ Path("/usr/local/bin/dotnet"),
+ Path("/usr/local/share/dotnet"),
+ Path("/usr/local/opt"),
+ Path("/usr/local/Cellar"),
+ ):
+ assert (str(candidate) in settings["sandbox"]["filesystem"]["allowRead"]) is (
+ candidate.exists()
+ )
+ _, dotnet_launcher_dir, dotnet_root = agent_qualification._dotnet_layout()
+ if dotnet_launcher_dir is None:
+ assert "DOTNET_ROOT" not in settings["env"]
+ else:
+ assert settings["env"]["PATH"].startswith(str(dotnet_launcher_dir))
+ assert settings["env"]["DOTNET_ROOT"] == str(dotnet_root)
+ assert settings["env"]["NuGetAudit"] == "false"
+
+
+def test_claude_command_ignores_user_and_project_settings(tmp_path):
+ command = agent_qualification._claude_command(tmp_path / "settings.json")
+
+ assert command[command.index("--setting-sources") + 1] == ""
+ assert command[command.index("--settings") + 1] == str(tmp_path / "settings.json")
+
+
+def test_dotnet_environment_preseeds_nuget_migration_state(tmp_path):
+ environment = agent_qualification._dotnet_environment(tmp_path)
+
+ xdg_data_home = tmp_path / "xdg-data"
+ assert environment["XDG_DATA_HOME"] == str(xdg_data_home)
+ assert (xdg_data_home / "NuGet" / "Migrations" / "1").is_file()
+
+
+def test_codex_command_uses_pure_permission_profile(tmp_path):
+ workspace = tmp_path / "workspace"
+ scratch = tmp_path / "scratch"
+ workspace.mkdir()
+ scratch.mkdir()
+
+ command = agent_qualification._codex_command(workspace, scratch)
+ permission_override = next(
+ item for item in command if item.startswith("permissions.agent-eval=")
+ )
+
+ assert command[-1] == "-"
+ assert "--sandbox" not in command
+ assert "workspace-write" not in command
+ assert command.index("exec") > command.index("--disable")
+ assert 'default_permissions="agent-eval"' in command
+ assert '":root"="deny"' in permission_override
+ assert '":workspace_roots"={"."="write",".git"="read"}' in permission_override
+ assert f'{scratch}"="write"' in permission_override
+ for candidate in (
+ Path("/usr/local/bin/dotnet"),
+ Path("/usr/local/share/dotnet"),
+ Path("/usr/local/opt"),
+ Path("/usr/local/Cellar"),
+ ):
+ assert (f'"{candidate}"="read"' in permission_override) is candidate.exists()
+ assert 'mode="limited"' in permission_override
+ assert "allow_upstream_proxy=false" in permission_override
+ assert "allow_local_binding=true" not in permission_override
+ assert '"pypi.org"="allow"' in permission_override
+ assert "nuget.org" not in permission_override
+ assert 'shell_environment_policy.inherit="core"' in command
+ assert any(item.startswith("shell_environment_policy.set=") for item in command)
+ assert "--search" in command
+ shell_environment = agent_qualification._codex_shell_environment(scratch)
+ _, dotnet_launcher_dir, dotnet_root = agent_qualification._dotnet_layout()
+ if dotnet_launcher_dir is None:
+ assert "DOTNET_ROOT" not in shell_environment
+ else:
+ assert shell_environment["PATH"].startswith(str(dotnet_launcher_dir))
+ assert shell_environment["DOTNET_ROOT"] == str(dotnet_root)
+
+
+def test_clean_environment_removes_secret_shaped_names(tmp_path, monkeypatch):
+ monkeypatch.setenv("AGENT_EVAL_SAFE_VALUE", "safe")
+ monkeypatch.setenv("UNLISTED_PRIVATE_TOKEN", "secret")
+ monkeypatch.setenv("SSH_AUTH_SOCK", "/private/tmp/agent.sock")
+
+ environment = agent_qualification._clean_agent_environment(tmp_path)
+
+ assert environment["AGENT_EVAL_SAFE_VALUE"] == "safe"
+ assert "UNLISTED_PRIVATE_TOKEN" not in environment
+ assert "SSH_AUTH_SOCK" not in environment
+
+
+def test_codex_home_is_outside_agent_writable_scratch(tmp_path, monkeypatch):
+ external_root = tmp_path / "run"
+ scratch = external_root / "scratch"
+ external_root.mkdir()
+ scratch.mkdir()
+ auth = tmp_path / "auth.json"
+ auth.write_text("{}\n")
+ monkeypatch.setattr(agent_qualification, "CODEX_AUTH_PATH", auth)
+ environment: dict[str, str] = {}
+
+ codex_home = agent_qualification._prepare_codex_environment(external_root, scratch, environment)
+
+ assert codex_home == external_root / "codex-home"
+ assert scratch not in codex_home.parents
+ assert (codex_home / "auth.json").is_symlink()
+ assert Path(environment["HOME"]) == scratch / "shell-home"
+ assert Path(environment["CODEX_HOME"]) == codex_home
+ assert Path(environment["CODEX_SQLITE_HOME"]) == codex_home / "state"
+
+
+def test_reported_model_uses_claude_transcript_and_codex_command(tmp_path):
+ transcript = tmp_path / "events.jsonl"
+ transcript.write_text(
+ '{"type":"system","model":"claude-opus-exact"}\n' '{"type":"result","result":"done"}\n'
+ )
+
+ assert agent_qualification._reported_model(transcript, "claude") == (
+ "claude-opus-exact",
+ "transcript",
+ )
+ assert agent_qualification._reported_model(transcript, "codex") == (
+ "gpt-5.6-sol",
+ "command",
+ )
+
+
+def test_workspace_evidence_includes_gitignored_files(tmp_path):
+ source = tmp_path / "source"
+ source.mkdir()
+ (source / ".gitignore").write_text("ignored.txt\n")
+ (source / "tracked.txt").write_text("before\n")
+ workspace = tmp_path / "workspace"
+ agent_qualification._initialize_snapshot(source, workspace)
+ (workspace / "tracked.txt").write_text("after\n")
+ (workspace / "ignored.txt").write_text("captured\n")
+ run_root = tmp_path / "run"
+ run_root.mkdir()
+
+ agent_qualification._write_workspace_evidence(workspace, run_root)
+
+ status = (run_root / "git-status.txt").read_text()
+ patch = (run_root / "workspace.patch").read_text()
+ inventory = json.loads((run_root / "workspace-files.json").read_text())["files"]
+ assert "ignored.txt" in status
+ assert "ignored.txt" in patch
+ assert "captured" in patch
+ assert "ignored.txt" in {item["path"] for item in inventory}
+
+
+def test_interrupted_run_is_quarantined_and_external_workspace_removed(tmp_path):
+ run_root = tmp_path / "runs" / "docs__case__codex__r1"
+ external = tmp_path / "external"
+ run_root.mkdir(parents=True)
+ external.mkdir()
+ (run_root / "partial.jsonl").write_text("partial\n")
+ (external / "workspace").mkdir()
+
+ archived = agent_qualification._quarantine_interrupted_run(run_root, external)
+
+ assert archived is not None
+ assert archived.parent.name == "interruptions"
+ assert (archived / "partial.jsonl").read_text() == "partial\n"
+ assert not run_root.exists()
+ assert not external.exists()
diff --git a/tests/test_agent_qualification_generalization.py b/tests/test_agent_qualification_generalization.py
new file mode 100644
index 0000000..cbef569
--- /dev/null
+++ b/tests/test_agent_qualification_generalization.py
@@ -0,0 +1,163 @@
+import hashlib
+import json
+from pathlib import Path
+
+import pytest
+
+from experiments import agent_qualification_generalization as generalization
+
+
+def _sha256(path: Path) -> str:
+ return hashlib.sha256(path.read_bytes()).hexdigest()
+
+
+def test_plan_is_seeded_complete_and_excludes_v1_cases():
+ manifest = {
+ "cases": [
+ {"id": "c4-compatible", "status": "ready"},
+ {"id": "c5-boundary", "status": "ready"},
+ ]
+ }
+
+ first = generalization.create_plan(manifest)
+ second = generalization.create_plan(manifest)
+
+ assert first == second
+ assert first["entry_count"] == 24
+ assert [entry["order"] for entry in first["entries"]] == list(range(1, 25))
+ assert {entry["agent"] for entry in first["entries"]} == {"codex", "claude"}
+ assert {entry["condition"] for entry in first["entries"]} == {"docs", "skill"}
+ assert {entry["case_id"] for entry in first["entries"]} == {
+ "c4-compatible",
+ "c5-boundary",
+ }
+ assert all("c1" not in entry["id"] and "c3" not in entry["id"] for entry in first["entries"])
+
+
+def test_pending_cases_are_not_scheduled():
+ manifest = {
+ "cases": [
+ {"id": "ready", "status": "ready"},
+ {"id": "pending", "status": "pending-gold"},
+ ]
+ }
+
+ plan = generalization.create_plan(manifest)
+
+ assert plan["entry_count"] == 12
+ assert {entry["case_id"] for entry in plan["entries"]} == {"ready"}
+
+
+def test_docs_and_skill_prompts_are_text_identical_before_skill_injection(tmp_path, monkeypatch):
+ prompt = tmp_path / "prompt.txt"
+ prompt.write_text(
+ "case={{case_id}}\n"
+ "snapshot={{snapshot_id}}\n"
+ "revision={{revision}}\n"
+ "commands={{commands}}\n"
+ "claim={{declared_claim}}\n"
+ "excluded={{excluded_scope}}\n"
+ )
+ monkeypatch.setattr(generalization, "PROMPT_PATH", prompt)
+ case = {
+ "id": "heldout",
+ "snapshot_id": "sha256:abc",
+ "revision": "pinned",
+ "commands": "native-test",
+ "declared_claim": "price-time",
+ "excluded_scope": "latency",
+ }
+
+ docs = generalization.render_prompt(case, "docs")
+ skill = generalization.render_prompt(case, "skill")
+
+ assert docs == skill
+ assert "tracebook-conformance==0.6.0" in docs
+ assert "{{" not in docs
+
+
+def test_snapshot_digest_binds_file_mode_and_content(tmp_path):
+ source = tmp_path / "source"
+ source.mkdir()
+ file_path = source / "engine.txt"
+ file_path.write_text("price-time\n")
+ first = generalization._snapshot_digest(source)
+
+ file_path.write_text("changed\n")
+ second = generalization._snapshot_digest(source)
+ file_path.write_text("price-time\n")
+ file_path.chmod(0o755)
+ third = generalization._snapshot_digest(source)
+
+ assert len({first, second, third}) == 3
+
+
+def test_tree_inventory_uses_same_git_exclusion_as_snapshot_digest(tmp_path):
+ source = tmp_path / "source"
+ git = source / ".git"
+ git.mkdir(parents=True)
+ (source / "engine.txt").write_text("price-time\n")
+ (git / "index").write_text("metadata\n")
+
+ assert [item["path"] for item in generalization._tree_inventory(source)] == ["engine.txt"]
+
+
+def test_dependency_target_is_explicit_and_restricted():
+ assert generalization._dependency_target("m2-repository") == "m2-repository"
+ assert generalization._dependency_target("m2-seed") == "m2-seed"
+ assert generalization._dependency_target("cargo-home") == "cargo-home"
+ assert generalization._dependency_target("cargo-vendor") == "cargo-vendor"
+
+ with pytest.raises(generalization.GeneralizationError, match="dependency_target"):
+ generalization._dependency_target("../operator-cache")
+
+
+def test_validate_tree_rejects_inventory_drift(tmp_path, monkeypatch):
+ private = tmp_path / "private"
+ source = private / "cache"
+ source.mkdir(parents=True)
+ (source / "dependency.bin").write_bytes(b"frozen")
+ manifests = private / "manifests"
+ manifests.mkdir()
+ snapshot_sha256 = "candidate-tree"
+ inventory = generalization._tree_inventory(source)
+ tree_sha256 = generalization._snapshot_digest(source)
+ tree_manifest = {
+ "kind": "frozen-tree-v1",
+ "case_id": "case",
+ "name": "dependency-cache",
+ "snapshot_sha256": snapshot_sha256,
+ "tree_sha256": tree_sha256,
+ "file_count": len(inventory),
+ "total_bytes": sum(item["bytes"] for item in inventory),
+ "files": inventory,
+ }
+ manifest_path = manifests / "tree.json"
+ manifest_path.write_text(json.dumps(tree_manifest))
+ monkeypatch.setattr(generalization, "REPOSITORY_ROOT", tmp_path)
+ monkeypatch.setattr(generalization, "PRIVATE_ROOT", private)
+ case = {"id": "case", "snapshot_sha256": snapshot_sha256}
+ declaration = {
+ "kind": "frozen-tree-v1",
+ "source": "private/cache",
+ "manifest": "private/manifests/tree.json",
+ "manifest_sha256": _sha256(manifest_path),
+ "tree_sha256": tree_sha256,
+ }
+
+ generalization._validate_tree(case, declaration, name="dependency-cache")
+ (source / "dependency.bin").write_bytes(b"drift")
+
+ with pytest.raises(generalization.GeneralizationError, match="inventory drifted"):
+ generalization._validate_tree(case, declaration, name="dependency-cache")
+
+
+def test_freeze_refuses_to_overwrite_existing_bindings(tmp_path, monkeypatch):
+ manifest = tmp_path / "cases.json"
+ manifest.write_text("{}\n")
+ monkeypatch.setattr(generalization, "MANIFEST_PATH", manifest)
+ monkeypatch.setattr(generalization, "PLAN_PATH", tmp_path / "plan.json")
+ monkeypatch.setattr(generalization, "FREEZE_PATH", tmp_path / "freeze.json")
+
+ with pytest.raises(generalization.GeneralizationError, match="refusing to overwrite"):
+ generalization.freeze()
diff --git a/tests/test_agent_qualification_generalization_runs.py b/tests/test_agent_qualification_generalization_runs.py
new file mode 100644
index 0000000..a14e4be
--- /dev/null
+++ b/tests/test_agent_qualification_generalization_runs.py
@@ -0,0 +1,463 @@
+import hashlib
+import json
+from contextlib import nullcontext
+from pathlib import Path
+
+import pytest
+
+from experiments import agent_qualification_generalization_runs as execution
+
+
+def test_plan_lookup_uses_exact_frozen_id():
+ plan = {"entries": [{"id": "docs__case__codex__r1", "order": 1}]}
+
+ assert execution._plan_entry(plan, "docs__case__codex__r1")["order"] == 1
+ with pytest.raises(execution.ExecutionError, match="not one frozen plan entry"):
+ execution._plan_entry(plan, "missing")
+
+
+def test_authorization_is_provider_and_case_specific(tmp_path, monkeypatch):
+ auth_root = tmp_path / "authorizations"
+ statement = "I approve C4 and C5 for Codex."
+ payload = {
+ "kind": execution.AUTHORIZATION_KIND,
+ "protocol": execution.PROTOCOL_ID,
+ "provider": "codex",
+ "case_ids": ["c4-compatible", "c5-boundary"],
+ "materials": [
+ "public repository snapshots",
+ "frozen prompt",
+ "docs-assisted treatment",
+ "skill-assisted treatment",
+ ],
+ "scope": "all-preregistered-runs-and-genuine-technical-restarts",
+ "user_statement": statement,
+ "statement_sha256": hashlib.sha256(statement.encode()).hexdigest(),
+ "recorded_at": "2026-08-01T00:00:00Z",
+ }
+ auth_root.mkdir()
+ (auth_root / "codex.json").write_text(json.dumps(payload))
+ monkeypatch.setattr(execution, "AUTHORIZATION_ROOT", auth_root)
+
+ assert execution.validate_authorization("codex")["provider"] == "codex"
+ with pytest.raises(execution.ExecutionError, match="cannot read claude authorization"):
+ execution.validate_authorization("claude")
+
+
+def test_authorization_rejects_statement_drift(tmp_path, monkeypatch):
+ auth_root = tmp_path / "authorizations"
+ auth_root.mkdir()
+ payload = {
+ "kind": execution.AUTHORIZATION_KIND,
+ "protocol": execution.PROTOCOL_ID,
+ "provider": "codex",
+ "case_ids": ["c4-compatible", "c5-boundary"],
+ "materials": [
+ "public repository snapshots",
+ "frozen prompt",
+ "docs-assisted treatment",
+ "skill-assisted treatment",
+ ],
+ "scope": "all-preregistered-runs-and-genuine-technical-restarts",
+ "user_statement": "changed",
+ "statement_sha256": "not-the-hash",
+ "recorded_at": "2026-08-01T00:00:00Z",
+ }
+ (auth_root / "codex.json").write_text(json.dumps(payload))
+ monkeypatch.setattr(execution, "AUTHORIZATION_ROOT", auth_root)
+
+ with pytest.raises(execution.ExecutionError, match="statement binding drifted"):
+ execution.validate_authorization("codex")
+
+
+def test_copy_fixture_starts_from_frozen_hash(tmp_path, monkeypatch):
+ repository = tmp_path / "repo"
+ source = repository / "private" / "cache"
+ source.mkdir(parents=True)
+ (source / "artifact").write_text("frozen")
+ tree_hash = execution.helpers._snapshot_digest(source)
+ scratch = tmp_path / "scratch"
+ scratch.mkdir()
+ (scratch / "cargo-home").mkdir()
+ monkeypatch.setattr(execution, "REPOSITORY_ROOT", repository)
+ case = {
+ "id": "c5-boundary",
+ "dependency_cache": {
+ "source": "private/cache",
+ "tree_sha256": tree_hash,
+ "manifest_sha256": "manifest",
+ "target": "cargo-vendor",
+ },
+ }
+
+ fixture = execution._copy_fixture(case, scratch)
+
+ assert fixture["initial_tree_sha256"] == tree_hash
+ assert Path(fixture["target"]).name == "cargo-vendor"
+ assert execution.helpers._snapshot_digest(Path(fixture["target"])) == tree_hash
+ config = scratch / "cargo-home" / "config.toml"
+ assert config.is_file()
+ assert f'directory = "{fixture["target"]}"' in config.read_text()
+ assert "offline = true" in config.read_text()
+
+ (scratch / "cargo-home" / ".global-cache").write_text("mutable Cargo metadata")
+ execution._record_fixture_final(fixture)
+ assert fixture["mutated"] is False
+
+ (Path(fixture["target"]) / "artifact").write_text("mutated")
+ execution._record_fixture_final(fixture)
+ assert fixture["mutated"] is True
+ assert fixture["final_tree_sha256"] != fixture["initial_tree_sha256"]
+
+
+def test_copy_maven_fixture_keeps_writable_repository_outside_hash(tmp_path, monkeypatch):
+ repository = tmp_path / "repo"
+ source = repository / "private" / "cache"
+ source.mkdir(parents=True)
+ (source / "artifact.jar").write_text("frozen")
+ tree_hash = execution.helpers._snapshot_digest(source)
+ scratch = tmp_path / "scratch"
+ scratch.mkdir()
+ monkeypatch.setattr(execution, "REPOSITORY_ROOT", repository)
+ case = {
+ "id": "c4-compatible",
+ "dependency_cache": {
+ "source": "private/cache",
+ "tree_sha256": tree_hash,
+ "manifest_sha256": "manifest",
+ "target": "m2-seed",
+ },
+ }
+
+ fixture = execution._copy_fixture(case, scratch)
+
+ assert Path(fixture["target"]).name == "m2-seed"
+ assert (scratch / "m2-repository" / "artifact.jar").read_text() == "frozen"
+ (scratch / "m2-repository" / "resolver-status.properties").write_text("mutable")
+ execution._record_fixture_final(fixture)
+ assert fixture["mutated"] is False
+
+
+def test_native_surface_has_no_skill_in_docs_condition(tmp_path, monkeypatch):
+ root = tmp_path / "run"
+ workspace = root / "workspace"
+ scratch = root / "scratch"
+ workspace.mkdir(parents=True)
+ scratch.mkdir()
+ monkeypatch.setattr(
+ execution,
+ "_configured_delivery",
+ lambda: nullcontext(),
+ )
+ monkeypatch.setattr(
+ execution.delivery,
+ "_prepare_native_surface",
+ lambda **kwargs: (
+ ["provider", "permissions.agent-eval={filesystem={}}"],
+ {
+ "skill_installed": kwargs["condition"] == "skill",
+ "semantic_file_count": 1 if kwargs["condition"] == "skill" else 0,
+ },
+ None,
+ ),
+ )
+
+ _, surface, _ = execution._native_surface(
+ provider="codex",
+ condition="docs",
+ workspace=workspace,
+ run_root=root,
+ scratch=scratch,
+ environment={},
+ skill=b"skill",
+ skill_name="probe",
+ )
+
+ assert surface == {"skill_installed": False, "semantic_file_count": 0}
+
+
+def test_codex_native_surface_can_read_declared_toolchain_roots(tmp_path, monkeypatch):
+ root = tmp_path / "run"
+ workspace = root / "workspace"
+ scratch = root / "scratch"
+ maven = tmp_path / "maven"
+ java = tmp_path / "java"
+ workspace.mkdir(parents=True)
+ scratch.mkdir()
+ (maven / "bin").mkdir(parents=True)
+ (java / "bin").mkdir(parents=True)
+ cargo_bin = Path.home() / ".cargo" / "bin"
+ existing_cargo_read = f'{json.dumps(str(cargo_bin))}="read",'
+ monkeypatch.setenv("TRACEBOOK_V2_MAVEN_HOME", str(maven))
+ monkeypatch.setenv("TRACEBOOK_V2_JAVA_HOME", str(java))
+ monkeypatch.setattr(execution, "_configured_delivery", lambda: nullcontext())
+ monkeypatch.setattr(
+ execution.delivery,
+ "_prepare_native_surface",
+ lambda **kwargs: (
+ [
+ "provider",
+ f"permissions.agent-eval={{filesystem={{{existing_cargo_read}}}}}",
+ ],
+ {"skill_installed": False},
+ None,
+ ),
+ )
+
+ command, _, _ = execution._native_surface(
+ provider="codex",
+ condition="docs",
+ workspace=workspace,
+ run_root=root,
+ scratch=scratch,
+ environment={},
+ skill=b"skill",
+ skill_name="probe",
+ )
+
+ permission = next(value for value in command if value.startswith("permissions.agent-eval="))
+ assert f'"{maven}"="read"' in permission
+ assert f'"{java}"="read"' in permission
+ assert permission.count(existing_cargo_read) == 1
+
+
+def test_validate_prior_runs_enforces_seeded_order(monkeypatch):
+ seen = []
+ monkeypatch.setattr(execution, "validate_run", seen.append)
+ plan = {
+ "entries": [
+ {"id": "first"},
+ {"id": "second"},
+ {"id": "third"},
+ ]
+ }
+
+ execution._validate_prior_runs(plan, 3)
+
+ assert seen == ["first", "second"]
+
+
+def test_validate_prior_runs_reuses_one_frozen_context(monkeypatch):
+ seen = []
+ freeze = {"frozen": True}
+ primary = ({"cases": []}, {"entries": []})
+
+ def validate(run_id, **kwargs):
+ seen.append((run_id, kwargs["freeze"], kwargs["primary"]))
+
+ monkeypatch.setattr(execution, "validate_run", validate)
+ plan = {"entries": [{"id": "first"}, {"id": "second"}, {"id": "third"}]}
+
+ execution._validate_prior_runs(plan, 3, freeze=freeze, primary=primary)
+
+ assert seen == [("first", freeze, primary), ("second", freeze, primary)]
+
+
+def test_execution_refuses_without_provider_authorization(monkeypatch):
+ monkeypatch.setattr(execution, "validate_freeze", lambda: {})
+ monkeypatch.setattr(
+ execution,
+ "_load_primary",
+ lambda: (
+ {"cases": []},
+ {
+ "entries": [
+ {
+ "id": "skill__c4-compatible__codex__r1",
+ "agent": "codex",
+ "case_id": "c4-compatible",
+ "condition": "skill",
+ "repetition": 1,
+ "order": 1,
+ }
+ ]
+ },
+ {},
+ ),
+ )
+ monkeypatch.setattr(
+ execution,
+ "validate_authorization",
+ lambda provider: (_ for _ in ()).throw(execution.ExecutionError("authorization missing")),
+ )
+
+ with pytest.raises(execution.ExecutionError, match="authorization missing"):
+ execution.execute_run("skill__c4-compatible__codex__r1")
+
+
+def test_wrapper_is_deterministic_and_refuses_drift(tmp_path, monkeypatch):
+ wrapper = tmp_path / ".claude-plugin" / "plugin.json"
+ monkeypatch.setattr(execution, "CLAUDE_WRAPPER_PATH", wrapper)
+
+ execution._write_wrapper()
+ first = wrapper.read_bytes()
+ execution._write_wrapper()
+ wrapper.write_text("{}")
+
+ assert json.loads(first) == execution._PLUGIN_MANIFEST
+ with pytest.raises(execution.ExecutionError, match="wrapper drifted"):
+ execution._write_wrapper()
+
+
+def test_toolchain_environment_prepends_frozen_tools(tmp_path, monkeypatch):
+ maven = tmp_path / "maven"
+ java = tmp_path / "java"
+ python = tmp_path / "python"
+ uv = tmp_path / "uv"
+ for root in (maven, java, python):
+ (root / "bin").mkdir(parents=True)
+ uv.write_text("binary")
+ monkeypatch.setenv("TRACEBOOK_V2_MAVEN_HOME", str(maven))
+ monkeypatch.setenv("TRACEBOOK_V2_JAVA_HOME", str(java))
+ monkeypatch.setenv("TRACEBOOK_V2_PYTHON_HOME", str(python))
+ monkeypatch.setenv("TRACEBOOK_V2_UV_BINARY", str(uv))
+ monkeypatch.setenv("PATH", "/operator/private/bin")
+
+ environment = execution._toolchain_environment()
+
+ assert environment["JAVA_HOME"] == str(java)
+ assert environment["PATH"].split(":")[:4] == [
+ str(maven / "bin"),
+ str(java / "bin"),
+ str(python / "bin"),
+ str(tmp_path),
+ ]
+ assert "/operator/private/bin" not in environment["PATH"]
+ assert "/usr/bin" in environment["PATH"].split(":")
+
+
+def test_provider_shells_receive_fresh_paths(tmp_path, monkeypatch):
+ monkeypatch.delenv("TRACEBOOK_V2_MAVEN_HOME", raising=False)
+ monkeypatch.delenv("TRACEBOOK_V2_JAVA_HOME", raising=False)
+ monkeypatch.delenv("TRACEBOOK_V2_PYTHON_HOME", raising=False)
+ monkeypatch.delenv("TRACEBOOK_V2_UV_BINARY", raising=False)
+
+ values = execution._codex_shell_environment(tmp_path)
+
+ assert values["FRESH_M2"] == str(tmp_path / "m2-repository")
+ assert values["FRESH_CARGO_HOME"] == str(tmp_path / "cargo-home")
+ assert values["FRESH_TARGET"] == str(tmp_path / "cargo-target")
+ assert values["FRESH_WORK_ROOT"] == str(tmp_path / "fresh-work")
+
+
+def test_claude_binding_probes_an_explicit_first_party_route(tmp_path, monkeypatch):
+ binary = tmp_path / "claude"
+ binary.write_text("claude")
+ observed = {}
+
+ monkeypatch.setattr(execution, "_provider_binary", lambda provider: binary)
+ monkeypatch.setattr(execution.helpers, "_binary_version", lambda *args: "2.1.220")
+
+ def run(command, **kwargs):
+ observed["command"] = command
+ return execution.subprocess.CompletedProcess(
+ command,
+ 0,
+ stdout=json.dumps(
+ {
+ "loggedIn": True,
+ "authMethod": "claude.ai",
+ "apiProvider": "firstParty",
+ "subscriptionType": "max",
+ }
+ ),
+ )
+
+ monkeypatch.setattr(execution.subprocess, "run", run)
+
+ binding = execution._provider_binding("claude")
+
+ command = observed["command"]
+ assert command[1:4] == ("--setting-sources", "", "--settings")
+ assert json.loads(command[4])["env"] == execution.CLAUDE_FIRST_PARTY_ENV
+ assert command[5:] == ("auth", "status")
+ assert binding["auth_route"]["apiProvider"] == "firstParty"
+
+
+def test_claude_run_settings_disable_third_party_routes(tmp_path, monkeypatch):
+ workspace = tmp_path / "workspace"
+ scratch = tmp_path / "scratch"
+ plugin_root = tmp_path / "plugin"
+ workspace.mkdir()
+ scratch.mkdir()
+ plugin_root.mkdir()
+ maven = tmp_path / "maven"
+ java = tmp_path / "java"
+ (maven / "bin").mkdir(parents=True)
+ (java / "bin").mkdir(parents=True)
+ monkeypatch.setenv("TRACEBOOK_V2_MAVEN_HOME", str(maven))
+ monkeypatch.setenv("TRACEBOOK_V2_JAVA_HOME", str(java))
+
+ settings = execution._claude_settings(workspace, scratch, plugin_root)
+
+ for name, value in execution.CLAUDE_FIRST_PARTY_ENV.items():
+ assert settings["env"][name] == value
+ assert str(maven) in settings["sandbox"]["filesystem"]["allowRead"]
+ assert str(java) in settings["sandbox"]["filesystem"]["allowRead"]
+
+
+def test_delivery_configuration_restores_imported_module_globals(tmp_path, monkeypatch):
+ binary = tmp_path / "provider"
+ binary.write_text("provider")
+ monkeypatch.setattr(execution, "_provider_binary", lambda provider: binary)
+ original = (
+ execution.helpers.CODEX_BINARY,
+ execution.helpers.CLAUDE_BINARY,
+ execution.helpers.CODEX_AUTH_PATH,
+ execution.delivery.DEFAULT_CLAUDE_WRAPPER_PATH,
+ execution.delivery.EXPECTED_MODELS,
+ execution.helpers._codex_shell_environment,
+ )
+
+ with execution._configured_delivery():
+ assert execution.helpers.CODEX_BINARY == binary
+ assert execution.helpers.CLAUDE_BINARY == binary
+ assert execution.helpers._codex_shell_environment is execution._codex_shell_environment
+
+ assert (
+ execution.helpers.CODEX_BINARY,
+ execution.helpers.CLAUDE_BINARY,
+ execution.helpers.CODEX_AUTH_PATH,
+ execution.delivery.DEFAULT_CLAUDE_WRAPPER_PATH,
+ execution.delivery.EXPECTED_MODELS,
+ execution.helpers._codex_shell_environment,
+ ) == original
+
+
+def test_interrupted_v2_run_is_quarantined_for_clean_restart(tmp_path, monkeypatch):
+ runs = tmp_path / "runs"
+ external = tmp_path / "external"
+ monkeypatch.setattr(execution, "RUNS_ROOT", runs)
+ monkeypatch.setattr(execution, "EXTERNAL_ROOT", external)
+
+ def fail(run_id, timeout_seconds):
+ (runs / run_id).mkdir(parents=True)
+ (external / run_id / "workspace").mkdir(parents=True)
+ raise KeyboardInterrupt("interrupted")
+
+ monkeypatch.setattr(execution, "_execute_run_once", fail)
+
+ with pytest.raises(KeyboardInterrupt, match="interrupted"):
+ execution.execute_run("docs__case__codex__r1")
+
+ assert not (runs / "docs__case__codex__r1").exists()
+ assert not (external / "docs__case__codex__r1").exists()
+ assert len(list((runs / "interruptions").iterdir())) == 1
+
+
+def test_shakedown_requires_first_party_claude_route(monkeypatch):
+ def binding(provider):
+ if provider == "claude":
+ return {
+ "auth_route": {
+ "loggedIn": True,
+ "authMethod": "third_party",
+ "apiProvider": "bedrock",
+ }
+ }
+ return {"requested_model": execution.EXPECTED_MODELS[provider]}
+
+ monkeypatch.setattr(execution, "_provider_binding", binding)
+
+ with pytest.raises(execution.ExecutionError, match="first-party route"):
+ execution._validated_provider_bindings()
diff --git a/tests/test_agent_qualification_treatments.py b/tests/test_agent_qualification_treatments.py
new file mode 100644
index 0000000..4580efd
--- /dev/null
+++ b/tests/test_agent_qualification_treatments.py
@@ -0,0 +1,401 @@
+import hashlib
+import json
+import random
+import subprocess
+import sys
+from pathlib import Path
+
+import pytest
+
+from experiments import agent_qualification
+from experiments import agent_qualification_treatments as treatments
+
+BASELINE_RUNNER_SHA256 = "71300722b273e983dcd4e830358677337255a39f52b7f10e4c57479c8dc5cb0d"
+
+
+def _sha256(path: Path) -> str:
+ return hashlib.sha256(path.read_bytes()).hexdigest()
+
+
+def _synthetic_manifest():
+ return {
+ "cases": [
+ {
+ "id": case_id,
+ "status": "ready",
+ "snapshot_id": f"sha256:{case_id}",
+ "revision": f"revision-{case_id}",
+ "commands": "native-test",
+ "declared_claim": "price-time matching",
+ "excluded_scope": "production readiness",
+ }
+ for case_id in ("case-alpha", "case-beta")
+ ]
+ }
+
+
+def test_frozen_baseline_runner_is_byte_identical():
+ assert _sha256(agent_qualification.RUNNER_PATH) == BASELINE_RUNNER_SHA256
+
+
+def test_treatment_prompt_is_identical_between_docs_and_skill():
+ manifest = _synthetic_manifest()
+ for case in manifest["cases"]:
+ if case["status"] != "ready":
+ continue
+ rendered = treatments.render_prompt(case)
+ assert rendered == agent_qualification.render_prompt(case, "docs")
+ assert rendered == agent_qualification.render_prompt(case, "skill")
+ assert "tracebook-conformance==0.6.0" in rendered
+
+
+def test_master_plan_is_two_matched_blocks_in_baseline_order():
+ manifest = _synthetic_manifest()
+ evidence_ids = [f"ev_{index:032x}" for index in range(24)]
+ entries = treatments._master_entries(manifest, evidence_ids=evidence_ids)
+ baseline_cells = [
+ {
+ "case_id": case["id"],
+ "agent": agent,
+ "repetition": repetition,
+ }
+ for case in manifest["cases"]
+ for agent in ("codex", "claude")
+ for repetition in range(1, treatments.REPETITIONS + 1)
+ ]
+ random.Random(agent_qualification.RANDOMIZATION_SEED).shuffle(baseline_cells)
+
+ assert len(entries) == 24
+ assert [entry["condition"] for entry in entries[:12]] == ["docs"] * 12
+ assert [entry["condition"] for entry in entries[12:]] == ["skill"] * 12
+ for offset in (0, 12):
+ assert [
+ {
+ "case_id": entry["case_id"],
+ "agent": entry["agent"],
+ "repetition": entry["repetition"],
+ }
+ for entry in entries[offset : offset + 12]
+ ] == baseline_cells
+ assert len({entry["run_id"] for entry in entries}) == 24
+ assert len({entry["evidence_id"] for entry in entries}) == 24
+ assert [entry["evidence_id"] for entry in entries] == evidence_ids
+
+
+def test_master_plan_preserves_manifest_case_order_before_seeded_shuffle():
+ manifest = {
+ "cases": [
+ {"id": "z-last-lexically", "status": "ready"},
+ {"id": "a-first-lexically", "status": "ready"},
+ ]
+ }
+ evidence_ids = [f"ev_{index:032x}" for index in range(24)]
+ entries = treatments._master_entries(manifest, evidence_ids=evidence_ids)
+ expected = [
+ {"case_id": case["id"], "agent": agent, "repetition": repetition}
+ for case in manifest["cases"]
+ for agent in ("codex", "claude")
+ for repetition in range(1, treatments.REPETITIONS + 1)
+ ]
+ random.Random(agent_qualification.RANDOMIZATION_SEED).shuffle(expected)
+
+ assert [
+ {
+ "case_id": entry["case_id"],
+ "agent": entry["agent"],
+ "repetition": entry["repetition"],
+ }
+ for entry in entries[:12]
+ ] == expected
+
+
+def test_evidence_ids_are_random_opaque_values():
+ first = treatments._new_evidence_id()
+ second = treatments._new_evidence_id()
+
+ assert first != second
+ assert len(first) == len("ev_") + 32
+ assert int(first.removeprefix("ev_"), 16) >= 0
+
+
+def test_skill_is_candidate_agnostic_and_exactly_one_measured_file(tmp_path, monkeypatch):
+ monkeypatch.setattr(treatments, "PRIVATE_ROOT", tmp_path)
+ skill_path = tmp_path / "skill" / "SKILL.md"
+ skill_path.parent.mkdir()
+ skill_path.write_text(
+ "---\n"
+ f"name: {treatments.SKILL_NAME}\n"
+ "description: Candidate-agnostic qualification workflow.\n"
+ "---\n\n"
+ "Inspect the supplied candidate and preserve identity lifecycles.\n"
+ )
+ skill = treatments._load_skill(skill_path, _synthetic_manifest())
+
+ assert skill["name"] == treatments.SKILL_NAME
+ assert skill["sha256"] == _sha256(skill_path)
+ assert b"heldout" not in skill["content"].lower()
+ assert b"order-matcher" not in skill["content"].lower()
+ assert b"matching-engine-rs" not in skill["content"].lower()
+
+
+def test_skill_leakage_audit_rejects_heldout_identifier(tmp_path, monkeypatch):
+ monkeypatch.setattr(treatments, "PRIVATE_ROOT", tmp_path)
+ skill = tmp_path / "SKILL.md"
+ skill.write_text(
+ "---\n"
+ f"name: {treatments.SKILL_NAME}\n"
+ "description: synthetic\n"
+ "---\n"
+ "Target c1-heldout directly.\n"
+ )
+ manifest = {"cases": [{"id": "c1-heldout"}]}
+
+ with pytest.raises(treatments.EvaluationError, match="leakage audit"):
+ treatments._load_skill(skill, manifest)
+
+
+def test_private_output_rejects_escape_before_creating_parent(tmp_path, monkeypatch):
+ private_root = tmp_path / "private"
+ private_root.mkdir()
+ outside = tmp_path / "outside" / "result.json"
+ monkeypatch.setattr(treatments, "PRIVATE_ROOT", private_root)
+
+ with pytest.raises(treatments.EvaluationError, match="must stay inside"):
+ treatments._private_output(outside, "test output")
+
+ assert not outside.parent.exists()
+
+
+def test_master_plan_shape_rejects_tampering():
+ entries = [
+ {
+ "ordinal": index,
+ "run_id": f"run-{index}",
+ "evidence_id": f"ev_{index:032x}",
+ "condition": "docs" if index <= 12 else "skill",
+ }
+ for index in range(1, 25)
+ ]
+ assert len(treatments._validate_plan_entry_shape(entries)) == 24
+ entries[12]["evidence_id"] = entries[0]["evidence_id"]
+
+ with pytest.raises(treatments.EvaluationError, match="evidence IDs"):
+ treatments._validate_plan_entry_shape(entries)
+
+
+def test_claude_treatment_command_is_skill_capable_and_customization_isolated(
+ tmp_path,
+):
+ command = treatments._claude_treatment_command(tmp_path / "settings.json", tmp_path / "plugin")
+
+ assert "--safe-mode" not in command
+ assert "--disable-slash-commands" not in command
+ assert command[command.index("--setting-sources") + 1] == ""
+ assert command[command.index("--plugin-dir") + 1] == str(tmp_path / "plugin")
+ assert "Skill" in command[command.index("--tools") + 1].split(",")
+ assert command[command.index("--model") + 1] == "claude-opus-4-8"
+ assert command[command.index("--mcp-config") + 1] == '{"mcpServers":{}}'
+
+
+def test_claude_wrapper_tree_contains_no_semantic_payload(tmp_path):
+ root = tmp_path / "claude-wrapper"
+ manifest = root / ".claude-plugin" / "plugin.json"
+ manifest.parent.mkdir(parents=True)
+ manifest.write_text('{"name":"agent-qualification-treatment","version":"1.0.0"}\n')
+ record = treatments._tree_record(root)
+
+ assert record["file_count"] == 1
+ assert record["files"][0]["path"] == ".claude-plugin/plugin.json"
+
+
+def test_claude_settings_make_plugin_read_only_to_agent(tmp_path):
+ workspace = tmp_path / "workspace"
+ scratch = tmp_path / "scratch"
+ plugin = scratch / "plugin"
+ workspace.mkdir()
+ scratch.mkdir()
+ plugin.mkdir()
+
+ settings = treatments._claude_treatment_settings(workspace, scratch, plugin)
+
+ assert "CLAUDE_CONFIG_DIR" not in settings["env"]
+ assert "HOME" not in settings["env"]
+ assert settings["env"]["ZDOTDIR"] == str(scratch / "zsh-env")
+ assert (scratch / "zsh-env" / ".zshenv").read_text() == (
+ f"export HOME={scratch / 'shell-home'}\n"
+ )
+ assert str(plugin) in settings["sandbox"]["filesystem"]["denyWrite"]
+ assert f"Edit(/{plugin}/**)" in settings["permissions"]["deny"]
+
+
+def test_codex_native_surface_differs_only_by_skill_file(tmp_path, monkeypatch):
+ auth = tmp_path / "auth.json"
+ auth.write_text("{}\n")
+ monkeypatch.setattr(agent_qualification, "CODEX_AUTH_PATH", auth)
+ workspace = tmp_path / "workspace"
+ workspace.mkdir()
+ skill = b"synthetic frozen skill\n"
+
+ docs_root = tmp_path / "docs"
+ docs_scratch = docs_root / "scratch"
+ docs_root.mkdir()
+ docs_scratch.mkdir()
+ docs_environment: dict[str, str] = {}
+ docs_command, docs_surface, _ = treatments._prepare_native_surface(
+ agent="codex",
+ condition="docs",
+ workspace=workspace,
+ external_run_root=docs_root,
+ scratch=docs_scratch,
+ environment=docs_environment,
+ skill_content=skill,
+ )
+
+ skill_root = tmp_path / "skill"
+ skill_scratch = skill_root / "scratch"
+ skill_root.mkdir()
+ skill_scratch.mkdir()
+ skill_environment: dict[str, str] = {}
+ skill_command, skill_surface, _ = treatments._prepare_native_surface(
+ agent="codex",
+ condition="skill",
+ workspace=workspace,
+ external_run_root=skill_root,
+ scratch=skill_scratch,
+ environment=skill_environment,
+ skill_content=skill,
+ )
+
+ assert docs_surface["skill_installed"] is False
+ assert skill_surface["skill_installed"] is True
+ assert not (docs_root / "codex-home" / "skills" / treatments.SKILL_NAME).exists()
+ installed = skill_root / "codex-home" / "skills" / treatments.SKILL_NAME / "SKILL.md"
+ assert installed.read_bytes() == skill
+ for command, root in ((docs_command, docs_root), (skill_command, skill_root)):
+ permission = next(item for item in command if item.startswith("permissions.agent-eval="))
+ assert f'"{root / "codex-home" / "skills"}"="read"' in permission
+
+
+def test_native_surface_final_audit_detects_absence_and_drift(tmp_path):
+ skill = b"synthetic frozen skill\n"
+ external_root = tmp_path / "run"
+ scratch = external_root / "scratch"
+ scratch.mkdir(parents=True)
+
+ docs = treatments._audit_native_surface(
+ agent="claude",
+ condition="docs",
+ external_run_root=external_root,
+ scratch=scratch,
+ skill_content=skill,
+ )
+ assert docs["mutated"] is False
+
+ path = scratch / "claude-plugin" / "skills" / treatments.SKILL_NAME / "SKILL.md"
+ path.parent.mkdir(parents=True)
+ path.write_bytes(b"drift")
+ treatment = treatments._audit_native_surface(
+ agent="claude",
+ condition="skill",
+ external_run_root=external_root,
+ scratch=scratch,
+ skill_content=skill,
+ )
+ assert treatment["mutated"] is True
+
+
+def test_transcript_terminal_state_requires_clean_completion(tmp_path):
+ codex = tmp_path / "codex.jsonl"
+ codex.write_text('{"type":"turn.completed"}\n')
+ claude = tmp_path / "claude.jsonl"
+ claude.write_text(
+ '{"type":"result","terminal_reason":"completed","subtype":"success",' '"result":"done"}\n'
+ )
+ failed = tmp_path / "failed.jsonl"
+ failed.write_text('{"type":"error","message":"network"}\n')
+
+ assert treatments._transcript_terminal_state(codex, "codex") == "completed"
+ assert treatments._transcript_terminal_state(claude, "claude") == "completed"
+ assert treatments._transcript_terminal_state(failed, "codex") == "failed"
+
+
+def test_claude_catalog_audit_requires_only_wrapper_and_expected_skill(tmp_path):
+ transcript = tmp_path / "claude.jsonl"
+ transcript.write_text(
+ "provider banner that is not JSON\n"
+ + json.dumps(
+ {
+ "type": "system",
+ "subtype": "init",
+ "model": "claude-opus-4-8",
+ "plugins": [{"name": "agent-qualification-treatment"}],
+ "skills": [f"agent-qualification-treatment:{treatments.SKILL_NAME}"],
+ "mcp_servers": [],
+ "tools": [
+ "Bash",
+ "Edit",
+ "Glob",
+ "Grep",
+ "Read",
+ "Skill",
+ "WebFetch",
+ "WebSearch",
+ "Write",
+ ],
+ }
+ )
+ + "\n"
+ )
+
+ assert treatments._claude_catalog_audit(transcript, condition="skill")["valid"] is True
+ assert treatments._claude_catalog_audit(transcript, condition="docs")["valid"] is False
+ assert treatments._transcript_result_text(transcript, "claude") == ""
+
+
+def test_transcript_result_text_skips_non_json_lines(tmp_path):
+ transcript = tmp_path / "claude-result.jsonl"
+ transcript.write_text('banner\n{"type":"result","result":"done"}\n')
+
+ assert treatments._transcript_result_text(transcript, "claude") == "done"
+
+
+def test_measurement_policy_uses_conservative_paired_timing_gate():
+ timing = treatments.MEASUREMENT_POLICY["timing_gate"]
+
+ assert timing["minimum_matched_autonomous_safe_pass_pairs"] == 3
+ assert timing["both_cases_required"] is True
+ assert timing["skill_safe_pass_rate_must_not_be_lower"] is True
+ assert timing["pass_threshold"] == 0.70
+
+
+def test_official_run_rejects_timeout_drift_before_freeze_validation():
+ with pytest.raises(treatments.EvaluationError, match="timeout must remain"):
+ treatments.execute_run("not-a-run", timeout_seconds=1)
+
+
+def test_main_converts_subprocess_failures_to_exit_two(monkeypatch, capsys):
+ monkeypatch.setattr(
+ treatments,
+ "run_native_delivery_shakedown",
+ lambda: (_ for _ in ()).throw(subprocess.CalledProcessError(1, ["git"])),
+ )
+
+ assert treatments.main(["shakedown"]) == 2
+ assert "error:" in capsys.readouterr().err
+
+
+def test_direct_script_entrypoint_smoke():
+ completed = subprocess.run(
+ [
+ sys.executable,
+ str(treatments.RUNNER_PATH),
+ "--help",
+ ],
+ cwd=treatments.REPOSITORY_ROOT,
+ check=False,
+ capture_output=True,
+ text=True,
+ )
+
+ assert completed.returncode == 0
+ assert "create the immutable 24-run plan" in completed.stdout
diff --git a/tests/test_book_replay.py b/tests/test_book_replay.py
new file mode 100644
index 0000000..065787b
--- /dev/null
+++ b/tests/test_book_replay.py
@@ -0,0 +1,518 @@
+import io
+import json
+import sys
+from decimal import Decimal, localcontext
+from pathlib import Path
+
+import pytest
+
+from tracebook.book_replay import (
+ PROFILE_NAME,
+ PROTOCOL_NAME,
+ PROTOCOL_VERSION,
+ BookReplayConfig,
+ BookReplayError,
+ BookReplayEvent,
+ BookReplayObservation,
+ BookReplayProtocolError,
+ BookReplaySnapshot,
+ BookReplayState,
+ EngineMetadata,
+ ExternalBookReplayAdapter,
+ ReferenceBookReplayAdapter,
+ run_book_replay,
+ serve_book_replay_stdio,
+)
+from tracebook.book_replay.cli import main
+from tracebook.book_replay.model import Outcome, SimulatedFill
+
+ROOT = Path(__file__).parents[1]
+PROFILE_TRACE = ROOT / "src" / "tracebook" / "book_replay" / "fixtures" / f"{PROFILE_NAME}.jsonl"
+EXAMPLE_ADAPTER = ROOT / "examples" / "book_replay_adapter.py"
+
+
+def _serve_test(factory, *messages, hello_overrides=None):
+ hello = dict(type="hello", protocol=PROTOCOL_NAME, protocol_version=PROTOCOL_VERSION)
+ hello.update(hello_overrides or {})
+ source = io.StringIO("".join(json.dumps(item) + "\n" for item in (hello, *messages)))
+ sink = io.StringIO()
+ status = serve_book_replay_stdio(factory, source, sink)
+ return status, [json.loads(line) for line in sink.getvalue().splitlines()]
+
+
+@pytest.mark.parametrize("exception", [ValueError, BrokenPipeError])
+def test_server_close_failure_is_one_terminal_adapter_error(exception):
+ instances = []
+
+ class BrokenClose(ReferenceBookReplayAdapter):
+ def __init__(self, config):
+ super().__init__(config)
+ self.close_calls = 0
+ instances.append(self)
+
+ def close(self):
+ self.close_calls += 1
+ raise exception("shutdown failed")
+
+ status, messages = _serve_test(BrokenClose, dict(type="finish", event_count=0))
+ assert status == 2
+ assert [item["type"] for item in messages] == ["ready", "error"]
+ assert messages[-1]["code"] == "ADAPTER_ERROR"
+ assert instances[0].close_calls == 1
+
+
+@pytest.mark.parametrize("stage", ["factory", "apply", "snapshot"])
+@pytest.mark.parametrize("exception", [TypeError, ValueError, BrokenPipeError])
+def test_server_adapter_exceptions_are_not_client_errors(stage, exception):
+ closed = []
+
+ class BrokenAdapter(ReferenceBookReplayAdapter):
+ def __init__(self, config):
+ if stage == "factory":
+ raise exception("adapter bug")
+ super().__init__(config)
+
+ def apply(self, event, index):
+ if stage == "apply":
+ raise exception("adapter bug")
+ return super().apply(event, index)
+
+ def snapshot(self):
+ raise exception("adapter bug")
+
+ def close(self):
+ closed.append(True)
+
+ request = (
+ dict(type="event", index=1, event=_event().to_dict())
+ if stage == "apply"
+ else dict(type="snapshot", index=0)
+ )
+ status, messages = _serve_test(BrokenAdapter, request)
+ assert status == 2
+ assert messages[-1]["code"] == "ADAPTER_ERROR"
+ assert len(closed) == (0 if stage == "factory" else 1)
+
+
+def test_server_broken_output_pipe_returns_failure_and_closes_adapter_once():
+ closed = []
+
+ class Adapter(ReferenceBookReplayAdapter):
+ def close(self):
+ closed.append(True)
+
+ class BrokenSink(io.StringIO):
+ def write(self, value):
+ raise BrokenPipeError("output pipe closed")
+
+ hello = dict(type="hello", protocol=PROTOCOL_NAME, protocol_version=PROTOCOL_VERSION)
+ source = io.StringIO(json.dumps(hello) + "\n")
+ assert serve_book_replay_stdio(Adapter, source, BrokenSink()) == 2
+ assert closed == [True]
+
+
+@pytest.mark.parametrize("kind,field", [("snapshot", "index"), ("finish", "event_count")])
+@pytest.mark.parametrize("value", [None, False, True, 0.0, "0", -1, "missing"])
+def test_server_requires_integer_session_counters(kind, field, value):
+ request = {"type": kind}
+ if value != "missing":
+ request[field] = value
+ status, messages = _serve_test(ReferenceBookReplayAdapter, request)
+ assert status == 2
+ assert [item["type"] for item in messages] == ["ready", "error"]
+ assert messages[-1]["code"] == "PROTOCOL_ERROR"
+
+
+def test_server_invalid_payload_is_a_client_error():
+ status, messages = _serve_test(
+ ReferenceBookReplayAdapter, dict(type="event", index=1, event=None)
+ )
+ assert status == 2
+ assert messages[-1]["code"] == "PROTOCOL_ERROR"
+ status, messages = _serve_test(ReferenceBookReplayAdapter, hello_overrides={"config": None})
+ assert status == 2
+ assert messages[0]["code"] == "PROTOCOL_ERROR"
+
+
+@pytest.mark.parametrize("field,value", [("outcome", {}), ("fills", (object(),)), ("fills", None)])
+def test_invalid_in_process_observation_is_classified_as_protocol_failure(field, value):
+ class Malformed(ReferenceBookReplayAdapter):
+ def apply(self, event, index):
+ fields = dict(
+ index=index,
+ outcome=Outcome("applied"),
+ fills=(),
+ state_hash=self.snapshot().digest(),
+ order_count=0,
+ )
+ fields[field] = value
+ return BookReplayObservation(**fields)
+
+ report = run_book_replay([_event()], Malformed)
+ assert report.operational_failure
+ assert report.divergence.category == "protocol"
+ assert report.divergence.kind == "adapter_error"
+
+
+@pytest.mark.parametrize("precision", [3, 28, 60])
+def test_probe_subtraction_and_canonical_values_are_exact_at_any_context_precision(precision):
+ tiny = "0.0000000000000000000000000000000000000001"
+ leftover = "0." + "9" * 40
+ with localcontext() as context:
+ context.prec = precision
+ adapter = ReferenceBookReplayAdapter(BookReplayConfig())
+ adapter.apply(_event(order_id=1, quantity=tiny), 1)
+ adapter.apply(_event(order_id=2, quantity="1"), 2)
+ before = adapter.snapshot()
+ observation = adapter.apply(_event("probe", quantity="1"), 3)
+ assert observation.fills == (SimulatedFill("100", tiny), SimulatedFill("100", leftover))
+ assert adapter.snapshot() == before
+ with localcontext() as context:
+ context.prec = 100
+ assert sum(Decimal(fill.quantity) for fill in observation.fills) == Decimal(1)
+
+
+def test_external_timeout_overflow_is_a_public_validation_error():
+ with pytest.raises(BookReplayError, match="positive finite"):
+ ExternalBookReplayAdapter(["never-started"], BookReplayConfig(), timeout_seconds=10**400)
+
+
+def test_external_invalid_metadata_is_a_protocol_error_and_child_is_reaped(monkeypatch):
+ import subprocess
+
+ processes = []
+ popen = subprocess.Popen
+
+ def capture_process(*args, **kwargs):
+ process = popen(*args, **kwargs)
+ processes.append(process)
+ return process
+
+ monkeypatch.setattr(subprocess, "Popen", capture_process)
+ script = (
+ "import json,sys; hello=json.loads(sys.stdin.readline()); "
+ "hello.update(type='ready',engine={}); print(json.dumps(hello),flush=True); "
+ "sys.stdin.read()"
+ )
+ with pytest.raises(BookReplayProtocolError, match="invalid ready engine metadata"):
+ ExternalBookReplayAdapter([sys.executable, "-c", script], BookReplayConfig())
+ assert len(processes) == 1
+ assert processes[0].poll() is not None
+
+
+def _event(op="add", **overrides):
+ values = {
+ "op": op,
+ "symbol": "TEST",
+ "order_id": 1,
+ "side": "BUY",
+ "price": "100",
+ "quantity": "2",
+ }
+ if op == "delete":
+ values = {"op": op, "symbol": "TEST", "order_id": 1}
+ elif op == "probe":
+ values = {
+ "op": op,
+ "symbol": "TEST",
+ "side": "SELL",
+ "price": "100",
+ "quantity": "2",
+ }
+ values.update(overrides)
+ return BookReplayEvent.from_mapping(values)
+
+
+def test_event_contract_is_canonical_and_strict():
+ event = _event(price="100.00", quantity=2.500)
+
+ assert event.to_dict() == {
+ "op": "add",
+ "symbol": "TEST",
+ "order_id": 1,
+ "side": "BUY",
+ "price": "100",
+ "quantity": "2.5",
+ }
+ with pytest.raises(BookReplayError, match="unsupported fields"):
+ BookReplayEvent.from_mapping({**event.to_dict(), "owner": 9})
+ with pytest.raises(BookReplayError, match="contain only"):
+ BookReplayEvent("delete", "TEST", 1, "BUY")
+ with pytest.raises(BookReplayError, match="must be positive"):
+ _event(quantity="0")
+
+
+def test_same_level_size_increase_retains_fifo_and_probe_does_not_mutate():
+ adapter = ReferenceBookReplayAdapter(BookReplayConfig())
+ adapter.apply(_event(order_id=1, quantity="2"), 1)
+ adapter.apply(_event(order_id=2, quantity="3"), 2)
+ updated = adapter.apply(_event("update", order_id=1, quantity="4"), 3)
+ before_probe = adapter.snapshot()
+
+ observation = adapter.apply(_event("probe", quantity="6"), 4)
+
+ assert [order.order_id for order in before_probe.books[0].bids] == [1, 2]
+ assert [fill.to_dict() for fill in observation.fills] == [
+ {"price": "100", "quantity": "4"},
+ {"price": "100", "quantity": "2"},
+ ]
+ assert adapter.snapshot() == before_probe
+ assert observation.state_hash == updated.state_hash
+
+
+def test_price_change_requeues_and_delete_is_keyed_by_order_id():
+ adapter = ReferenceBookReplayAdapter(BookReplayConfig())
+ adapter.apply(_event(order_id=1, price="99"), 1)
+ adapter.apply(_event(order_id=2, price="99"), 2)
+ adapter.apply(_event("update", order_id=1, price="100", quantity="2"), 3)
+ adapter.apply(_event("update", order_id=1, price="99", quantity="2"), 4)
+
+ assert [order.order_id for order in adapter.snapshot().books[0].bids] == [2, 1]
+ deleted = adapter.apply(_event("delete", order_id=2), 5)
+ rejected = adapter.apply(_event("delete", order_id=999), 6)
+ assert deleted.outcome.status == "applied"
+ assert rejected.outcome.to_dict()["reason"] == "ORDER_NOT_ACTIVE"
+
+
+def test_probe_obeys_limit_price_and_price_time_order():
+ adapter = ReferenceBookReplayAdapter(BookReplayConfig())
+ events = [
+ _event(order_id=10, side="SELL", price="102", quantity="5"),
+ _event(order_id=11, side="SELL", price="101", quantity="1"),
+ _event(order_id=12, side="SELL", price="101", quantity="2"),
+ ]
+ for index, event in enumerate(events, 1):
+ adapter.apply(event, index)
+
+ observation = adapter.apply(
+ _event("probe", side="BUY", price="101", quantity="5"),
+ 4,
+ )
+
+ assert [fill.to_dict() for fill in observation.fills] == [
+ {"price": "101", "quantity": "1"},
+ {"price": "101", "quantity": "2"},
+ ]
+ assert observation.order_count == 3
+
+
+class _ReorderingAdapter:
+ def __init__(self, config):
+ self._inner = ReferenceBookReplayAdapter(config)
+ self.metadata = EngineMetadata("reordering-test-adapter", "1", "Python")
+ self._reorder = False
+
+ def apply(self, event, index):
+ observation = self._inner.apply(event, index)
+ if event.op == "update" and event.order_id == 1:
+ self._reorder = True
+ state = self.snapshot()
+ return BookReplayObservation(
+ index,
+ observation.outcome,
+ observation.fills,
+ state.digest(),
+ state.order_count,
+ )
+
+ def snapshot(self):
+ state = self._inner.snapshot()
+ if not self._reorder:
+ return state
+ books = []
+ for book in state.books:
+ bids = list(book.bids)
+ if book.symbol == "TEST" and len(bids) >= 2:
+ bids[0], bids[1] = bids[1], bids[0]
+ books.append(BookReplaySnapshot(book.symbol, tuple(bids), book.asks))
+ return BookReplayState(tuple(books))
+
+ def close(self):
+ self._inner.close()
+
+
+def test_report_localizes_same_level_requeue_as_book_state_divergence():
+ events = [
+ _event(order_id=1),
+ _event(order_id=2),
+ _event("update", order_id=1, quantity="4"),
+ ]
+
+ report = run_book_replay(events, _ReorderingAdapter, trace_name="priority")
+
+ assert report.conformant is False
+ assert report.compared_events == 3
+ assert report.divergence is not None
+ assert report.divergence.category == "book_state"
+ assert report.divergence.path.endswith("order_id")
+ assert report.to_dict()["artifact_type"] == "tracebook.book-replay.report"
+ assert report.to_dict()["config"] == {"profile": PROFILE_NAME}
+
+
+def test_reference_candidate_conforms_on_bundled_profile():
+ events = [
+ BookReplayEvent.from_mapping(json.loads(line))
+ for line in PROFILE_TRACE.read_text(encoding="utf-8").splitlines()
+ ]
+
+ report = run_book_replay(events, ReferenceBookReplayAdapter, trace_name=PROFILE_NAME)
+
+ assert report.conformant is True
+ assert report.event_count == 17
+ assert report.compared_events == 17
+ assert (
+ report.final_state_hash
+ == "9e0af6be935dce940a87497788c7a9a799c71f05e34a0204c9d294fce611b002"
+ )
+
+
+def test_stdio_transcript_uses_separate_protocol_identity():
+ event = _event()
+ messages = [
+ {
+ "type": "hello",
+ "protocol": PROTOCOL_NAME,
+ "protocol_version": PROTOCOL_VERSION,
+ "config": BookReplayConfig().to_dict(),
+ },
+ {"type": "event", "index": 1, "event": event.to_dict()},
+ {"type": "snapshot", "index": 1},
+ {"type": "finish", "event_count": 1},
+ ]
+ source = io.StringIO("".join(json.dumps(message) + "\n" for message in messages))
+ sink = io.StringIO()
+
+ assert serve_book_replay_stdio(ReferenceBookReplayAdapter, source, sink) == 0
+ responses = [json.loads(line) for line in sink.getvalue().splitlines()]
+ assert [response["type"] for response in responses] == [
+ "ready",
+ "observation",
+ "snapshot",
+ "complete",
+ ]
+ assert responses[0]["protocol"] == "tracebook.book-replay"
+ assert responses[2]["state"]["books"][0]["bids"][0]["order_id"] == 1
+
+
+def test_external_example_adapter_conforms():
+ events = [_event(order_id=1), _event("probe", quantity="1")]
+
+ report = run_book_replay(
+ events,
+ lambda config: ExternalBookReplayAdapter(
+ [sys.executable, str(EXAMPLE_ADAPTER)],
+ config,
+ timeout_seconds=10,
+ ),
+ )
+
+ assert report.conformant is True
+ assert report.candidate_engine.name == "example-book-replay-adapter"
+
+
+def _shutdown_test_adapter(delay, exit_code, *, handle_terminate=False):
+ script = """
+import json
+import signal
+import sys
+import time
+
+delay, exit_code, handle_terminate = json.loads(sys.argv[1])
+if handle_terminate:
+ signal.signal(signal.SIGTERM, lambda *_: sys.exit(0))
+hello = json.loads(sys.stdin.readline())
+print(json.dumps({
+ "type": "ready",
+ "protocol": hello["protocol"],
+ "protocol_version": hello["protocol_version"],
+ "engine": {"name": "shutdown-test", "version": "1", "language": "Python"},
+}), flush=True)
+finish = json.loads(sys.stdin.readline())
+print(json.dumps({"type": "complete", "event_count": finish["event_count"]}), flush=True)
+time.sleep(delay)
+sys.exit(exit_code)
+"""
+ return ExternalBookReplayAdapter(
+ [
+ sys.executable,
+ "-I",
+ "-u",
+ "-c",
+ script,
+ json.dumps([delay, exit_code, handle_terminate]),
+ ],
+ BookReplayConfig(),
+ timeout_seconds=10,
+ )
+
+
+def test_external_close_allows_configured_time_for_clean_exit():
+ adapter = _shutdown_test_adapter(0.75, 0)
+
+ adapter.close()
+ adapter.close()
+
+ assert adapter._process.returncode == 0
+
+
+def test_external_close_rejects_nonzero_exit_after_complete():
+ adapter = _shutdown_test_adapter(0, 7)
+
+ with pytest.raises(BookReplayProtocolError, match="exited with code 7 after complete"):
+ adapter.close()
+
+
+def test_external_close_rejects_timeout_even_when_termination_exits_zero():
+ adapter = _shutdown_test_adapter(60, 0, handle_terminate=True)
+ # Startup has its own generous budget; now isolate the exit timeout.
+ adapter.timeout_seconds = 0.2
+
+ with pytest.raises(BookReplayProtocolError, match="timed out exiting after complete"):
+ adapter.close()
+
+ assert adapter._process.returncode is not None
+
+
+def test_cli_sample_and_external_run(tmp_path, capsys):
+ sample_dir = tmp_path / "sample"
+ assert main(["sample", str(sample_dir)]) == 0
+ copied = sample_dir / f"{PROFILE_NAME}.jsonl"
+ assert copied.read_bytes() == PROFILE_TRACE.read_bytes()
+
+ output = tmp_path / "report.json"
+ assert (
+ main(
+ [
+ "run",
+ str(copied),
+ "--output",
+ str(output),
+ "--candidate-cmd",
+ f"{sys.executable} {EXAMPLE_ADAPTER}",
+ ]
+ )
+ == 0
+ )
+ payload = json.loads(output.read_text(encoding="utf-8"))
+ assert payload["conformant"] is True
+ assert payload["artifact_type"] == "tracebook.book-replay.report"
+ assert "Book-replay trace copied" in capsys.readouterr().out
+
+
+def test_cli_rejects_overwriting_input(tmp_path, capsys):
+ trace = tmp_path / "trace.jsonl"
+ trace.write_text(json.dumps(_event().to_dict()) + "\n", encoding="utf-8")
+
+ exit_code = main(
+ [
+ "run",
+ str(trace),
+ "--output",
+ str(trace),
+ "--candidate-cmd",
+ f"{sys.executable} {EXAMPLE_ADAPTER}",
+ ]
+ )
+
+ assert exit_code == 2
+ assert "input and output paths must be distinct" in capsys.readouterr().err
diff --git a/tests/test_book_replay_artifacts.py b/tests/test_book_replay_artifacts.py
new file mode 100644
index 0000000..1461a8c
--- /dev/null
+++ b/tests/test_book_replay_artifacts.py
@@ -0,0 +1,640 @@
+"""L3 output failures must not truncate prior evidence or run invalid targets."""
+
+from __future__ import annotations
+
+import json
+import os
+from pathlib import Path
+from types import SimpleNamespace
+
+import pytest
+
+from tracebook.book_replay import BookReplayEvent, ReferenceBookReplayAdapter
+from tracebook.book_replay import artifacts, cli
+
+EVENT = BookReplayEvent("add", "TEST", 1, "BUY", "100", "2")
+PAYLOAD = {"artifact_type": "tracebook.book-replay.report", "conformant": True}
+
+
+def _trace(path: Path) -> None:
+ path.write_text(json.dumps(EVENT.to_dict()) + "\n", encoding="utf-8")
+
+
+def _command(command: str, trace: Path, report: Path, reduced: Path) -> list[str]:
+ arguments = [command]
+ if command != "campaign":
+ arguments.append(str(trace))
+ arguments.extend(("--output", str(report)))
+ if command == "campaign":
+ arguments.extend(("--reduced-events-output", str(reduced)))
+ elif command == "minimize":
+ arguments.extend(("--events-output", str(reduced)))
+ arguments.extend(("--candidate-cmd", "unused-adapter"))
+ return arguments
+
+
+def _forbid_candidate(monkeypatch) -> None:
+ def fail(_args):
+ pytest.fail("candidate factory reached despite invalid output paths")
+
+ monkeypatch.setattr(cli, "_candidate_factory", fail)
+
+
+def _stub_result(monkeypatch, command: str) -> None:
+ monkeypatch.setattr(cli, "_candidate_factory", lambda _args: object())
+ report = SimpleNamespace(conformant=True, operational_failure=False, to_dict=lambda: PAYLOAD)
+ minimization = SimpleNamespace(events=[EVENT], report=report, to_dict=lambda: PAYLOAD)
+ result = {
+ "run": report,
+ "minimize": minimization,
+ "campaign": SimpleNamespace(
+ failure=SimpleNamespace(minimization=minimization), to_dict=lambda: PAYLOAD
+ ),
+ }[command]
+ function = {
+ "run": "run_book_replay",
+ "minimize": "minimize_book_replay_failure",
+ "campaign": "run_book_replay_campaign",
+ }[command]
+ monkeypatch.setattr(cli, function, lambda *_args, **_kwargs: result)
+
+
+@pytest.mark.parametrize("command", ("run", "minimize", "campaign"))
+@pytest.mark.parametrize("existing_kind", ("file", "directory", "symlink", "dangling-symlink"))
+def test_cli_refuses_existing_report_before_candidate(
+ tmp_path, monkeypatch, command, existing_kind
+):
+ trace, report, reduced = (
+ tmp_path / name for name in ("trace.jsonl", "report.json", "reduced.jsonl")
+ )
+ _trace(trace)
+ if existing_kind == "file":
+ report.write_bytes(b"prior evidence\n")
+ elif existing_kind == "directory":
+ report.mkdir()
+ else:
+ report.symlink_to(tmp_path / "prior.json")
+ if existing_kind == "symlink":
+ (tmp_path / "prior.json").write_bytes(b"prior evidence\n")
+ before = set(tmp_path.iterdir())
+ _forbid_candidate(monkeypatch)
+
+ assert cli.main(_command(command, trace, report, reduced)) == 2
+
+ assert set(tmp_path.iterdir()) == before
+ if existing_kind == "file":
+ assert report.read_bytes() == b"prior evidence\n"
+ if existing_kind == "symlink":
+ assert report.is_symlink()
+ assert report.read_bytes() == b"prior evidence\n"
+ if existing_kind == "dangling-symlink":
+ assert report.is_symlink()
+ assert not (tmp_path / "prior.json").exists()
+
+
+@pytest.mark.parametrize("command", ("minimize", "campaign"))
+def test_cli_reserves_conditional_reduced_output_before_candidate(tmp_path, monkeypatch, command):
+ trace, report, reduced = (
+ tmp_path / name for name in ("trace.jsonl", "report.json", "reduced.jsonl")
+ )
+ _trace(trace)
+ reduced.write_bytes(b"prior reduced trace\n")
+ _forbid_candidate(monkeypatch)
+
+ assert cli.main(_command(command, trace, report, reduced)) == 2
+
+ assert reduced.read_bytes() == b"prior reduced trace\n"
+ assert set(tmp_path.iterdir()) == {trace, reduced}
+
+
+@pytest.mark.parametrize("command", ("minimize", "campaign"))
+@pytest.mark.parametrize(
+ "collision", ("same", "nested", "reverse-nested", "sidecar", "reverse-sidecar")
+)
+def test_cli_rejects_colliding_output_paths_before_candidate(
+ tmp_path, monkeypatch, command, collision
+):
+ trace, report = tmp_path / "trace.jsonl", tmp_path / "report.json"
+ _trace(trace)
+ reduced = {
+ "same": tmp_path / "unused" / ".." / "report.json",
+ "nested": report / "reduced.jsonl",
+ "reverse-nested": tmp_path,
+ "sidecar": tmp_path / ".report.json.tracebook-in-progress",
+ "reverse-sidecar": report,
+ }[collision]
+ if collision == "reverse-sidecar":
+ report = tmp_path / ".report.json.tracebook-in-progress"
+ _forbid_candidate(monkeypatch)
+
+ assert cli.main(_command(command, trace, report, reduced)) == 2
+
+ assert set(tmp_path.iterdir()) == {trace}
+
+
+@pytest.mark.parametrize("command", ("run", "minimize", "campaign"))
+def test_cli_rejects_non_directory_parent_before_candidate(tmp_path, monkeypatch, command):
+ trace, parent = tmp_path / "trace.jsonl", tmp_path / "not-a-directory"
+ _trace(trace)
+ parent.write_bytes(b"keep this file\n")
+ _forbid_candidate(monkeypatch)
+
+ assert cli.main(_command(command, trace, parent / "report.json", tmp_path / "reduced")) == 2
+ assert parent.read_bytes() == b"keep this file\n"
+
+
+def test_cli_refuses_another_active_reservation_before_candidate(tmp_path, monkeypatch):
+ trace, report = tmp_path / "trace.jsonl", tmp_path / "report.json"
+ _trace(trace)
+ _forbid_candidate(monkeypatch)
+ with artifacts.reserve_outputs(str(report)):
+ before = set(tmp_path.iterdir())
+ assert cli.main(_command("run", trace, report, tmp_path / "unused")) == 2
+ assert set(tmp_path.iterdir()) == before
+ assert set(tmp_path.iterdir()) == {trace}
+
+
+@pytest.mark.parametrize("command", ("run", "minimize", "campaign", "sample"))
+def test_partial_stage_write_never_publishes_an_artifact(tmp_path, monkeypatch, command):
+ trace, report, reduced = (
+ tmp_path / name for name in ("trace.jsonl", "report.json", "reduced.jsonl")
+ )
+ _trace(trace)
+ if command != "sample":
+ _stub_result(monkeypatch, command)
+
+ def partial_write(descriptor, payload):
+ os.write(descriptor, payload[:5])
+ raise OSError("injected partial write failure")
+
+ monkeypatch.setattr(artifacts, "_write_payload", partial_write)
+ arguments = (
+ ["sample", str(tmp_path)]
+ if command == "sample"
+ else _command(command, trace, report, reduced)
+ )
+
+ assert cli.main(arguments) == 2
+ assert set(tmp_path.iterdir()) == {trace}
+
+
+@pytest.mark.parametrize("command", ("minimize", "campaign"))
+def test_second_publication_failure_rolls_back_first_file(tmp_path, monkeypatch, command):
+ trace, report, reduced = (
+ tmp_path / name for name in ("trace.jsonl", "report.json", "reduced.jsonl")
+ )
+ _trace(trace)
+ _stub_result(monkeypatch, command)
+ link = os.link
+ published = []
+
+ def fail_second(source, target, **kwargs):
+ published.append(target)
+ if len(published) == 2:
+ raise OSError("injected report publication failure")
+ link(source, target, **kwargs)
+
+ monkeypatch.setattr(artifacts.os, "link", fail_second)
+ # The runtime supports descriptor-relative links; the wrapper remains the
+ # same operation for this injected failure.
+ monkeypatch.setattr(artifacts.os, "supports_dir_fd", os.supports_dir_fd | {fail_second})
+
+ assert cli.main(_command(command, trace, report, reduced)) == 2
+ assert published == [reduced.name, report.name]
+ assert set(tmp_path.iterdir()) == {trace}
+
+
+@pytest.mark.parametrize("command", ["run", "sample"])
+def test_cli_symlink_loop_is_controlled_before_candidate(tmp_path, monkeypatch, capsys, command):
+ trace = tmp_path / "trace.jsonl"
+ _trace(trace)
+ loop = tmp_path / "loop"
+ loop.symlink_to(loop)
+ _forbid_candidate(monkeypatch)
+ arguments = (
+ ["sample", str(loop)]
+ if command == "sample"
+ else _command("run", trace, loop / "report.json", tmp_path / "unused")
+ )
+ assert cli.main(arguments) == 2
+ assert "Traceback" not in capsys.readouterr().err
+ assert set(tmp_path.iterdir()) == {trace, loop}
+
+
+@pytest.mark.parametrize("command", ["run", "minimize"])
+def test_cli_input_symlink_loop_is_controlled_before_candidate(
+ tmp_path, monkeypatch, capsys, command
+):
+ loop = tmp_path / "loop"
+ loop.symlink_to(loop)
+ _forbid_candidate(monkeypatch)
+
+ arguments = _command(command, loop, tmp_path / "report.json", tmp_path / "reduced.jsonl")
+ assert cli.main(arguments) == 2
+ error = capsys.readouterr().err
+ assert error.startswith("Error:")
+ assert "Traceback" not in error
+ assert set(tmp_path.iterdir()) == {loop}
+
+
+def test_late_destination_creation_is_never_overwritten(tmp_path, monkeypatch):
+ trace, report = tmp_path / "trace.jsonl", tmp_path / "report.json"
+ _trace(trace)
+ _stub_result(monkeypatch, "run")
+ link = os.link
+
+ def race_link(source, target, **kwargs):
+ report.write_bytes(b"another writer's evidence\n")
+ link(source, target, **kwargs)
+
+ monkeypatch.setattr(artifacts.os, "link", race_link)
+ monkeypatch.setattr(artifacts.os, "supports_dir_fd", os.supports_dir_fd | {race_link})
+
+ assert cli.main(_command("run", trace, report, tmp_path / "unused")) == 2
+ assert report.read_bytes() == b"another writer's evidence\n"
+ assert set(tmp_path.iterdir()) == {trace, report}
+
+
+def test_replaced_output_parent_cannot_redirect_publication(tmp_path, monkeypatch):
+ trace = tmp_path / "trace.jsonl"
+ output_parent, moved_parent, other_parent = (
+ tmp_path / name for name in ("out", "moved", "other")
+ )
+ _trace(trace)
+ other_parent.mkdir()
+ _stub_result(monkeypatch, "run")
+
+ def replace_parent(*_args, **_kwargs):
+ output_parent.rename(moved_parent)
+ output_parent.symlink_to(other_parent, target_is_directory=True)
+ return SimpleNamespace(to_dict=lambda: PAYLOAD, conformant=True, operational_failure=False)
+
+ monkeypatch.setattr(cli, "run_book_replay", replace_parent)
+
+ assert cli.main(_command("run", trace, output_parent / "report.json", tmp_path / "unused")) == 2
+ assert list(other_parent.iterdir()) == []
+ assert list(moved_parent.iterdir()) == []
+ assert output_parent.is_symlink()
+
+
+def test_retargeted_parent_alias_cannot_publish_at_an_invisible_old_path(tmp_path, monkeypatch):
+ trace = tmp_path / "trace.jsonl"
+ original, other, alias = (tmp_path / name for name in ("original", "other", "alias"))
+ _trace(trace)
+ original.mkdir()
+ other.mkdir()
+ alias.symlink_to(original, target_is_directory=True)
+ _stub_result(monkeypatch, "run")
+
+ def retarget_alias(*_args, **_kwargs):
+ alias.unlink()
+ alias.symlink_to(other, target_is_directory=True)
+ return SimpleNamespace(to_dict=lambda: PAYLOAD, conformant=True, operational_failure=False)
+
+ monkeypatch.setattr(cli, "run_book_replay", retarget_alias)
+
+ assert cli.main(_command("run", trace, alias / "report.json", tmp_path / "unused")) == 2
+ assert list(original.iterdir()) == []
+ assert list(other.iterdir()) == []
+ assert alias.resolve() == other
+
+
+def test_replaced_reduced_file_invalidates_and_rolls_back_the_report(tmp_path, monkeypatch):
+ trace, report, reduced = (
+ tmp_path / name for name in ("trace.jsonl", "report.json", "reduced.jsonl")
+ )
+ _trace(trace)
+ _stub_result(monkeypatch, "minimize")
+ link = os.link
+
+ def replace_reduced_before_report(source, target, **kwargs):
+ if target == report.name:
+ reduced.unlink()
+ reduced.write_bytes(b"another writer's replacement\n")
+ link(source, target, **kwargs)
+
+ monkeypatch.setattr(artifacts.os, "link", replace_reduced_before_report)
+ monkeypatch.setattr(
+ artifacts.os, "supports_dir_fd", os.supports_dir_fd | {replace_reduced_before_report}
+ )
+
+ assert cli.main(_command("minimize", trace, report, reduced)) == 2
+ assert not report.exists()
+ assert reduced.read_bytes() == b"another writer's replacement\n"
+ assert set(tmp_path.iterdir()) == {trace, reduced}
+
+
+def test_interruption_after_report_link_rolls_back_both_published_files(tmp_path, monkeypatch):
+ trace, report, reduced = (
+ tmp_path / name for name in ("trace.jsonl", "report.json", "reduced.jsonl")
+ )
+ _trace(trace)
+ _stub_result(monkeypatch, "minimize")
+ link = os.link
+
+ def interrupt_after_link(source, target, **kwargs):
+ link(source, target, **kwargs)
+ if target == report.name:
+ raise KeyboardInterrupt
+
+ monkeypatch.setattr(artifacts.os, "link", interrupt_after_link)
+ monkeypatch.setattr(
+ artifacts.os, "supports_dir_fd", os.supports_dir_fd | {interrupt_after_link}
+ )
+
+ with pytest.raises(KeyboardInterrupt):
+ cli.main(_command("minimize", trace, report, reduced))
+ assert set(tmp_path.iterdir()) == {trace}
+
+
+@pytest.mark.skipif(os.geteuid() == 0, reason="root can deliberately bypass directory permissions")
+def test_stage_entry_cannot_be_replaced_between_validation_and_link(tmp_path, monkeypatch):
+ report = tmp_path / "report.json"
+ link = os.link
+
+ def replace_stage(source, target, **kwargs):
+ # The old implementation allowed this unlink, then linked the foreign
+ # replacement and left it at report.json after its failed validation.
+ os.unlink(source, dir_fd=kwargs["src_dir_fd"])
+ descriptor = os.open(
+ source, os.O_WRONLY | os.O_CREAT | os.O_EXCL, dir_fd=kwargs["src_dir_fd"]
+ )
+ os.write(descriptor, b"foreign bytes")
+ os.close(descriptor)
+ link(source, target, **kwargs)
+
+ monkeypatch.setattr(artifacts.os, "link", replace_stage)
+ monkeypatch.setattr(artifacts.os, "supports_dir_fd", os.supports_dir_fd | {replace_stage})
+ with artifacts.reserve_outputs(str(report)) as outputs:
+ with pytest.raises(PermissionError):
+ artifacts.publish_outputs([(outputs[str(report)], b"owned bytes")])
+ assert not report.exists()
+ assert list(tmp_path.iterdir()) == []
+
+
+def test_replacing_stage_directory_path_cannot_change_published_inode(tmp_path, monkeypatch):
+ report = tmp_path / "report.json"
+ link = os.link
+ with artifacts.reserve_outputs(str(report)) as outputs:
+ output = outputs[str(report)]
+ stage = tmp_path / output._stage_directory_name
+ moved = tmp_path / "moved-stage"
+
+ def replace_directory(source, target, **kwargs):
+ # macOS requires write permission on a directory being renamed;
+ # emulate a writer that can rename it, then restore its protection.
+ stage.chmod(0o700)
+ stage.rename(moved)
+ moved.chmod(0o500)
+ stage.mkdir()
+ (stage / source).write_bytes(b"foreign replacement")
+ link(source, target, **kwargs)
+
+ monkeypatch.setattr(artifacts.os, "link", replace_directory)
+ artifacts.publish_outputs([(output, b"owned bytes")])
+ assert report.read_bytes() == b"owned bytes"
+ assert (stage / output._stage_name).read_bytes() == b"foreign replacement"
+ assert list(moved.iterdir()) == []
+
+
+@pytest.mark.parametrize(
+ "replace_directory,fail_open", [(False, True), (True, True), (True, False)]
+)
+def test_stage_directory_setup_failure_cleans_only_owned_entries(
+ tmp_path, monkeypatch, replace_directory, fail_open
+):
+ reservation = artifacts.OutputReservation(tmp_path / "report.json")
+ stage = tmp_path / reservation._stage_directory_name
+ moved = tmp_path / "moved-stage"
+ open_descriptor = os.open
+ descriptors = []
+
+ def inject_stage_open(path, *args, **kwargs):
+ if path == reservation._stage_directory_name:
+ if replace_directory:
+ stage.rename(moved)
+ stage.mkdir(mode=0o500)
+ if fail_open:
+ raise OSError("injected stage directory open failure")
+ descriptor = open_descriptor(path, *args, **kwargs)
+ descriptors.append(descriptor)
+ return descriptor
+
+ monkeypatch.setattr(artifacts.os, "open", inject_stage_open)
+ monkeypatch.setattr(artifacts.os, "supports_dir_fd", os.supports_dir_fd | {inject_stage_open})
+ error = OSError if fail_open else artifacts.BookReplayError
+ with pytest.raises(error, match="open failure" if fail_open else "reservation changed"):
+ reservation.__enter__()
+ for descriptor in descriptors:
+ with pytest.raises(OSError):
+ os.fstat(descriptor)
+ assert set(tmp_path.iterdir()) == ({stage, moved} if replace_directory else set())
+ if replace_directory:
+ assert list(stage.iterdir()) == list(moved.iterdir()) == []
+ assert stage.stat().st_mode & 0o777 == 0o500
+
+
+@pytest.mark.parametrize("replacement", ["none", "empty", "populated", "symlink", "missing"])
+def test_initial_stage_stat_failure_reports_unverified_entry_without_removing_it(
+ tmp_path, monkeypatch, replacement
+):
+ reservation = artifacts.OutputReservation(tmp_path / "report.json")
+ stage = tmp_path / reservation._stage_directory_name
+ moved = tmp_path / "moved-stage"
+ stat, open_descriptor = os.stat, os.open
+ descriptors = []
+ failure = OSError("injected initial staging stat failure")
+
+ def fail_stage_stat(path, *args, **kwargs):
+ if path == reservation._stage_directory_name:
+ if replacement != "none":
+ stage.rename(moved)
+ if replacement in ("empty", "populated"):
+ stage.mkdir()
+ if replacement == "populated":
+ (stage / "foreign").write_bytes(b"another writer's evidence")
+ stage.chmod(0o500)
+ elif replacement == "symlink":
+ stage.symlink_to(moved, target_is_directory=True)
+ raise failure
+ return stat(path, *args, **kwargs)
+
+ def track_open(*args, **kwargs):
+ descriptor = open_descriptor(*args, **kwargs)
+ descriptors.append(descriptor)
+ return descriptor
+
+ monkeypatch.setattr(artifacts.os, "stat", fail_stage_stat)
+ monkeypatch.setattr(artifacts.os, "open", track_open)
+ monkeypatch.setattr(
+ artifacts.os, "supports_dir_fd", os.supports_dir_fd | {fail_stage_stat, track_open}
+ )
+ monkeypatch.setattr(
+ artifacts.os, "rmdir", lambda *_args, **_kwargs: pytest.fail("removing unverified entry")
+ )
+ monkeypatch.setattr(
+ artifacts.os, "fchmod", lambda *_args: pytest.fail("changing unverified permissions")
+ )
+
+ with pytest.raises(
+ artifacts.BookReplayError, match="automatic directory cleanup skipped"
+ ) as caught:
+ reservation.__enter__()
+ assert caught.value.__cause__ is failure
+ assert repr(str(stage)) in str(caught.value)
+ assert "possible staging residue" in str(caught.value)
+ assert "ownership could not be verified" in str(caught.value)
+ assert len(descriptors) == 2 # Only the parent and lock were opened.
+ for descriptor in descriptors:
+ with pytest.raises(OSError):
+ os.fstat(descriptor)
+ expected = {stage} if replacement == "none" else {moved}
+ if replacement not in ("none", "missing"):
+ expected.add(stage)
+ assert set(tmp_path.iterdir()) == expected
+ if replacement in ("empty", "populated"):
+ assert stage.stat().st_mode & 0o777 == 0o500
+ if replacement == "populated":
+ assert (stage / "foreign").read_bytes() == b"another writer's evidence"
+ elif replacement == "symlink":
+ assert stage.is_symlink() and stage.readlink() == moved
+ if moved.exists():
+ assert list(moved.iterdir()) == []
+
+
+@pytest.mark.parametrize("command", ["run", "minimize", "campaign", "sample"])
+def test_cli_reports_initial_stage_stat_failure_before_candidate(
+ tmp_path, monkeypatch, capsys, command
+):
+ trace = tmp_path / "trace.jsonl"
+ _trace(trace)
+ _forbid_candidate(monkeypatch)
+ stat = os.stat
+ stages = []
+
+ def fail_stage_stat(path, *args, **kwargs):
+ if isinstance(path, str) and path.startswith(".tracebook-stage-"):
+ stages.append(path)
+ raise OSError("injected initial staging stat failure")
+ return stat(path, *args, **kwargs)
+
+ monkeypatch.setattr(artifacts.os, "stat", fail_stage_stat)
+ monkeypatch.setattr(artifacts.os, "supports_dir_fd", os.supports_dir_fd | {fail_stage_stat})
+ arguments = (
+ ["sample", str(tmp_path / "sample")]
+ if command == "sample"
+ else _command(command, trace, tmp_path / "report.json", tmp_path / "reduced.jsonl")
+ )
+ assert cli.main(arguments) == 2
+ output = capsys.readouterr()
+ assert output.out == ""
+ assert output.err.startswith("Error:") and "Traceback" not in output.err
+ assert "automatic directory cleanup skipped" in output.err
+ assert len(stages) == 1
+ parent = tmp_path / "sample" if command == "sample" else tmp_path
+ stage = parent / stages[0]
+ assert repr(str(stage)) in output.err
+ assert stage.is_dir() and list(stage.iterdir()) == []
+ if command == "sample":
+ assert set(parent.iterdir()) == {stage}
+ assert set(tmp_path.iterdir()) == {trace, parent}
+ else:
+ assert set(tmp_path.iterdir()) == {trace, stage}
+
+
+def test_cleanup_failure_still_closes_all_descriptors_and_other_sidecars(tmp_path, monkeypatch):
+ reservation = artifacts.OutputReservation(tmp_path / "report.json")
+ reservation.__enter__()
+ descriptors = (
+ reservation._stage_fd,
+ reservation._lock_fd,
+ reservation._stage_directory_fd,
+ reservation._parent_fd,
+ )
+ unlink = os.unlink
+
+ def fail_stage_cleanup(name, **kwargs):
+ if name == reservation._stage_name:
+ raise OSError("injected temporary file cleanup failure")
+ unlink(name, **kwargs)
+
+ monkeypatch.setattr(artifacts.os, "unlink", fail_stage_cleanup)
+
+ with pytest.raises(OSError, match="cleanup failure"):
+ reservation.close()
+ for descriptor in descriptors:
+ with pytest.raises(OSError):
+ os.fstat(descriptor)
+ assert {path.name for path in tmp_path.iterdir()} == {reservation._stage_directory_name}
+ assert (tmp_path / reservation._stage_directory_name / reservation._stage_name).is_file()
+
+
+def test_trace_serialization_failure_precedes_any_artifact_write(tmp_path, monkeypatch):
+ trace, report, reduced = (
+ tmp_path / name for name in ("trace.jsonl", "report.json", "reduced.jsonl")
+ )
+ _trace(trace)
+ _stub_result(monkeypatch, "minimize")
+
+ def bad_event():
+ raise ValueError("injected event serialization error")
+
+ result = SimpleNamespace(
+ to_dict=lambda: PAYLOAD,
+ events=[EVENT, SimpleNamespace(to_dict=bad_event)],
+ )
+ monkeypatch.setattr(cli, "minimize_book_replay_failure", lambda *_args, **_kwargs: result)
+ monkeypatch.setattr(
+ artifacts,
+ "_write_payload",
+ lambda *_args: pytest.fail("writing before serialization finished"),
+ )
+
+ assert cli.main(_command("minimize", trace, report, reduced)) == 2
+ assert set(tmp_path.iterdir()) == {trace}
+
+
+@pytest.mark.parametrize("command", ("run", "minimize", "campaign"))
+def test_successful_publication_preserves_existing_json_bytes(tmp_path, monkeypatch, command):
+ trace, report, reduced = (
+ tmp_path / name for name in ("trace.jsonl", "report.json", "reduced.jsonl")
+ )
+ _trace(trace)
+ _stub_result(monkeypatch, command)
+
+ assert cli.main(_command(command, trace, report, reduced)) == (0 if command == "run" else 1)
+
+ assert report.read_bytes() == (json.dumps(PAYLOAD, sort_keys=True, indent=2) + "\n").encode()
+ if command != "run":
+ assert (
+ reduced.read_bytes()
+ == (json.dumps(EVENT.to_dict(), sort_keys=True, separators=(",", ":")) + "\n").encode()
+ )
+ assert not list(tmp_path.glob(".*"))
+
+
+def test_passing_campaign_removes_unused_reduced_reservation(tmp_path, monkeypatch):
+ report, reduced = tmp_path / "campaign.json", tmp_path / "reduced.jsonl"
+ monkeypatch.setattr(cli, "_candidate_factory", lambda _args: ReferenceBookReplayAdapter)
+
+ assert (
+ cli.main(
+ [
+ "campaign",
+ "--output",
+ str(report),
+ "--reduced-events-output",
+ str(reduced),
+ "--traces",
+ "1",
+ "--events-per-trace",
+ "18",
+ "--candidate-cmd",
+ "unused",
+ ]
+ )
+ == 0
+ )
+
+ assert json.loads(report.read_text())["conformant"] is True
+ assert set(tmp_path.iterdir()) == {report}
diff --git a/tests/test_book_replay_campaign.py b/tests/test_book_replay_campaign.py
new file mode 100644
index 0000000..82a720f
--- /dev/null
+++ b/tests/test_book_replay_campaign.py
@@ -0,0 +1,234 @@
+import json
+import sys
+from pathlib import Path
+
+import pytest
+from tests.fixtures.faulty_book_replay_adapter import FaultyBookReplayAdapter
+
+from tracebook.book_replay import (
+ BOOK_REPLAY_CAPABILITIES,
+ BookReplayError,
+ BookReplayEvent,
+ ReferenceBookReplayAdapter,
+ generate_book_replay_trace,
+ measure_book_replay_coverage,
+ minimize_book_replay_failure,
+ run_book_replay_campaign,
+)
+from tracebook.book_replay.cli import main
+from tracebook.book_replay.model import trace_sha256
+
+ROOT = Path(__file__).parents[1]
+FAULTY_ADAPTER = ROOT / "tests" / "fixtures" / "faulty_book_replay_adapter.py"
+
+
+class _ReorderingAdapter(FaultyBookReplayAdapter):
+ def __init__(self, config):
+ super().__init__(config, engine_name="reordering-test-adapter")
+
+
+def test_side_only_update_does_not_claim_price_relocation_coverage():
+ add = BookReplayEvent("add", "TEST", 1, "BUY", "100", "1")
+ side_change = BookReplayEvent("update", "TEST", 1, "SELL", "100", "1")
+ price_change = BookReplayEvent("update", "TEST", 1, "SELL", "101", "1")
+ assert "price-relocation" not in measure_book_replay_coverage([[add, side_change]]).covered
+ assert "price-relocation" in measure_book_replay_coverage([[add, price_change]]).covered
+
+
+def test_generator_is_deterministic_and_scaffold_covers_every_capability():
+ first = generate_book_replay_trace(123, 40)
+ repeated = generate_book_replay_trace(123, 40)
+ different = generate_book_replay_trace(124, 40)
+
+ assert first == repeated
+ assert first != different
+ assert len(first) == 40
+ assert trace_sha256(first) == (
+ "4199853cf0496a08beb964283932a6880bd23e534a5e1f72952a4d2d616d0fad"
+ )
+ assert trace_sha256(first[:18]) == (
+ "25180672acc45a7487cf3f87abcdf9db5426bdefa82a0d2a00efb259581e41ee"
+ )
+ coverage = measure_book_replay_coverage([first[:18]])
+ assert coverage.complete is True
+ assert coverage.covered == BOOK_REPLAY_CAPABILITIES
+
+
+def test_reference_campaign_is_reproducible_and_complete():
+ result = run_book_replay_campaign(
+ ReferenceBookReplayAdapter,
+ seed=20260831,
+ traces=3,
+ events_per_trace=60,
+ )
+
+ assert result.conformant is True
+ assert result.campaign_id == (
+ "sha256:ae989f8ddd45385471c3e6b4b7736ec89410947bd028aadfc1c46f9e294aa876"
+ )
+ assert result.coverage.complete is True
+ assert result.failure_id is None
+ assert [trace.seed for trace in result.traces] == [
+ 10864503699890266417,
+ 10644828502866079881,
+ 3445208458078199137,
+ ]
+
+
+def test_minimizer_produces_one_minimal_priority_failure():
+ events = generate_book_replay_trace(123, 40)
+
+ result = minimize_book_replay_failure(events, _ReorderingAdapter)
+
+ assert [event.to_dict() for event in result.events] == [
+ {
+ "op": "add",
+ "symbol": "TEST",
+ "order_id": 1,
+ "side": "BUY",
+ "price": "100",
+ "quantity": "2",
+ },
+ {
+ "op": "add",
+ "symbol": "TEST",
+ "order_id": 2,
+ "side": "BUY",
+ "price": "100",
+ "quantity": "3",
+ },
+ {
+ "op": "update",
+ "symbol": "TEST",
+ "order_id": 1,
+ "side": "BUY",
+ "price": "100",
+ "quantity": "4",
+ },
+ ]
+ assert result.target_category == "book_state"
+ assert result.one_minimal is True
+ assert result.budget_exhausted is False
+ assert result.report.divergence is not None
+ assert result.report.divergence.event_index == 3
+
+
+def test_minimizer_reports_an_exhausted_budget_without_claiming_minimality():
+ result = minimize_book_replay_failure(
+ generate_book_replay_trace(123, 18),
+ _ReorderingAdapter,
+ max_runs=1,
+ )
+
+ assert result.runs == 1
+ assert result.one_minimal is False
+ assert result.budget_exhausted is True
+ assert len(result.events) == 3
+
+
+def test_faulty_campaign_stops_and_minimizes_first_divergence():
+ result = run_book_replay_campaign(
+ _ReorderingAdapter,
+ seed=20260831,
+ traces=10,
+ events_per_trace=100,
+ )
+
+ assert result.conformant is False
+ assert result.failure is not None
+ assert len(result.traces) == 1
+ assert result.traces[0].report.compared_events == 3
+ assert len(result.failure.minimization.events) == 3
+ assert result.failure.minimization.one_minimal is True
+ assert result.failure_id == "failure-dfe5c23848b63211b655"
+ assert trace_sha256(result.failure.minimization.events) == (
+ "d9eeb771c7c469c0762c67be7dc76a63a09f36d46c1dd2cda1dbb3cbb47c0b29"
+ )
+ assert result.coverage.covered == (
+ "l3-add",
+ "same-level-fifo",
+ "same-level-upsize",
+ )
+
+
+def test_cli_campaign_writes_report_and_reduced_trace(tmp_path):
+ report_path = tmp_path / "campaign.json"
+ reduced_path = tmp_path / "reduced.jsonl"
+
+ exit_code = main(
+ [
+ "campaign",
+ "--output",
+ str(report_path),
+ "--reduced-events-output",
+ str(reduced_path),
+ "--seed",
+ "20260831",
+ "--traces",
+ "10",
+ "--events-per-trace",
+ "100",
+ "--candidate-cmd",
+ f"{sys.executable} {FAULTY_ADAPTER}",
+ ]
+ )
+
+ report = json.loads(report_path.read_text(encoding="utf-8"))
+ reduced = [
+ BookReplayEvent.from_mapping(json.loads(line))
+ for line in reduced_path.read_text(encoding="utf-8").splitlines()
+ ]
+ assert exit_code == 1
+ assert report["artifact_type"] == "tracebook.book-replay.campaign"
+ assert report["conformant"] is False
+ assert report["failure"]["minimized_event_count"] == 3
+ assert len(reduced) == 3
+
+
+def test_cli_minimize_writes_a_semantic_failure_artifact(tmp_path):
+ trace_path = tmp_path / "trace.jsonl"
+ report_path = tmp_path / "minimization.json"
+ reduced_path = tmp_path / "reduced.jsonl"
+ trace_path.write_text(
+ "".join(
+ json.dumps(event.to_dict(), sort_keys=True) + "\n"
+ for event in generate_book_replay_trace(123, 18)
+ ),
+ encoding="utf-8",
+ )
+
+ exit_code = main(
+ [
+ "minimize",
+ str(trace_path),
+ "--events-output",
+ str(reduced_path),
+ "--output",
+ str(report_path),
+ "--candidate-cmd",
+ f"{sys.executable} {FAULTY_ADAPTER}",
+ ]
+ )
+
+ report = json.loads(report_path.read_text(encoding="utf-8"))
+ assert exit_code == 1
+ assert report["artifact_type"] == "tracebook.book-replay.minimization"
+ assert report["target_category"] == "book_state"
+ assert report["minimized_event_count"] == 3
+ assert report["one_minimal"] is True
+ assert len(reduced_path.read_text(encoding="utf-8").splitlines()) == 3
+
+
+@pytest.mark.parametrize(
+ ("keyword", "value", "message"),
+ [
+ ("seed", -1, "seed must be an integer"),
+ ("traces", 0, "traces must be a positive integer"),
+ ("events_per_trace", 0, "events_per_trace must be a positive integer"),
+ ("max_minimize_runs", 0, "max_minimize_runs must be a positive integer"),
+ ],
+)
+def test_campaign_rejects_invalid_bounds(keyword, value, message):
+ arguments = {keyword: value}
+ with pytest.raises(BookReplayError, match=message):
+ run_book_replay_campaign(ReferenceBookReplayAdapter, **arguments)
diff --git a/tests/test_conformance.py b/tests/test_conformance.py
index ce8fa42..276f902 100644
--- a/tests/test_conformance.py
+++ b/tests/test_conformance.py
@@ -595,7 +595,7 @@ def test_external_adapter_rejects_an_invalid_complete_handshake():
report = run_conformance(
[],
lambda config: ExternalProcessAdapter(
- [sys.executable, "-c", script], config, timeout_seconds=1
+ [sys.executable, "-c", script], config, timeout_seconds=10
),
)
diff --git a/tests/test_conformance_exit_codes.py b/tests/test_conformance_exit_codes.py
index d6bf906..3466b49 100644
--- a/tests/test_conformance_exit_codes.py
+++ b/tests/test_conformance_exit_codes.py
@@ -301,7 +301,7 @@ def test_cli_returns_two_for_malformed_mid_run_stdout(tmp_path):
assert "not valid JSON" in payload["divergence"]["message"]
-def test_cli_returns_two_for_mid_run_timeout(tmp_path):
+def test_cli_returns_two_for_mid_run_timeout(tmp_path, monkeypatch):
events = tmp_path / "events.jsonl"
report_path = tmp_path / "report.json"
adapter = tmp_path / "timeout_adapter.py"
@@ -325,6 +325,16 @@ def test_cli_returns_two_for_mid_run_timeout(tmp_path):
encoding="utf-8",
)
+ class MidRunTimeoutFactory(conformance_cli.ExternalProcessAdapterFactory):
+ def __call__(self, config):
+ candidate = super().__call__(config)
+ # Exercise the response timeout after ready, not interpreter startup
+ # on a busy host. Startup failures intentionally have no run report.
+ candidate.timeout_seconds = 0.2
+ return candidate
+
+ monkeypatch.setattr(conformance_cli, "ExternalProcessAdapterFactory", MidRunTimeoutFactory)
+
exit_code = main(
[
"run",
@@ -332,7 +342,7 @@ def test_cli_returns_two_for_mid_run_timeout(tmp_path):
"--output",
str(report_path),
"--timeout",
- "0.2",
+ "10",
"--candidate",
sys.executable,
str(adapter),
@@ -342,7 +352,7 @@ def test_cli_returns_two_for_mid_run_timeout(tmp_path):
payload = json.loads(report_path.read_text(encoding="utf-8"))
assert exit_code == 2
assert payload["divergence"]["category"] == "protocol"
- assert "timed out" in payload["divergence"]["message"]
+ assert "timed out waiting for 'observation'" in payload["divergence"]["message"]
@pytest.mark.parametrize(
diff --git a/tests/test_conformance_shutdown.py b/tests/test_conformance_shutdown.py
new file mode 100644
index 0000000..cb611c0
--- /dev/null
+++ b/tests/test_conformance_shutdown.py
@@ -0,0 +1,122 @@
+"""A complete frame is valid evidence only when the peer exits cleanly in time."""
+
+import json
+import os
+import sys
+import time
+
+import pytest
+
+from tracebook.conformance import (
+ AdapterProtocolError,
+ ConformanceConfig,
+ ExternalProcessAdapter,
+ run_conformance,
+)
+from tracebook.conformance.exit_codes import exit_code_for_artifact
+
+
+def _shutdown_peer(delay, exit_code, *, handle_terminate=False, broken_snapshot=False):
+ script = """
+import json
+import signal
+import sys
+import time
+
+delay, exit_code, handle_terminate, broken_snapshot = json.loads(sys.argv[1])
+if handle_terminate:
+ signal.signal(signal.SIGTERM, lambda *_: sys.exit(0))
+hello = json.loads(sys.stdin.readline())
+print(json.dumps({
+ "type": "ready",
+ "protocol": hello["protocol"],
+ "protocol_version": hello["protocol_version"],
+ "engine": {"name": "shutdown-test", "version": "1", "language": "Python"},
+}), flush=True)
+for line in sys.stdin:
+ message = json.loads(line)
+ if message["type"] == "snapshot":
+ if broken_snapshot:
+ print("broken snapshot", flush=True)
+ time.sleep(60)
+ else:
+ print(json.dumps({"type": "snapshot", "index": 0, "state": {"books": []}}), flush=True)
+ elif message["type"] == "finish":
+ print(json.dumps({"type": "complete", "event_count": message["event_count"]}), flush=True)
+ time.sleep(delay)
+ sys.exit(exit_code)
+"""
+ return ExternalProcessAdapter(
+ [
+ sys.executable,
+ "-I",
+ "-u",
+ "-c",
+ script,
+ json.dumps([delay, exit_code, handle_terminate, broken_snapshot]),
+ ],
+ ConformanceConfig(),
+ timeout_seconds=10,
+ )
+
+
+def test_close_allows_configured_time_for_clean_exit():
+ adapter = _shutdown_peer(0.75, 0)
+
+ adapter.close()
+ adapter.close()
+
+ assert adapter._process.returncode == 0
+
+
+def test_close_rejects_nonzero_exit_after_complete():
+ adapter = _shutdown_peer(0, 7)
+
+ with pytest.raises(AdapterProtocolError, match="exited with code 7 after complete"):
+ adapter.close()
+
+ assert adapter._process.returncode == 7
+
+
+@pytest.mark.skipif(os.name == "nt", reason="requires a catchable POSIX SIGTERM")
+def test_close_rejects_timeout_even_when_termination_exits_zero():
+ adapter = _shutdown_peer(60, 0, handle_terminate=True)
+ # Give startup its own generous budget before testing just the exit bound.
+ adapter.timeout_seconds = 0.5
+
+ with pytest.raises(AdapterProtocolError, match="timed out exiting after complete"):
+ adapter.close()
+
+ assert adapter._process.returncode == 0
+ adapter.close()
+
+
+def test_forced_shutdown_invalidates_otherwise_conformant_report():
+ adapter = _shutdown_peer(60, 0, handle_terminate=True)
+ adapter.timeout_seconds = 0.5
+
+ report = run_conformance([], lambda config: adapter)
+
+ assert report.conformant is False
+ assert report.divergence.kind == "adapter_close_error"
+ assert "timed out exiting after complete" in report.divergence.message
+ assert report.operational_failure is True
+ assert exit_code_for_artifact(report.to_dict()) == 2
+ assert adapter._process.returncode is not None
+
+
+def test_broken_stream_keeps_original_error_and_uses_prompt_cleanup():
+ adapter = _shutdown_peer(60, 0, handle_terminate=True, broken_snapshot=True)
+ started = time.monotonic()
+
+ report = run_conformance([], lambda config: adapter)
+
+ assert report.conformant is False
+ assert report.divergence.kind == "adapter_error"
+ assert "not valid JSON" in report.divergence.message
+ assert report.divergence.close_error is None
+ assert report.operational_failure is True
+ assert exit_code_for_artifact(report.to_dict()) == 2
+ assert adapter._process.returncode is not None
+ # A broken stream should not consume the healthy peer's ten-second budget.
+ assert time.monotonic() - started < 5
diff --git a/tests/test_dashboard_demo_construction.py b/tests/test_dashboard_demo_construction.py
index 7edf45e..52c2732 100644
--- a/tests/test_dashboard_demo_construction.py
+++ b/tests/test_dashboard_demo_construction.py
@@ -29,8 +29,12 @@ def __getattr__(self, name):
return lambda *args, **kwargs: (name, args, kwargs)
class FakeDash:
+ last_args = None
+ last_kwargs = None
+
def __init__(self, *args, **kwargs):
- pass
+ FakeDash.last_args = args
+ FakeDash.last_kwargs = kwargs
def callback(self, *args, **kwargs):
return lambda func: func
@@ -54,6 +58,20 @@ def callback(self, *args, **kwargs):
monkeypatch.setitem(sys.modules, "pandas", pandas_module)
+def test_dashboard_sets_tracebook_browser_title(monkeypatch):
+ _install_dashboard_dependency_stubs(monkeypatch)
+ sys.modules.pop("tracebook.visualization.dashboard", None)
+ dashboard_module = importlib.import_module("tracebook.visualization.dashboard")
+
+ try:
+ dashboard_module.PerformanceDashboard()
+ assert dashboard_module.dash.Dash.last_kwargs["title"] == (
+ "Tracebook · Performance Dashboard"
+ )
+ finally:
+ sys.modules.pop("tracebook.visualization.dashboard", None)
+
+
def test_dashboard_demo_mode_constructs_engine_and_dashboard_without_running_server(
monkeypatch,
):
diff --git a/tests/test_distribution_privacy.py b/tests/test_distribution_privacy.py
new file mode 100644
index 0000000..c8bbec9
--- /dev/null
+++ b/tests/test_distribution_privacy.py
@@ -0,0 +1,198 @@
+"""Release archives must not inherit private files or stale source inventories."""
+
+from __future__ import annotations
+
+import io
+import os
+import subprocess
+import sys
+import tarfile
+import zipfile
+import zlib
+from pathlib import Path
+
+import pytest
+
+from tools.verify_distribution_privacy import PrivacyError, main, verify_artifact
+
+ROOT = Path(__file__).resolve().parents[1]
+
+
+def test_privacy_gate_handles_corrupt_deflated_inventory(tmp_path, capsys):
+ artifact = tmp_path / "corrupt.whl"
+ with zipfile.ZipFile(artifact, "w", compression=zipfile.ZIP_DEFLATED) as archive:
+ archive.writestr("fixture.egg-info/SOURCES.txt", "pyproject.toml\n")
+ # Preserve the ZIP index, but corrupt the compressed block's type bits.
+ data = bytearray(artifact.read_bytes())
+ filename_length = int.from_bytes(data[26:28], "little")
+ extra_length = int.from_bytes(data[28:30], "little")
+ data[30 + filename_length + extra_length] = 7
+ artifact.write_bytes(data)
+ with pytest.raises(zlib.error):
+ verify_artifact(artifact)
+ assert main([str(artifact)]) == 1
+ assert "distribution privacy check failed" in capsys.readouterr().err
+
+
+PRIVATE_PATHS = (
+ "experiments/private/agent-qualification-v2/gold.json",
+ "experiments/private/agent-qualification-v2/execution/controller.py",
+ ".agents/skills/local/SKILL.md",
+ ".codex/settings.json",
+ "openwiki/quickstart.md",
+ "graphify-out/graph.json",
+ ".local-tools/helper.py",
+ "docs/openwiki.md",
+ "docs/graphify.md",
+ "AGENTS.md",
+ "CLAUDE.md",
+ "skills-lock.json",
+ "examples/AGENTS.md",
+)
+
+
+def _write_archive(path: Path, files: dict[str, bytes]) -> None:
+ if path.name.endswith(".whl"):
+ with zipfile.ZipFile(path, "w") as archive:
+ for name, data in files.items():
+ archive.writestr(name, data)
+ else:
+ with tarfile.open(path, "w:gz") as archive:
+ for name, data in files.items():
+ member = tarfile.TarInfo(f"privacy_fixture-1.0/{name}")
+ member.size = len(data)
+ archive.addfile(member, io.BytesIO(data))
+
+
+@pytest.mark.parametrize("suffix", (".whl", ".tar.gz"))
+@pytest.mark.parametrize(
+ "private_path",
+ (
+ *PRIVATE_PATHS,
+ "tracebook.data/data/experiments/private/gold.json",
+ "src/tracebook/.agents/x.py",
+ ),
+)
+def test_privacy_gate_rejects_private_archive_members(tmp_path, suffix, private_path):
+ artifact = tmp_path / f"contaminated{suffix}"
+ _write_archive(artifact, {private_path: b"synthetic private sentinel\n"})
+
+ with pytest.raises(PrivacyError, match="private path"):
+ verify_artifact(artifact)
+
+
+@pytest.mark.parametrize("suffix", (".whl", ".tar.gz"))
+def test_privacy_gate_rejects_stale_sources_without_private_members(tmp_path, suffix):
+ artifact = tmp_path / f"stale-inventory{suffix}"
+ _write_archive(
+ artifact,
+ {
+ "pyproject.toml": b"# synthetic public file\n",
+ "src/privacy_fixture.egg-info/SOURCES.txt": (
+ b"pyproject.toml\nexperiments/private/synthetic-gold.json\n"
+ ),
+ },
+ )
+
+ with pytest.raises(PrivacyError, match="SOURCES.txt.*experiments/private/synthetic-gold.json"):
+ verify_artifact(artifact)
+
+
+def test_privacy_gate_cli_checks_every_artifact_and_fails_closed(tmp_path, capsys):
+ clean = tmp_path / "clean.whl"
+ leaked = tmp_path / "leaked.tar.gz"
+ _write_archive(clean, {"tracebook/__init__.py": b"# synthetic public source\n"})
+ _write_archive(leaked, {"experiments/private/gold.json": b"{}\n"})
+
+ assert main([str(clean), str(leaked)]) == 1
+ assert "leaked.tar.gz" in capsys.readouterr().err
+ assert main([str(clean)]) == 0
+ assert "verified for 1 artifacts" in capsys.readouterr().out
+ assert main([str(tmp_path / "missing.whl")]) == 1
+
+
+def test_real_manifest_excludes_synthetic_private_tree_and_stale_cache(tmp_path):
+ """Build only synthetic files with the real manifest, never the private tree."""
+
+ project = tmp_path / "source"
+ project.mkdir()
+ (project / "pyproject.toml").write_text(
+ """\
+[build-system]
+requires = ["setuptools", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "privacy-fixture"
+version = "1.0"
+readme = "README.md"
+
+[tool.setuptools]
+package-dir = { "" = "src" }
+include-package-data = true
+
+[tool.setuptools.packages.find]
+where = ["src"]
+""",
+ encoding="utf-8",
+ )
+ (project / "MANIFEST.in").write_text(
+ (ROOT / "MANIFEST.in").read_text(encoding="utf-8"), encoding="utf-8"
+ )
+ public_files = {
+ "README.md": "Synthetic build regression.\n",
+ "LICENSE": "MIT\n",
+ "src/tracebook/__init__.py": "# synthetic public package\n",
+ "src/tracebook/py.typed": "",
+ "experiments/public-evidence.json": '{"synthetic": true}\n',
+ "experiments/public_runner.py": "# synthetic public runner\n",
+ "integrations/intrepid_orderbook/main.go": "package main\n",
+ "integrations/intrepid_orderbook/go.mod": "module privacy-fixture\n",
+ "integrations/intrepid_orderbook/go.sum": "synthetic public lockfile\n",
+ }
+ for path, content in public_files.items():
+ target = project / path
+ target.parent.mkdir(parents=True, exist_ok=True)
+ target.write_text(content, encoding="utf-8")
+ for path in PRIVATE_PATHS:
+ target = project / path
+ target.parent.mkdir(parents=True, exist_ok=True)
+ target.write_text("synthetic private sentinel\n", encoding="utf-8")
+
+ # setuptools reads an existing inventory before applying MANIFEST.in.
+ # Seed paths that ordinary recursive includes would not otherwise discover.
+ cache = project / "src/privacy_fixture.egg-info/SOURCES.txt"
+ cache.parent.mkdir(parents=True)
+ cache.write_text("\n".join((*public_files, *PRIVATE_PATHS)) + "\n", encoding="utf-8")
+ environment = os.environ.copy()
+ environment["PIP_NO_INDEX"] = "1"
+ result = subprocess.run(
+ [
+ sys.executable,
+ "-m",
+ "build",
+ "--sdist",
+ "--wheel",
+ "--no-isolation",
+ "--outdir",
+ str(tmp_path / "dist"),
+ str(project),
+ ],
+ env=environment,
+ capture_output=True,
+ text=True,
+ )
+ assert result.returncode == 0, result.stdout + result.stderr
+ artifacts = sorted((tmp_path / "dist").iterdir())
+ assert len(artifacts) == 2
+ for artifact in artifacts:
+ verify_artifact(artifact)
+
+ sdist = next(path for path in artifacts if path.name.endswith(".tar.gz"))
+ with tarfile.open(sdist, "r:gz") as archive:
+ members = {name.partition("/")[2] for name in archive.getnames()}
+ assert set(public_files) <= members
+ assert not set(PRIVATE_PATHS) & members
+ cached_paths = set(cache.read_text(encoding="utf-8").splitlines())
+ assert not set(PRIVATE_PATHS) & cached_paths
+ assert all((project / path).is_file() for path in PRIVATE_PATHS)
diff --git a/tests/test_external_integrations.py b/tests/test_external_integrations.py
index b56816a..e099202 100644
--- a/tests/test_external_integrations.py
+++ b/tests/test_external_integrations.py
@@ -1,3 +1,4 @@
+import importlib.util
import sys
from pathlib import Path
@@ -14,6 +15,17 @@
UPSTREAM_REPOSITORY,
PythonMatchingEngineAdapter,
)
+from integrations.nautilus_trader.adapter import (
+ UPSTREAM_COMMIT as NAUTILUS_COMMIT,
+ UPSTREAM_REPOSITORY as NAUTILUS_REPOSITORY,
+ UPSTREAM_TAG as NAUTILUS_TAG,
+ UPSTREAM_VERSION as NAUTILUS_VERSION,
+)
+from tracebook.book_replay import (
+ ExternalBookReplayAdapter,
+ load_book_replay_events,
+ run_book_replay,
+)
from tracebook.conformance import (
AdapterProtocolError,
ConformanceConfig,
@@ -27,6 +39,8 @@
INTEGRATION = ROOT / "integrations" / "python_matching_engine"
RUST_INTEGRATION = ROOT / "integrations" / "orderbook_rs"
GOCRONX_INTEGRATION = ROOT / "integrations" / "gocronx_matcher"
+NAUTILUS_INTEGRATION = ROOT / "integrations" / "nautilus_trader"
+INTREPID_INTEGRATION = ROOT / "integrations" / "intrepid_orderbook"
RUST_PROTOCOL = ROOT / "integrations" / "rust_protocol"
@@ -59,6 +73,57 @@ def test_python_matching_engine_adapter_explains_a_missing_upstream(monkeypatch,
)
+def test_nautilus_book_replay_integration_is_version_and_source_pinned():
+ readme = (NAUTILUS_INTEGRATION / "README.md").read_text(encoding="utf-8")
+ workflow = (ROOT / ".github" / "workflows" / "nautilus-trader-book-replay.yml").read_text(
+ encoding="utf-8"
+ )
+
+ assert NAUTILUS_REPOSITORY == "https://github.com/nautechsystems/nautilus_trader.git"
+ assert NAUTILUS_TAG == "v2.0.0rc3"
+ assert NAUTILUS_COMMIT == "648970ce64a304d93da0a29320cb6e19b905fa39"
+ assert NAUTILUS_VERSION == "2.0.0rc3"
+ assert "LGPL-3.0-only" in readme
+ assert "does not claim" in readme
+ assert '"nautilus-trader==2.0.0rc3"' in workflow
+ assert "actions/setup-python@v7" in workflow
+ assert 'python -m pip install -e . "nautilus-trader==2.0.0rc3" "pytest>=9.1.1"' in workflow
+ assert "python -m pytest" in workflow
+ assert "9e0af6be935dce940a87497788c7a9a799c71f05e34a0204c9d294fce611b002" in workflow
+ assert "sha256:09b1599eaeb474d98617acc7869ace26759ed5ab8350803f06197ef572864bab" in workflow
+ assert "failure-dfe5c23848b63211b655" in readme
+ assert (NAUTILUS_INTEGRATION / "faulty_adapter.py").is_file()
+ assert (NAUTILUS_INTEGRATION / "regressions" / "upsize-requeue-reduced.jsonl").is_file()
+ assert (NAUTILUS_INTEGRATION / "regressions" / "upsize-requeue-failure.json").is_file()
+
+
+@pytest.mark.skipif(
+ importlib.util.find_spec("nautilus_trader") is None,
+ reason="the optional pinned NautilusTrader runtime is not installed",
+)
+def test_pinned_nautilus_runtime_conforms_to_book_replay_profile():
+ events = load_book_replay_events(
+ ROOT / "src" / "tracebook" / "book_replay" / "fixtures" / "l3-book-replay-v1.jsonl"
+ )
+
+ report = run_book_replay(
+ events,
+ lambda config: ExternalBookReplayAdapter(
+ [sys.executable, str(NAUTILUS_INTEGRATION / "adapter.py")],
+ config,
+ timeout_seconds=10,
+ ),
+ )
+
+ assert report.conformant is True
+ assert report.compared_events == 17
+ assert report.candidate_engine.version == NAUTILUS_VERSION
+ assert (
+ report.final_state_hash
+ == "9e0af6be935dce940a87497788c7a9a799c71f05e34a0204c9d294fce611b002"
+ )
+
+
def test_orderbook_rs_integration_pins_engine_toolchain_and_dependency_graph():
cargo = (RUST_INTEGRATION / "Cargo.toml").read_text(encoding="utf-8")
lock = (RUST_INTEGRATION / "Cargo.lock").read_text(encoding="utf-8")
@@ -138,6 +203,39 @@ def test_gocronx_matcher_integration_is_pinned_qualified_and_honest_about_assump
assert "actions/upload-artifact@v7" in workflow
+def test_intrepid_orderbook_is_pinned_qualified_and_retains_the_fok_boundary():
+ go_mod = (INTREPID_INTEGRATION / "go.mod").read_text(encoding="utf-8")
+ go_sum = (INTREPID_INTEGRATION / "go.sum").read_text(encoding="utf-8")
+ readme = (INTREPID_INTEGRATION / "README.md").read_text(encoding="utf-8")
+ workflow = (ROOT / ".github" / "workflows" / "intrepid-orderbook.yml").read_text(
+ encoding="utf-8"
+ )
+ root_readme = (ROOT / "README.md").read_text(encoding="utf-8")
+ architecture = (ROOT / "docs" / "architecture.md").read_text(encoding="utf-8")
+
+ assert "github.com/intrepidkarthi/orderbook v0.26.0" in go_mod
+ assert "github.com/intrepidkarthi/orderbook v0.26.0 h1:" in go_sum
+ assert "actions/setup-python@v7" in workflow
+ assert "51d480cdb68b9989febb0b075d291cf891f425b3" in readme
+ assert "submitted-order versus submitted-order" in readme
+ assert "Inputs requiring more than six" in readme
+ assert "non-zero decimal places are explicitly rejected" in readme
+ assert "rounds output quantities only" in readme
+ assert "maps to a distinct native account" in readme
+ assert "incoming quantity times each potentially crossed maker's" in readme
+ assert "6/9" in readme
+ assert "failure-1ce2954857800b3a068d" in readme
+ assert "sha256:f8db775baabe7e665d45bbb920a67e43d405acf445de4951617df68ffa24eb69" in workflow
+ assert "sha256:08bf641bfa57da1892d744c08bca0be00f5ac1ade580ae8fcf7f195f0fcad6eb" in workflow
+ assert "sha256:81a095f9804852ff7035f4c3d428a51ae3d0e8e4b81ec335128c6af15b6f65bd" in workflow
+ assert "$.observation.outcome.reason" in workflow
+ assert "intrepidkarthi/orderbook` v0.26.0" in root_readme
+ assert "integrations/intrepid_orderbook/" in architecture
+ assert (INTREPID_INTEGRATION / "adapter_test.go").is_file()
+ assert (INTREPID_INTEGRATION / "regressions" / "fok-rejection-reduced.jsonl").is_file()
+ assert (INTREPID_INTEGRATION / "regressions" / "fok-rejection-failure.json").is_file()
+
+
def test_native_adapters_share_one_rust_protocol_contract():
shared_source = (RUST_PROTOCOL / "src" / "lib.rs").read_text(encoding="utf-8")
shared_server = (RUST_PROTOCOL / "src" / "server.rs").read_text(encoding="utf-8")
@@ -162,6 +260,10 @@ def test_source_manifest_includes_native_integration_files():
assert "recursive-include integrations/orderbook_rs/src *.rs" in manifest
assert package_data["tracebook.conformance.fixtures.v2"] == ["*.json", "*.jsonl"]
assert "recursive-include integrations *.py *.md *.json *.jsonl" in manifest
+ assert "recursive-include integrations/intrepid_orderbook *.go" in manifest
+ assert "include integrations/intrepid_orderbook/go.mod" in manifest
+ assert "include integrations/intrepid_orderbook/go.sum" in manifest
+ assert "recursive-include src/tracebook/book_replay/fixtures *.jsonl" in manifest
assert "include integrations/orderbook_rs/Cargo.lock" in manifest
assert "prune integrations/orderbook_rs/target" in manifest
assert "recursive-include integrations/gocronx_matcher/src *.rs" in manifest
diff --git a/tests/test_integration_workflow_coverage.py b/tests/test_integration_workflow_coverage.py
new file mode 100644
index 0000000..eccf8ec
--- /dev/null
+++ b/tests/test_integration_workflow_coverage.py
@@ -0,0 +1,195 @@
+"""Keep native CI attached to the reference/runtime code it actually executes."""
+
+import ast
+import os
+import re
+from pathlib import Path
+
+import pytest
+
+ROOT = Path(__file__).resolve().parents[1]
+WORKFLOWS = {
+ "intrepid-orderbook": "intrepid_orderbook",
+ "nautilus-trader-book-replay": "nautilus_trader",
+ "orderbook-rs": "orderbook_rs",
+ "gocronx-matcher": "gocronx_matcher",
+ "python-matching-engine": "python_matching_engine",
+}
+
+
+def _mapping_block(lines, key):
+ """Read a block in our workflows' block-style YAML subset, not job text."""
+ lines = [line for line in lines if line.strip() and not line.lstrip().startswith("#")]
+ assert lines, f"missing YAML mapping: {key}"
+ base = min(len(line) - len(line.lstrip()) for line in lines)
+ for index, line in enumerate(lines):
+ indent = len(line) - len(line.lstrip())
+ if indent != base or not re.fullmatch(rf"['\"]?{key}['\"]?:\s*(?:#.*)?", line.strip()):
+ continue
+ block = []
+ for child in lines[index + 1 :]:
+ if len(child) - len(child.lstrip()) <= indent:
+ break
+ block.append(child)
+ return block
+ raise AssertionError(f"missing YAML mapping: {key}")
+
+
+def _parse_pull_request_paths(source):
+ # Intentionally support block mappings and quoted positive path sequences
+ # used here. Unsupported YAML fails closed, without a new runtime dependency.
+ block = source.splitlines()
+ for key in ("on", "pull_request", "paths"):
+ block = _mapping_block(block, key)
+ assert block and all(line.strip().startswith("- ") for line in block)
+ patterns = [ast.literal_eval(line.strip().removeprefix("- ")) for line in block]
+ assert all(isinstance(pattern, str) and not pattern.startswith("!") for pattern in patterns)
+ return patterns
+
+
+def _pull_request_paths(workflow):
+ source = (ROOT / ".github" / "workflows" / f"{workflow}.yml").read_text(encoding="utf-8")
+ return _parse_pull_request_paths(source)
+
+
+@pytest.mark.parametrize("indent", [2, 4])
+def test_path_reader_allows_comments_blank_lines_and_other_trigger_keys(indent):
+ space = " " * indent
+ source = "\n".join(
+ [
+ "on:",
+ f"{space}pull_request: # keep native proofs attached",
+ f"{space * 2}branches:",
+ f"{space * 3}- 'main'",
+ "",
+ f"{space * 2}# runtime inputs",
+ f"{space * 2}paths:",
+ f"{space * 3}- 'src/**' # reference",
+ "",
+ f"{space * 3}# shared crate",
+ f'{space * 3}- "integrations/**"',
+ f"{space}push:",
+ f"{space * 2}paths:",
+ f"{space * 3}- 'ignored/**'",
+ "jobs:",
+ f"{space}pull_request:",
+ f"{space * 2}paths:",
+ f"{space * 3}- 'wrong/**'",
+ ]
+ )
+ assert _parse_pull_request_paths(source) == ["src/**", "integrations/**"]
+
+
+def test_path_reader_cannot_use_paths_from_jobs_or_another_trigger():
+ for source in (
+ "jobs:\n pull_request:\n paths:\n - 'src/**'",
+ "on:\n push:\n paths:\n - 'src/**'\n"
+ " pull_request:\n branches:\n - 'main'",
+ ):
+ with pytest.raises(AssertionError, match="missing YAML mapping"):
+ _parse_pull_request_paths(source)
+
+
+def _matches(path, pattern):
+ # The checked workflows use only literal paths and positive * / ** globs.
+ # Unlike fnmatch, a single * must not match a directory separator.
+ expression = re.escape(pattern).replace(r"\*\*", ".*").replace(r"\*", "[^/]*")
+ return re.fullmatch(expression, path) is not None
+
+
+def _require_covered(workflow, paths):
+ patterns = _pull_request_paths(workflow)
+ missing = sorted(path for path in paths if not any(_matches(path, p) for p in patterns))
+ assert not missing, f"{workflow} misses direct runtime inputs: {missing}"
+
+
+def _python_sources(relative_root):
+ return [path.relative_to(ROOT).as_posix() for path in (ROOT / relative_root).rglob("*.py")]
+
+
+@pytest.mark.parametrize("workflow", WORKFLOWS)
+def test_integration_workflows_cover_installed_reference_runtime(workflow):
+ # Even L3 imports conformance.model and tracebook's package initializer.
+ # Every CLOB proof directly runs the shared Python reference matching engine.
+ paths = [
+ "pyproject.toml",
+ "MANIFEST.in",
+ "src/tracebook/__init__.py",
+ "src/tracebook/_version.py",
+ f".github/workflows/{workflow}.yml",
+ "tests/test_integration_workflow_coverage.py",
+ ]
+ for package in ("core", "events", "conformance"):
+ paths.extend(_python_sources(f"src/tracebook/{package}"))
+ paths.extend(
+ path.relative_to(ROOT).as_posix()
+ for path in (ROOT / "src/tracebook/conformance/fixtures").rglob("*")
+ if path.suffix in {".json", ".jsonl"}
+ )
+ _require_covered(workflow, paths)
+
+
+@pytest.mark.parametrize("workflow", WORKFLOWS)
+def test_integration_workflows_cover_own_source_pins_and_retained_evidence(workflow):
+ integration = ROOT / "integrations" / WORKFLOWS[workflow]
+ paths = []
+ for directory, directories, filenames in os.walk(integration):
+ directories[:] = [name for name in directories if name not in {"target", "__pycache__"}]
+ for name in filenames:
+ path = Path(directory) / name
+ if path.suffix in {".go", ".rs", ".py", ".json", ".jsonl"} or name in {
+ "go.mod",
+ "go.sum",
+ "Cargo.toml",
+ "Cargo.lock",
+ "rust-toolchain.toml",
+ }:
+ paths.append(path.relative_to(ROOT).as_posix())
+ assert paths
+ _require_covered(workflow, paths)
+
+
+@pytest.mark.parametrize("workflow", ["orderbook-rs", "gocronx-matcher"])
+def test_rust_workflows_cover_the_shared_protocol_crate(workflow):
+ paths = ["integrations/rust_protocol/Cargo.toml", "integrations/rust_protocol/Cargo.lock"]
+ paths.extend(
+ path.relative_to(ROOT).as_posix()
+ for path in (ROOT / "integrations/rust_protocol/src").rglob("*.rs")
+ )
+ _require_covered(workflow, paths)
+
+
+def test_nautilus_workflow_covers_book_replay_and_executed_test_helpers():
+ paths = _python_sources("src/tracebook/book_replay")
+ paths.extend(
+ [
+ "src/tracebook/book_replay/fixtures/l3-book-replay-v1.jsonl",
+ "examples/book_replay_adapter.py",
+ "integrations/__init__.py",
+ "integrations/python_matching_engine/__init__.py",
+ "integrations/python_matching_engine/adapter.py",
+ "tests/conftest.py",
+ "tests/test_book_replay.py",
+ "tests/test_book_replay_campaign.py",
+ "tests/test_book_replay_artifacts.py",
+ "tests/fixtures/faulty_book_replay_adapter.py",
+ "tests/test_external_integrations.py",
+ ]
+ )
+ _require_covered("nautilus-trader-book-replay", paths)
+
+
+def test_orderbook_rs_workflow_covers_the_executed_flash_bridge():
+ _require_covered(
+ "orderbook-rs",
+ [
+ "integrations/flash_benchmark/bridge.py",
+ "integrations/flash_benchmark/artifacts/orderbook-rs-issue-88-divergence.json",
+ ],
+ )
+
+
+def test_path_glob_matching_keeps_single_star_inside_one_directory():
+ assert _matches("src/tracebook/core/order.py", "src/tracebook/core/**")
+ assert _matches("tests/test_book_replay_campaign.py", "tests/test_book_replay*.py")
+ assert not _matches("tests/nested/test_book_replay.py", "tests/*.py")
diff --git a/tests/test_release_metadata.py b/tests/test_release_metadata.py
index 7e233e6..d1b85ee 100644
--- a/tests/test_release_metadata.py
+++ b/tests/test_release_metadata.py
@@ -47,6 +47,7 @@ def test_distribution_name_cli_and_typing_metadata_are_release_ready():
}
assert package_data["tracebook.corpus.fixtures"] == ["coinbase-btcusd-synthetic-v1/*"]
assert package_data["tracebook.conformance.fixtures.v1"] == ["*.json", "*.jsonl"]
+ assert package_data["tracebook.book_replay.fixtures"] == ["*.jsonl"]
assert (ROOT / "src" / "tracebook" / "py.typed").is_file()
assert (ROOT / ".github" / "workflows" / "release.yml").is_file()
@@ -184,7 +185,7 @@ def test_release_publishes_and_verifies_the_source_owner_before_the_facade():
)
-def test_sdist_excludes_local_navigation_material():
+def test_sdist_excludes_private_experiments_and_local_navigation_material():
manifest = (ROOT / "MANIFEST.in").read_text(encoding="utf-8")
for directive in (
@@ -195,6 +196,10 @@ def test_sdist_excludes_local_navigation_material():
"prune openwiki",
"prune graphify-out",
"prune .local-tools",
+ "prune experiments/private",
+ "prune .agents",
+ "prune .codex",
+ "global-exclude AGENTS.md CLAUDE.md skills-lock.json",
"recursive-include packaging *.md",
"include packaging/tracebook-sim/pyproject.toml",
"include packaging/tracebook-sim/LICENSE",
@@ -202,6 +207,25 @@ def test_sdist_excludes_local_navigation_material():
):
assert directive in manifest
+ for path in (
+ "Makefile",
+ ".github/workflows/ci.yml",
+ ".github/workflows/release.yml",
+ "CONTRIBUTING.md",
+ "docs/release.md",
+ ):
+ workflow = (ROOT / path).read_text(encoding="utf-8")
+ assert (
+ "tools/verify_distribution_privacy.py dist/conformance/* dist/simulator/*" in workflow
+ )
+
+
+def test_local_typecheck_uses_the_active_python_like_ci():
+ for path in ("Makefile", "CONTRIBUTING.md", "docs/release.md"):
+ contents = (ROOT / path).read_text(encoding="utf-8")
+ assert "mypy --python-version" in contents
+ assert "sys.version_info[:2]" in contents
+
def test_public_install_surfaces_warn_before_the_legacy_ownership_handoff():
install_surfaces = (
diff --git a/tests/test_release_virtualenvs.py b/tests/test_release_virtualenvs.py
new file mode 100644
index 0000000..403981c
--- /dev/null
+++ b/tests/test_release_virtualenvs.py
@@ -0,0 +1,47 @@
+"""Release helpers must preserve the platform defaults of `python -m venv`."""
+
+from pathlib import Path
+from types import SimpleNamespace
+
+import pytest
+
+from tools import smoke_conformance_wheel, verify_distribution_split
+
+
+@pytest.mark.parametrize("helper", [smoke_conformance_wheel, verify_distribution_split])
+@pytest.mark.parametrize("platform, use_symlinks", [("posix", True), ("nt", False)])
+def test_release_helpers_preserve_platform_venv_link_mode(
+ tmp_path, monkeypatch, helper, platform, use_symlinks
+):
+ class EnvironmentRequested(Exception):
+ pass
+
+ options = {}
+ destinations = []
+
+ class RecordingBuilder:
+ def __init__(self, **kwargs):
+ options.update(kwargs)
+
+ def create(self, destination):
+ destinations.append(Path(destination))
+ # Stop before installation or qualification: this unit test checks
+ # the actual builder call, not copies of the configuration text.
+ raise EnvironmentRequested
+
+ monkeypatch.setattr(helper, "os", SimpleNamespace(name=platform))
+ monkeypatch.setattr(helper.venv, "EnvBuilder", RecordingBuilder)
+
+ with pytest.raises(EnvironmentRequested):
+ if helper is verify_distribution_split:
+ helper._new_environment(tmp_path, "environment")
+ else:
+ wheel = tmp_path / "example.whl"
+ adapter = tmp_path / "adapter.py"
+ wheel.write_bytes(b"not installed by this test")
+ adapter.write_bytes(b"not executed by this test")
+ helper.main([str(wheel), "--adapter", str(adapter)])
+
+ assert options == {"with_pip": True, "clear": True, "symlinks": use_symlinks}
+ assert len(destinations) == 1
+ assert not destinations[0].exists()
diff --git a/tools/smoke_conformance_wheel.py b/tools/smoke_conformance_wheel.py
index 620e243..b58e690 100644
--- a/tools/smoke_conformance_wheel.py
+++ b/tools/smoke_conformance_wheel.py
@@ -51,9 +51,14 @@ def _assert_lightweight_environment(python: Path) -> None:
raise SystemExit(f"{package} unexpectedly installed in isolated environment")
import tracebook.conformance
+import tracebook.book_replay
import importlib.metadata
+from importlib import resources
assert importlib.metadata.version("tracebook-conformance") == tracebook.__version__
+assert resources.files("tracebook.book_replay.fixtures").joinpath(
+ "l3-book-replay-v1.jsonl"
+).is_file()
"""
_run([str(python), "-c", probe])
@@ -136,7 +141,9 @@ def main(argv: Sequence[str] | None = None) -> int:
root = Path(temporary)
environment = root / "venv"
output = root / "qualification"
- venv.EnvBuilder(with_pip=True, clear=True).create(environment)
+ # Match `python -m venv`: keep executable-relative runtime libraries
+ # reachable on POSIX, while retaining the default Windows copy mode.
+ venv.EnvBuilder(with_pip=True, clear=True, symlinks=os.name != "nt").create(environment)
python = _environment_python(environment)
conformance = _environment_script(environment, "tracebook-conformance")
diff --git a/tools/verify_distribution_privacy.py b/tools/verify_distribution_privacy.py
new file mode 100644
index 0000000..f67fadf
--- /dev/null
+++ b/tools/verify_distribution_privacy.py
@@ -0,0 +1,124 @@
+#!/usr/bin/env python3
+"""Reject private experiment and local navigation paths in release artifacts.
+
+Checks archive member names and setuptools' SOURCES.txt inventories without
+extracting artifacts. The latter prevents publishing stale local path metadata
+even when the referenced private files are absent from the archive.
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+import tarfile
+import zipfile
+import zlib
+from pathlib import Path, PurePosixPath, PureWindowsPath
+from typing import Sequence
+
+PRIVATE_ROOTS = (
+ "experiments/private",
+ ".agents",
+ ".codex",
+ "openwiki",
+ "graphify-out",
+ ".local-tools",
+)
+PRIVATE_FILES = {"docs/openwiki.md", "docs/graphify.md"}
+LOCAL_POINTER_NAMES = {"AGENTS.md", "CLAUDE.md", "skills-lock.json"}
+
+
+class PrivacyError(ValueError):
+ """A release artifact contains private paths or cannot be checked."""
+
+
+def _check_path(path: str, *, context: str) -> None:
+ normalized = path.rstrip("/")
+ parts = normalized.split("/")
+ if (
+ not normalized
+ or "\\" in normalized
+ or "\x00" in normalized
+ or any(part in {"", ".", ".."} for part in parts)
+ or PureWindowsPath(normalized).drive
+ ):
+ raise PrivacyError(f"unsafe path in {context}: {path!r}")
+ if (
+ any(normalized == path or normalized.endswith("/" + path) for path in PRIVATE_FILES)
+ or PurePosixPath(normalized).name in LOCAL_POINTER_NAMES
+ or any(
+ parts[index : index + len(root.split("/"))] == root.split("/")
+ for root in PRIVATE_ROOTS
+ for index in range(len(parts))
+ )
+ ):
+ raise PrivacyError(f"private path in {context}: {normalized}")
+
+
+def _check_source_inventory(data: bytes, *, context: str) -> None:
+ try:
+ inventory = data.decode("utf-8")
+ except UnicodeDecodeError as error:
+ raise PrivacyError(f"cannot decode source inventory in {context}") from error
+ for path in inventory.splitlines():
+ if path:
+ _check_path(path, context=context)
+
+
+def verify_artifact(artifact: Path) -> None:
+ """Check one built wheel or gzip source distribution, without extraction."""
+
+ if artifact.name.endswith(".whl"):
+ with zipfile.ZipFile(artifact) as wheel:
+ for member in wheel.infolist():
+ _check_path(member.filename, context=artifact.name)
+ if PurePosixPath(member.filename).name == "SOURCES.txt" and not member.is_dir():
+ _check_source_inventory(
+ wheel.read(member), context=f"{artifact.name}:{member.filename}"
+ )
+ elif artifact.name.endswith(".tar.gz"):
+ with tarfile.open(artifact, "r:gz") as archive:
+ roots = set()
+ for source_member in archive:
+ _check_path(source_member.name, context=artifact.name)
+ root, separator, relative = source_member.name.rstrip("/").partition("/")
+ roots.add(root)
+ if not separator:
+ if not source_member.isdir():
+ raise PrivacyError(
+ f"sdist member has no project root: {source_member.name!r}"
+ )
+ continue
+ _check_path(relative, context=artifact.name)
+ if PurePosixPath(relative).name == "SOURCES.txt":
+ source = archive.extractfile(source_member) if source_member.isreg() else None
+ if source is None:
+ raise PrivacyError(
+ f"cannot inspect source inventory: {source_member.name!r}"
+ )
+ with source:
+ _check_source_inventory(
+ source.read(), context=f"{artifact.name}:{relative}"
+ )
+ if len(roots) != 1:
+ raise PrivacyError(f"sdist must contain one project root: {artifact.name}")
+ else:
+ raise PrivacyError(f"expected a wheel or .tar.gz sdist: {artifact}")
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("artifacts", type=Path, nargs="+")
+ args = parser.parse_args(argv)
+ try:
+ for artifact in args.artifacts:
+ verify_artifact(artifact)
+ except (PrivacyError, OSError, tarfile.TarError, zipfile.BadZipFile, zlib.error) as error:
+ print(f"distribution privacy check failed: {error}", file=sys.stderr)
+ return 1
+ print(f"Distribution privacy verified for {len(args.artifacts)} artifacts.")
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/tools/verify_distribution_split.py b/tools/verify_distribution_split.py
index ddb503e..764a5f7 100644
--- a/tools/verify_distribution_split.py
+++ b/tools/verify_distribution_split.py
@@ -574,7 +574,9 @@ def validate_legacy_wheel(
def _new_environment(root: Path, name: str) -> tuple[Path, Path]:
environment = root / name
- venv.EnvBuilder(with_pip=True, clear=True).create(environment)
+ # Match `python -m venv`: copying a POSIX executable can break its
+ # executable-relative libpython/framework loader paths (notably on macOS).
+ venv.EnvBuilder(with_pip=True, clear=True, symlinks=os.name != "nt").create(environment)
return environment, _environment_python(environment)